valor-lite 0.36.5__py3-none-any.whl → 0.37.5__py3-none-any.whl

valor_lite/profiling.py

@@ -1,374 +0,0 @@
-import json
-import math
-import multiprocessing as mp
-import resource
-import time
-from collections import deque
-from multiprocessing import Queue
-from typing import Any
-
-from tqdm import tqdm
-
-
-class BenchmarkError(Exception):
-    def __init__(
-        self, benchmark: str, error_type: str, error_message: str
-    ) -> None:
-        super().__init__(
-            f"'{benchmark}' raised '{error_type}' with the following message: {error_message}"
-        )
-
-
-def _timeit_subprocess(*args, __fn, __queue: Queue, **kwargs):
-    """
-    Multiprocessing subprocess that reports either runtime or errors.
-
-    This is handled within a subprocess to protect the benchmark against OOM errors.
-    """
-    try:
-        timer_start = time.perf_counter()
-        __fn(*args, **kwargs)
-        timer_end = time.perf_counter()
-        __queue.put(timer_end - timer_start)
-    except Exception as e:
-        __queue.put(e)
-
-
-def create_runtime_profiler(
-    time_limit: float | None,
-    repeat: int = 1,
-):
-    """
-    Creates a runtime profiler as a decorating function.
-
-    The profiler reports runtime of the wrapped function from a subprocess to protect against OOM errors.
-
-    Parameters
-    ----------
-    time_limit : float, optional
-        An optional time limit to constrain the benchmark.
-    repeat : int, default=1
-        The number of times to repeat the benchmark to produce an average runtime.
-    """
-    ctx = mp.get_context("spawn")
-
-    def decorator(fn):
-        def wrapper(*args, **kwargs):
-            # Record average runtime over repeated runs.
-            elapsed = 0
-            for _ in range(repeat):
-                q = ctx.Queue()
-                p = ctx.Process(
-                    target=_timeit_subprocess,
-                    args=args,
-                    kwargs={"__fn": fn, "__queue": q, **kwargs},
-                )
-                p.start()
-                p.join(timeout=time_limit)
-
-                # Check if computation finishes within the timeout
-                if p.is_alive():
-                    p.terminate()
-                    p.join()
-                    q.close()
-                    q.join_thread()
-                    raise TimeoutError(
-                        f"Function '{fn.__name__}' did not complete within {time_limit} seconds."
-                    )
-
-                # Retrieve the result
-                result = q.get(timeout=1)
-                if isinstance(result, Exception):
-                    raise result
-                elif isinstance(result, float):
-                    elapsed += result
-                else:
-                    raise TypeError(type(result).__name__)
-
-            return elapsed / repeat
-
-        return wrapper
-
-    return decorator
-
-
-def pretty_print_results(results: tuple):
-    valid, invalid, permutations = results
-
-    print(
-        "====================================================================="
-    )
-    print("Details")
-    print(json.dumps(permutations, indent=4))
-
-    if len(valid) > 0:
-        print()
-        print("Passed")
-        keys = ["complexity", "runtime", *valid[0]["details"].keys()]
-        header = " | ".join(f"{header:^15}" for header in keys)
-        print(header)
-        print("-" * len(header))
-        for entry in valid:
-            values = [
-                entry["complexity"],
-                round(entry["runtime"], 4),
-                *entry["details"].values(),
-            ]
-            row = " | ".join(f"{str(value):^15}" for value in values)
-            print(row)
-
-    if len(invalid) > 0:
-        print()
-        print("Failed")
-        keys = ["complexity", "error", *invalid[0]["details"].keys(), "msg"]
-        header = " | ".join(f"{header:^15}" for header in keys)
-        print(header)
-        print("-" * len(header))
-        for entry in invalid:
-            values = [
-                entry["complexity"],
-                entry["error"],
-                *entry["details"].values(),
-                entry["msg"],
-            ]
-            row = " | ".join(f"{str(value):^15}" for value in values)
-            print(row)
-
-
-def _calculate_complexity(params: list[int | tuple[int]]) -> int:
-    """
-    Basic metric of benchmark complexity.
-    """
-    flattened_params = [
-        math.prod(p) if isinstance(p, tuple) else p for p in params
-    ]
-    return math.prod(flattened_params)
-
-
-class Benchmark:
-    def __init__(
-        self,
-        time_limit: float | None,
-        memory_limit: int | None,
-        *_,
-        repeat: int | None = 1,
-        verbose: bool = False,
-    ):
-        self.time_limit = time_limit
-        self.memory_limit = memory_limit
-        self.repeat = repeat
-        self.verbose = verbose
-
-    def get_limits(
-        self,
-        *_,
-        readable: bool = True,
-        memory_unit: str = "GB",
-        time_unit: str = "seconds",
-    ) -> dict[str, str | int | float | None]:
-        """
-        Returns a dictionary of benchmark limits.
-
-        Parameters
-        ----------
-        readable : bool, default=True
-            Toggles whether the output should be human readable.
-        memory_unit : str, default="GB"
-            Toggles what unit to display the memory limit with when 'readable=True'.
-        time_unit : str, default="seconds"
-            Toggles what unit to display the time limit with when 'readable=True'.
-
-        Returns
-        -------
-        dict[str, str | int | float | None]
-            The benchmark limits.
-        """
-
-        memory_value = self.memory_limit
-        if readable and memory_value is not None:
-            match memory_unit:
-                case "TB":
-                    memory_value /= 1024**4
-                case "GB":
-                    memory_value /= 1024**3
-                case "MB":
-                    memory_value /= 1024**2
-                case "KB":
-                    memory_value /= 1024
-                case "B":
-                    pass
-                case _:
-                    valid_set = {"TB", "GB", "MB", "KB", "B"}
-                    raise ValueError(
-                        f"Expected memory unit to be in the set {valid_set}, received '{memory_unit}'."
-                    )
-            memory_value = f"{memory_value} {memory_unit}"
-
-        time_value = self.time_limit
-        if readable and time_value is not None:
-            match time_unit:
-                case "minutes":
-                    time_value /= 60
-                case "seconds":
-                    pass
-                case "milliseconds":
-                    time_value *= 1000
-                case _:
-                    valid_set = {"minutes", "seconds", "milliseconds"}
-                    raise ValueError(
-                        f"Expected time unit to be in the set {valid_set}, received '{time_unit}'."
-                    )
-            time_value = f"{time_value} {time_unit}"
-
-        return {
-            "memory_limit": memory_value,
-            "time_limit": time_value,
-            "repeat": self.repeat,
-        }
-
-    @property
-    def memory_limit(self) -> int | None:
-        """
-        The memory limit in bytes (B).
-        """
-        return self._memory_limit
-
-    @memory_limit.setter
-    def memory_limit(self, limit: int | None):
-        """
-        Stores the memory limit and restricts resources.
-        """
-        self._memory_limit = limit
-        if limit is not None:
-            _, hard = resource.getrlimit(resource.RLIMIT_AS)
-            resource.setrlimit(resource.RLIMIT_AS, (limit, hard))
-
-    def run(
-        self,
-        benchmark,
-        **kwargs: list[Any],
-    ):
-        """
-        Runs a benchmark with ranges of parameters.
-
-        Parameters
-        ----------
-        benchmark : Callable
-            The benchmark function.
-        **kwargs : list[Any]
-            Keyword arguments passing lists of parameters to benchmark. The values should be sorted in
-            decreasing complexity. For example, if the number of labels is a parameter then a higher
-            number of unique labels would be considered "more" complex.
-
-        Example
-        -------
-        >>> b = Benchmark(
-        ...     time_limit=10.0,
-        ...     memory_limit=8 * (1024**3),
-        ...     repeat=1,
-        ...     verbose=False,
-        ... )
-        >>> results = b.run(
-        ...     benchmark=semseg_add_data,
-        ...     n_labels=[
-        ...         100,
-        ...         10,
-        ...     ],
-        ...     shape=[
-        ...         (1000, 1000),
-        ...         (100, 100),
-        ...     ],
-        ... )
-        """
-
-        nvars = len(kwargs)
-        keys = tuple(kwargs.keys())
-        vars = tuple(kwargs[key] for key in keys)
-
-        initial_indices = tuple(0 for _ in range(nvars))
-        max_indices = tuple(len(v) for v in vars)
-        permutations = math.prod(max_indices)
-
-        # Initialize queue with the starting index (0, ...)
-        queue = deque()
-        queue.append(initial_indices)
-
-        # Keep track of explored combinations to avoid duplicates
-        explored = set()
-        explored.add(initial_indices)
-
-        # Store valid combinations that finish within the time limit
-        valid_combinations = []
-        invalid_combinations = []
-
-        pbar = tqdm(total=math.prod(max_indices), disable=(not self.verbose))
-        prev_count = 0
-        while queue:
-
-            current_indices = queue.popleft()
-            parameters = {
-                k: v[current_indices[idx]]
-                for idx, (k, v) in enumerate(zip(keys, vars))
-            }
-            complexity = _calculate_complexity(list(parameters.values()))
-
-            details: dict = {k: str(v) for k, v in parameters.items()}
-
-            # update terminal with status
-            count = len(valid_combinations) + len(invalid_combinations)
-            pbar.update(count - prev_count)
-            prev_count = count
-
-            try:
-                runtime = benchmark(
-                    time_limit=self.time_limit,
-                    repeat=self.repeat,
-                    **parameters,
-                )
-                valid_combinations.append(
-                    {
-                        "complexity": complexity,
-                        "runtime": runtime,
-                        "details": details,
-                    }
-                )
-                continue
-            except Exception as e:
-                invalid_combinations.append(
-                    {
-                        "complexity": complexity,
-                        "error": type(e).__name__,
-                        "msg": str(e),
-                        "details": details,
-                    }
-                )
-
-            for idx in range(nvars):
-                new_indices = list(current_indices)
-                if new_indices[idx] + 1 < max_indices[idx]:
-                    new_indices[idx] += 1
-                    new_indices_tuple = tuple(new_indices)
-                    if new_indices_tuple not in explored:
-                        queue.append(new_indices_tuple)
-                        explored.add(new_indices_tuple)
-
-        valid_combinations.sort(key=lambda x: -x["complexity"])
-        invalid_combinations.sort(key=lambda x: -x["complexity"])
-
-        # clear terminal and display results
-        results = (
-            valid_combinations,
-            invalid_combinations,
-            {
-                "benchmark": benchmark.__name__,
-                "limits": self.get_limits(readable=True),
-                "passed": permutations - len(invalid_combinations),
-                "failed": len(invalid_combinations),
-                "total": permutations,
-            },
-        )
-        pbar.close()
-        if self.verbose:
-            pretty_print_results(results)
-
-        return results

valor_lite/semantic_segmentation/benchmark.py

@@ -1,237 +0,0 @@
-import numpy as np
-
-from valor_lite.profiling import create_runtime_profiler
-from valor_lite.semantic_segmentation import Bitmask, DataLoader, Segmentation
-
-
-def generate_segmentation(
-    datum_uid: str,
-    number_of_unique_labels: int,
-    mask_height: int,
-    mask_width: int,
-) -> Segmentation:
-    """
-    Generates a semantic segmentation annotation.
-
-    Parameters
-    ----------
-    datum_uid : str
-        The datum UID for the generated segmentation.
-    number_of_unique_labels : int
-        The number of unique labels.
-    mask_height : int
-        The height of the mask in pixels.
-    mask_width : int
-        The width of the mask in pixels.
-
-    Returns
-    -------
-    Segmentation
-        A generated semantic segmenatation annotation.
-    """
-
-    if number_of_unique_labels > 1:
-        common_proba = 0.4 / (number_of_unique_labels - 1)
-        min_proba = min(common_proba, 0.1)
-        labels = [str(i) for i in range(number_of_unique_labels)] + [None]
-        proba = (
-            [0.5]
-            + [common_proba for _ in range(number_of_unique_labels - 1)]
-            + [0.1]
-        )
-    elif number_of_unique_labels == 1:
-        labels = ["0", None]
-        proba = [0.9, 0.1]
-        min_proba = 0.1
-    else:
-        raise ValueError(
-            "The number of unique labels should be greater than zero."
-        )
-
-    probabilities = np.array(proba, dtype=np.float64)
-    weights = (probabilities / min_proba).astype(np.int32)
-
-    indices = np.random.choice(
-        np.arange(len(weights)),
-        size=(mask_height * 2, mask_width),
-        p=probabilities,
-    )
-
-    N = len(labels)
-
-    masks = np.arange(N)[:, None, None] == indices
-
-    gts = []
-    pds = []
-    for lidx in range(N):
-        label = labels[lidx]
-        if label is None:
-            continue
-        gts.append(
-            Bitmask(
-                mask=masks[lidx, :mask_height, :],
-                label=label,
-            )
-        )
-        pds.append(
-            Bitmask(
-                mask=masks[lidx, mask_height:, :],
-                label=label,
-            )
-        )
-
-    return Segmentation(
-        uid=datum_uid,
-        groundtruths=gts,
-        predictions=pds,
-        shape=(mask_height, mask_width),
-    )
-
-
-def benchmark_add_data(
-    n_labels: int,
-    shape: tuple[int, int],
-    time_limit: float | None,
-    repeat: int = 1,
-) -> float:
-    """
-    Benchmarks 'Dataloader.add_data' for semantic segmentation.
-
-    Parameters
-    ----------
-    n_labels : int
-        The number of unique labels to generate.
-    shape : tuple[int, int]
-        The size (h,w) of the mask to generate.
-    time_limit : float, optional
-        An optional time limit to constrain the benchmark.
-    repeat : int
-        The number of times to run the benchmark to produce a runtime average.
-
-    Returns
-    -------
-    float
-        The average runtime.
-    """
-
-    profile = create_runtime_profiler(
-        time_limit=time_limit,
-        repeat=repeat,
-    )
-
-    elapsed = 0
-    for _ in range(repeat):
-        data = generate_segmentation(
-            datum_uid="uid",
-            number_of_unique_labels=n_labels,
-            mask_height=shape[0],
-            mask_width=shape[1],
-        )
-        loader = DataLoader()
-        elapsed += profile(loader.add_data)([data])
-    return elapsed / repeat
-
-
-def benchmark_finalize(
-    n_datums: int,
-    n_labels: int,
-    time_limit: float | None,
-    repeat: int = 1,
-):
-    """
-    Benchmarks 'Dataloader.finalize' for semantic segmentation.
-
-    Parameters
-    ----------
-    n_datums : int
-        The number of datums to generate.
-    n_labels : int
-        The number of unique labels to generate.
-    time_limit : float, optional
-        An optional time limit to constrain the benchmark.
-    repeat : int
-        The number of times to run the benchmark to produce a runtime average.
-
-    Returns
-    -------
-    float
-        The average runtime.
-    """
-
-    profile = create_runtime_profiler(
-        time_limit=time_limit,
-        repeat=repeat,
-    )
-
-    elapsed = 0
-    for _ in range(repeat):
-
-        data = [
-            generate_segmentation(
-                datum_uid=str(i),
-                number_of_unique_labels=n_labels,
-                mask_height=5,
-                mask_width=5,
-            )
-            for i in range(10)
-        ]
-        loader = DataLoader()
-        for datum_idx in range(n_datums):
-            segmentation = data[datum_idx % 10]
-            segmentation.uid = str(datum_idx)
-            loader.add_data([segmentation])
-        elapsed += profile(loader.finalize)()
-    return elapsed / repeat
-
-
-def benchmark_evaluate(
-    n_datums: int,
-    n_labels: int,
-    time_limit: float | None,
-    repeat: int = 1,
-):
-    """
-    Benchmarks 'Evaluator.evaluate' for semantic segmentation.
-
-    Parameters
-    ----------
-    n_datums : int
-        The number of datums to generate.
-    n_labels : int
-        The number of unique labels to generate.
-    time_limit : float, optional
-        An optional time limit to constrain the benchmark.
-    repeat : int
-        The number of times to run the benchmark to produce a runtime average.
-
-    Returns
-    -------
-    float
-        The average runtime.
-    """
-
-    profile = create_runtime_profiler(
-        time_limit=time_limit,
-        repeat=repeat,
-    )
-
-    elapsed = 0
-    for _ in range(repeat):
-
-        data = [
-            generate_segmentation(
-                datum_uid=str(i),
-                number_of_unique_labels=n_labels,
-                mask_height=5,
-                mask_width=5,
-            )
-            for i in range(10)
-        ]
-        loader = DataLoader()
-        for datum_idx in range(n_datums):
-            segmentation = data[datum_idx % 10]
-            segmentation.uid = str(datum_idx)
-            loader.add_data([segmentation])
-        evaluator = loader.finalize()
-        elapsed += profile(evaluator.evaluate)()
-    return elapsed / repeat