BenchMatcha 0.0.1__py3-none-any.whl

BenchMatcha/__init__.py

@@ -0,0 +1,32 @@
+# BSD 3-Clause License
+#
+# Copyright (c) 2025, Spill-Tea
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""BenchMatcha Project."""
+
+__version__: str = "v0.0.1"

BenchMatcha/complexity.py

@@ -0,0 +1,208 @@
+# BSD 3-Clause License
+#
+# Copyright (c) 2025, Spill-Tea
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Complexity calculations."""
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import google_benchmark as gbench
+import numpy as np
+from scipy.optimize import curve_fit  # type: ignore[import-untyped]
+
+from .utils import _simple_stats
+
+
+@dataclass
+class FitResult:
+    """Curve fit result.
+
+    Args:
+        bigo (str): Big O notation string identifier.
+        params (np.ndarray): coefficient value(s).
+        cov (np.ndarray): covariance std of coefficients
+        rms (float): root mean square error of fit.
+
+    """
+
+    bigo: str
+    params: np.ndarray
+    cov: np.ndarray
+    rms: float
+
+    @staticmethod
+    def _handle(x: np.ndarray) -> str:
+        a = " ".join([f"{j:.3E}" for j in x.tolist()])
+
+        return f"[{a}]"
+
+    def __repr__(self) -> str:
+        return (
+            f"FitResult(bigo={self.bigo},params={self._handle(self.params)}"
+            f",cov={self._handle(self.cov)},rms={self.rms:.3f})"
+        )
+
+
+# Define common complexity functions with all coefficients and intercept
+Equation = (
+    Callable[[np.ndarray, float, float], np.ndarray]
+    | Callable[[np.ndarray, float, float, float], np.ndarray]
+    | Callable[[np.ndarray, float, float, float, float], np.ndarray]
+)
+
+
+def constant(n: np.ndarray, a: float, b: float) -> np.ndarray:
+    """Constant O(1) equation."""
+    return a * np.ones_like(n) + b
+
+
+def logn(n: np.ndarray, a: float, b: float) -> np.ndarray:
+    """Log O(logN) equation."""
+    return a * np.log2(n) + b
+
+
+def linear(n: np.ndarray, a: float, b: float) -> np.ndarray:
+    """Linear O(N) equation."""
+    return a * n + b
+
+
+def nlogn(n: np.ndarray, a: float, b: float) -> np.ndarray:
+    """Log linear O(NlogN) equation."""
+    return a * n * np.log2(n) + b
+
+
+def quadratic(n: np.ndarray, a: float, b: float, c: float) -> np.ndarray:
+    """Quadratic O(N^2) equation."""
+    return a * np.power(n, 2) + linear(n, b, c)
+
+
+def cubic(n: np.ndarray, a: float, b: float, c: float, d: float) -> np.ndarray:
+    """Cubic O(N^3) equation."""
+    return a * np.power(n, 3) + quadratic(n, b, c, d)
+
+
+complexity_functions: dict[str, Equation] = {
+    # gbench.oNone.name: "",
+    gbench.o1.name: constant,
+    gbench.oLogN.name: logn,
+    gbench.oN.name: linear,
+    gbench.oNLogN.name: nlogn,
+    gbench.oNSquared.name: quadratic,
+    gbench.oNCubed.name: cubic,
+}
+
+
+def compute_rmsd(y_true: np.ndarray, y_pred: np.ndarray, k: int) -> float:
+    r"""Mean normalized root mean square deviation (RMSD).
+
+    Args:
+        y_true (np.ndarray): observed y values.
+        y_pred (np.ndarray): predicted y values.
+        k (int): number of parameters used to estimate predicted values.
+
+    Returns:
+        (float): normalized RMSD
+
+    Equations:
+        $\frac{1}{\bar{y}} \sqrt{\frac{\sum_{i=0}^{N} (y_i - \hat{y}_i)^2}{N - k}}$
+
+    """
+    residuals: np.ndarray = y_true - y_pred
+    sum_square_error: np.float64 = (residuals * residuals).sum()
+    dof: np.int64 = np.prod(y_pred.size) - k
+
+    return float(np.sqrt(sum_square_error / dof) / y_true.mean())
+
+
+def fit(
+    func: Callable,
+    label: str,
+    x: np.ndarray,
+    y: np.ndarray,
+    sigma: np.ndarray,
+) -> FitResult | None:
+    """Fit observed data to an equation.
+
+    Args:
+        func (Callable): equation to fit.
+        label (str): complexity label
+        x (np.ndarray): x input values
+        y (np.ndarray): observed y values
+        sigma (np.ndarray): observed error in y values
+
+    Returns:
+        (FitResult | None) returns fit result if converged.
+
+    """
+    popt: np.ndarray
+    pcov: np.ndarray
+    try:
+        popt, pcov, *_ = curve_fit(
+            func,
+            x,
+            y,
+            sigma=sigma,
+            absolute_sigma=True,
+        )
+        pred = func(x, *popt)
+        cov = np.sqrt(pcov.diagonal())
+        rms = compute_rmsd(y, pred, len(popt))
+
+        return FitResult(
+            bigo=label,
+            params=popt,
+            cov=cov,
+            rms=rms,
+        )
+
+    except RuntimeError:
+        return None
+
+
+def fit_complexity(x: np.ndarray, y: np.ndarray, sigma: np.ndarray) -> list[FitResult]:
+    """Perform curve fitting to available complexity algorithms."""
+    results: list[FitResult] = []
+
+    for label, func in complexity_functions.items():
+        if (res := fit(func, label, x, y, sigma)) is not None:
+            results.append(res)
+
+    return results
+
+
+def analyze_complexity(x: np.ndarray, y: np.ndarray) -> list[FitResult]:
+    """Analyze algorithmic complexity."""
+    mean, std = _simple_stats(y)
+
+    return sorted(fit_complexity(x, mean, std), key=lambda x: x.rms)
+
+
+def get_best_fit(fits: list[FitResult]) -> FitResult:
+    """Return best fit by minimizing RMSD."""
+    return min(fits, key=lambda x: x.rms)

BenchMatcha/config.py

@@ -0,0 +1,108 @@
+# BSD 3-Clause License
+#
+# Copyright (c) 2025, Spill-Tea
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Default runner configuration."""
+
+import logging
+
+import toml  # type: ignore[import-untyped]
+
+from . import plotting
+
+
+log: logging.Logger = logging.getLogger(__name__)
+
+
+class Config:
+    """default configuration.
+
+    Attributes:
+        color (str): plot marker color.
+        line_color (str): plot line color.
+        font (str): plot font family style.
+        x_axis (int): Maximum number of line ticks on x-axis.
+
+    """
+
+    color: str = plotting.Prism[3]
+    line_color: str = plotting.Prism[4]
+    font: str = "Space Grotesk Light, Courier New, monospace"
+    x_axis: int = 13
+
+
+class ConfigUpdater:
+    """Configuration updater through pyproject config file.
+
+    Args:
+        path (str): path to valid configuration file.
+        config (Config): configuration class to update.
+
+    """
+
+    path: str
+    config: type[Config]
+
+    def __init__(self, path: str, config: type[Config] = Config) -> None:
+        self.path = path
+        self.config = config
+
+    def load(self) -> dict:
+        """Load toml data from path."""
+        return toml.load(self.path)
+
+    def _update(self, data: dict) -> None:
+        for key, value in data.get("tool", {}).get("BenchMatcha", {}).items():
+            if not hasattr(self.config, key):
+                log.info("Unsupported tool key: %s", key)
+                continue
+
+            setattr(self.config, key, value)
+
+    def update(self) -> None:
+        """Parse toml path and update default configuration."""
+        data: dict = self.load()
+        self._update(data)
+
+
+def update_config_from_pyproject(path: str) -> None:
+    """Update default config from pyproject toml file.
+
+    Example:
+
+        .. code-block: toml
+
+            [tool.BenchMatcha]
+            color="#FFF"
+            line_color="#333"
+            font="Courier"
+            x_axis=5
+
+    """
+    cu = ConfigUpdater(path)
+    cu.update()

BenchMatcha/errors.py

@@ -0,0 +1,89 @@
+# BSD 3-Clause License
+#
+# Copyright (c) 2025, Spill-Tea
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Custom BenchMatcha exception definitions.
+
+Considerations:
+    * All custom exceptions should be defined within this module for better project
+      organization. This also prevents proliferation of custom errors as project scales.
+    * Custom exceptions should define a class method named `response` to construct and
+      standardize verbiage of output message. It has the nice side effect of providing
+      example context for usage.
+    * Before creating a custom error, determine if available exceptions can be used
+      instead.
+
+"""
+
+from __future__ import annotations
+
+from json import JSONDecodeError
+from typing import Self, TypeVar
+
+
+E = TypeVar("E", bound=Exception)
+
+_exception_register: set[type[Exception]] = {
+    TypeError,
+    ValueError,
+    RuntimeError,
+    JSONDecodeError,
+    FileNotFoundError,
+}
+
+
+def register_custom_exception(cls: type[E]) -> type[E]:
+    """Register custom exceptions."""
+    _exception_register.add(cls)
+
+    return cls
+
+
+@register_custom_exception
+class SchemaError(Exception):
+    """Unsupported json schema."""
+
+    @classmethod
+    def response(cls, version: str) -> Self:
+        """Define standard response message."""
+        msg: str = f"Unsupported json schema version: {version}"
+
+        return cls(msg)
+
+
+@register_custom_exception
+class ParsingError(Exception):
+    """Failed to parse json output (from Google Benchmark)."""
+
+    @classmethod
+    def response(cls) -> Self:
+        """Define standard response message."""
+        return cls(
+            "Failed to parse json data. Please confirm benchmarks do not contain "
+            "print statements or write to stdout, which can interfere with output."
+        )

BenchMatcha/handlers.py

@@ -0,0 +1,154 @@
+# BSD 3-Clause License
+#
+# Copyright (c) 2025, Spill-Tea
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""IO handlers to transform using json library."""
+
+from __future__ import annotations
+
+import json
+import os
+from abc import ABC, abstractmethod
+from io import IOBase
+from typing import IO, Any
+
+
+def is_readable_io_protocol(obj: object) -> bool:
+    """Determine if object implements readable IO interface."""
+    methods: set[str] = {
+        "read",
+        "readable",
+        "write",
+        # "writeable",  # temporary file handler does not have this method.
+        "seek",
+        "seekable",
+        "tell",
+        "close",
+        "closed",  # property
+        "flush",
+    }
+
+    return (
+        isinstance(obj, IO | IOBase) or all(map(lambda x: hasattr(obj, x), methods))
+    ) and obj.readable()  # type: ignore[attr-defined]
+
+
+class Handler(ABC):
+    """Abstract Handler protocol."""
+
+    @abstractmethod
+    def handle(self) -> dict[str, Any]:
+        """Handle parsing of object."""
+        raise NotImplementedError("Must implement.")
+
+
+class HandleText(Handler):
+    """Handle loading text (string) to json object."""
+
+    text: str
+
+    def __init__(self, text: str):
+        self.text = text
+
+    def handle(self) -> dict[str, Any]:
+        return json.loads(self.text)
+
+
+class HandleBytes(Handler):
+    """Handle loading bytes to json object."""
+
+    text: bytes
+    encoding: str
+
+    def __init__(self, text: bytes, encoding: str = "utf8"):
+        self.text = text
+        self.encoding = encoding
+
+    def handle(self) -> dict[str, Any]:
+        return HandleText(self.text.decode(self.encoding)).handle()
+
+
+class HandleIO(Handler):
+    """Handle loading io data to json object."""
+
+    stream: IOBase
+    encoding: str
+
+    def __init__(self, stream: IOBase, encoding: str = "utf8"):
+        self.stream = stream
+        if not self.stream.readable():
+            raise TypeError("Unreadable stream.")
+
+        self.encoding = encoding
+
+    def handle(self) -> dict[str, Any]:
+        text = self.stream.read()
+        if isinstance(text, bytes):
+            handler: Handler = HandleBytes(text, self.encoding)
+
+        else:
+            handler = HandleText(text)
+
+        return handler.handle()
+
+
+class HandlePath(Handler):
+    """Handle loading data from file to json object."""
+
+    path: str
+    encoding: str
+
+    def __init__(self, path: str, encoding: str = "utf8"):
+        self.path = path
+        self.encoding = encoding
+
+    def handle(self) -> dict[str, Any]:
+        if not os.path.exists(self.path):
+            return HandleText(self.path).handle()
+
+        with open(self.path, "r", encoding=self.encoding) as f:
+            return HandleIO(f).handle()
+
+
+def dispatch(obj: object, encoding: str = "utf8") -> Handler:
+    """Dispatch appropriate handler in response to input object type."""
+    if isinstance(obj, str):
+        return HandlePath(obj)
+
+    elif isinstance(obj, bytes):
+        return HandleBytes(obj, encoding)
+
+    elif is_readable_io_protocol(obj):
+        return HandleIO(obj, encoding)  # type: ignore[arg-type]
+
+    raise TypeError(f"Unsupported object type: {type(obj)}")
+
+
+def load(obj: object, encoding: str = "utf8") -> dict[str, Any]:
+    """Load json data."""
+    return dispatch(obj, encoding).handle()