benchstats 1.0.0__py3-none-any.whl

benchstats/__main__.py

@@ -0,0 +1,114 @@
+# using abs path only here to alleviate debugging
+from benchstats.compare import compareStats
+from benchstats.common import LoggingConsole, detectExportFormat, bmNamesTransform
+from benchstats.cli_parser import makeParser
+from benchstats.render import renderComparisonResults
+from benchstats.parsers import getParserFor
+
+import os
+from rich.terminal_theme import DIMMED_MONOKAI as DarkTheme, DEFAULT_TERMINAL_THEME as LightTheme
+
+
+def main():
+    parser = makeParser()
+    args = parser.parse_args()
+
+    export_fmt = detectExportFormat(args.export_to, args.export_fmt)
+    if export_fmt is not None:
+        if os.path.isfile(args.export_to):
+            os.remove(args.export_to)
+        else:
+            assert not os.path.exists(args.export_to), "Invalid --export_to value"
+
+    console = LoggingConsole(
+        no_color=not args.colors,
+        record=export_fmt is not None,
+        log_level=(
+            LoggingConsole.LogLevel.Debug if args.show_debug else LoggingConsole.LogLevel.Warning
+        ),
+    )
+
+    if args.alpha > 0.5 or args.alpha <= 0:
+        console.critical(
+            "--alpha must be a positive number less than 0.5 (%.2f is given)" % args.alpha
+        )
+        exit(2)
+
+    Parser1 = getParserFor(args.files_parser if args.file1_parser is None else args.file1_parser)
+    Parser2 = getParserFor(args.files_parser if args.file2_parser is None else args.file2_parser)
+
+    s1 = Parser1(args.file1, args.filter1, args.metrics, debug_log=console).getStats()
+    s2 = Parser2(args.file2, args.filter2, args.metrics, debug_log=console).getStats()
+
+    s1 = bmNamesTransform(
+        s1,
+        getattr(args, "from") if args.from1 is None else args.from1,
+        args.to if args.to1 is None else args.to1,
+        1,
+        console,
+    )
+    s2 = bmNamesTransform(
+        s2,
+        getattr(args, "from") if args.from2 is None else args.from2,
+        args.to if args.to2 is None else args.to2,
+        2,
+        console,
+    )
+
+    alpha = args.alpha
+    if args.bonferroni:
+        alpha = alpha / len(s1)
+        console.info(
+            f"Bonferroni correction for {len(s1)} comparisons turns alpha={args.alpha:.3e} into {alpha:.3e}"
+        )
+
+    if args.main_metrics is None or len(args.main_metrics) < 1:
+        main_metrics = [args.metrics[0]]
+    else:
+        main_metrics = [args.metrics[mi] for mi in args.main_metrics]
+
+    cr = compareStats(
+        s1,
+        s2,
+        method=args.method,
+        main_metrics=main_metrics,
+        alpha=alpha,
+        debug_log=console,
+        store_sets=args.sample_stats is not None,
+    )
+
+    renderComparisonResults(
+        cr,
+        console,
+        expect_same=args.expect_same,
+        main_metrics=main_metrics,
+        sample_stats=args.sample_stats,
+        dark_theme=not args.export_light,
+        show_sample_sizes=args.sample_sizes,
+        always_show_pvalues=args.always_show_pvalues,
+        multiline=args.multiline,
+    )
+
+    if cr.at_least_one_differs:
+        console.warning(
+            "At least one significant difference in main metrics was detected. Returning with exit(1)."
+        )
+
+    if export_fmt is not None:
+        if "txt" == export_fmt:
+            console.save_text(args.export_to)
+        elif "svg" == export_fmt:
+            console.save_svg(
+                args.export_to, title="", theme=LightTheme if args.export_light else DarkTheme
+            )
+        elif "html" == export_fmt:
+            console.save_html(args.export_to, theme=LightTheme if args.export_light else DarkTheme)
+        else:
+            assert False, "NOT IMPLEMENTED?!"
+
+    if cr.at_least_one_differs:
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

benchstats/cli_parser.py

@@ -0,0 +1,297 @@
+import argparse
+from .common import kAvailableFormats
+from .compare import kMethods, kDefaultAlpha
+from .parsers import getBuiltinParsers
+from .render import kPossibleStatNames
+
+
+def makeParser():
+    parser = argparse.ArgumentParser(
+        description="A tool to compare two sets of the same benchmarks with repetitions.\n"
+        "Homepage: https://github.com/Arech/benchstats\n",
+        epilog="On custom parsers:\n"
+        "You could supply virtually any source data to the tool by utilizing --files_parser (or "
+        "--file1_parser or --file2_parser) command line argument. Each of these arguments in "
+        "addition to built-in parser identifiers also accepts a path to a Python file that "
+        "defines a custom parser. The simplest possible parser to read a single one column CSV "
+        "file is this:\n\n"
+        """# save to ./myCSV.py
+import numpy as np
+from benchstats.common import ParserBase
+
+class myCSV(ParserBase):
+    def __init__(self, fpath, filter, metrics, debug_log=None) -> None:
+        self.stats = np.loadtxt(fpath, dtype=np.float64)
+
+    def getStats(self) -> dict[str, dict[str, np.ndarray]]:
+        return {"bm": {"real_time": self.stats}}
+
+"""
+        "It doesn't support filtering, different benchmarks and the metric is hardcoded - but you "
+        "get the idea. It's that simple.\n"
+        "Now to compare two datasets in csvs, just run:\n"
+        "python -m benchstats ./src1.csv ./src2.csv --files_parser ./myCSV.py\n\n"
+        "If you'll make a parser that could be useful to other people, please consider adding it "
+        "to the project's built-in parsers set by opening a thread with a suggestion in "
+        "https://github.com/Arech/benchstats/issues or by making a PR into the repo.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--show_debug",
+        help="Shows some additional debugging info. Default: %(default)s",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+    )
+
+    g_inputs = parser.add_argument_group(
+        "Input data",
+        "Arguments describing how a source data sets are obtained and are optionally transformed.",
+    )
+
+    g_inputs.add_argument(
+        "file1",
+        help="Path to the first data file with benchmark results. See also --file1_parser and "
+        "--filter1 arguments",
+        metavar="<path/to/file1>",
+    )
+
+    g_inputs.add_argument(
+        "file2",
+        help="Path to the second data file with benchmark results. See also --file2_parser and "
+        "--filter2 arguments",
+        metavar="<path/to/file2>",
+    )
+
+    g_inputs.add_argument(
+        "--files_parser",
+        help="Sets files parser class identifier, if a built-in parser is used (options are: "
+        f"{', '.join(getBuiltinParsers())}). Or sets a path to .py file defining a custom parser, "
+        "inherited from 'benchstats.common.ParserBase' class. The parser class name must be the "
+        "same as the file name. See below for an example.",
+        default="GbenchJson",
+        metavar="<files parser class or path>",
+    )
+
+    g_inputs.add_argument(
+        "--file1_parser",
+        help="Same as --files_parser, but only applies to file1.",
+        default=None,
+        metavar="<file1 parser class or path>",
+    )
+
+    g_inputs.add_argument(
+        "--filter1",
+        help="If specified, sets a Python regular expression (see "
+        "https://docs.python.org/3/howto/regex.html#regex-howto) to select benchmarks by name "
+        "from <file1>",
+        metavar="<reg expr>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "--file2_parser",
+        help="Same as --files_parser, but only applies to file2.",
+        default=None,
+        metavar="<file2 parser class or path>",
+    )
+
+    g_inputs.add_argument(
+        "--filter2",
+        help="Same as --filter1, but for <file2>",
+        metavar="<reg expr>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "--from",
+        help="--from and --to works in a pair and are used to describe benchmark names transformation. "
+        "--from sets a regular expression (Python re module flavor) to define a pattern to replace to --to replacement "
+        "string. "
+        "Typical use-cases include removing unnecessary parts of benchmark names (for example, Google Benchmark adds "
+        "some suffixes that might not convey useful information) or \"glueing\" results of different benchmarks "
+        "so they become comparable by the tool (for example, you have two benchmarks, the first is 'old_foo' with old algorithm "
+        "implementation and the other is 'foo' with a new competing algorithm implementation, - to compare their performance "
+        "against each other, you need to remove 'old_' prefix from the first benchmark with `--from old_`"
+        " and apply --filter1 and --filter2 to restrict loading only corresponding data for old or new variations). "
+        "Remember to escape characters that have special meaning for the shell.",
+        metavar="<reg expr>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "--to",
+        help="See help for --from.",
+        metavar="<replacement string>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "--from1",
+        help="Like --from, but for <file1> only",
+        metavar="<reg expr>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "--to1",
+        help="Like --to, but for <file1> only",
+        metavar="<replacement string>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "--from2",
+        help="Like --from, but for <file2> only",
+        metavar="<reg expr>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "--to2",
+        help="Like --to, but for <file2> only",
+        metavar="<replacement string>",
+        default=None,
+    )
+
+    g_inputs.add_argument(
+        "metrics",
+        help="List of metric identifiers to use in tests for each benchmark. Default: %(default)s. "
+        "For deterministic algorithms highly recommend to measure and to use a minimum latency per "
+        "repetition.",
+        nargs="*",
+        default=["real_time"],
+    )
+
+    g_inputs.add_argument(
+        "--main_metrics",
+        help="Indexes in 'metrics' list specifying the main metrics. These are displayed differently and differences "
+        "detected cause script to exit(1). Default: %(default)s.",
+        nargs="+",
+        metavar="<metric idx>",
+        type=int,
+        default=[0],
+    )
+
+    g_stats = parser.add_argument_group("Statistical settings")
+
+    g_stats.add_argument(
+        "--method",
+        help=(
+            "Selects a method of statistical testing. Possible values are are: "
+            + ", ".join(
+                [
+                    "'" + id + f"' for {descr['name']} ({descr['url'].replace('%','%%')})"
+                    for (id, descr) in kMethods.items()
+                ]
+            )
+            + ". Default is %(default)s"
+        ),
+        metavar="<method id>",
+        choices=kMethods.keys(),
+        default=list(kMethods.keys())[0],
+    )
+
+    g_stats.add_argument(
+        "--alpha",
+        help="Set statistical significance level (a desired mistake probability level)\n"
+        "Note! This is an actual probability of a mistake only when all preconditions of the "
+        "chosen statistical test are met. One of the most important preconditions like independence of "
+        "individual measurements, or constancy of underlying distribution parameters are almost "
+        "never met in a real-life benchmarking, even on a properly quiesced hardware. Real "
+        "mistake rates are higher. Default %(default)s",
+        metavar="<positive float less than 0.5>",
+        type=float,
+        default=kDefaultAlpha,
+    )
+
+    g_stats.add_argument(
+        "--bonferroni",
+        help="If set, applies a Bonferroni multiple comparisons correction "
+        "(https://en.wikipedia.org/wiki/Bonferroni_correction).",
+        action="store_true",
+        default=False,
+    )
+
+    g_rendering=parser.add_argument_group("Rendering", "Controls how results are rendered")
+
+    g_rendering.add_argument(
+        "--always_show_pvalues",
+        help="If set, always show pvalues. By default it shows pvalues only for significant differences. Note that "
+        "when two sets are compared stochastically same (~), or more precisely, not stochastically less and not "
+        "stochastically greater (see https://en.wikipedia.org/wiki/Stochastic_ordering), pvalue shown is a minimum of "
+        "two pvalues for less and greater comparison.",
+        action="store_true",
+        default=False,
+    )
+
+    g_rendering.add_argument(
+        "--sample_sizes",
+        help="Controls whether to show sizes of datasets used in a test. Default %(default)s",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+    )
+
+    # py3.11 doesn't accept backslashes in f-strings, so preparing that in advance
+    stat_options = "', '".join(kPossibleStatNames.keys())
+    g_rendering.add_argument(
+        "--sample_stats",
+        help="Sets which additional statistics about compared samples sets to show. Could be any sequence of floats "
+        "(must be in range [0,100], designating a percentile value to calculate), or aliases '"
+        f"{stat_options}' (shortenings are accepted). Use of 'std', though isn't "
+        "recommended as standard deviation doesn't really mean anything for most distributions "
+        "beyond Normal.",
+        nargs="+",  # default value doesn't work with *, while it must
+        metavar="<stat ids>",
+        default=None,
+    )
+
+    g_rendering.add_argument(
+        "--expect_same",
+        help="If set, assumes that distributions are the same (i.e. H0 hypothesis is true) and shows some additional "
+        "statistics useful for ensuring that a benchmark code is stable enough, or the machine is quiesced enough. "
+        "One good example is when <file1> and <file2> were made with exactly the same binary running on exactly the "
+        "same machine in the same state - on a machine with a proper setup tests shouldn't find a difference.",
+        action="store_true",
+        default=False,
+    )
+
+    g_rendering.add_argument(
+        "--colors",
+        help="Controls if the output should be colored. Default: %(default)s",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+    )
+
+    g_rendering.add_argument(
+        "--multiline",
+        help="Controls if benchmark results could span several lines. Default: %(default)s",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+    )
+
+    g_export = parser.add_argument_group("Export","Controls how to export results.")
+
+    g_export.add_argument(
+        "--export_to",
+        help="Path to file to store comparison results to.",
+        metavar="<path/to/export_file>",
+    )
+
+    g_export.add_argument(
+        "--export_fmt",
+        help=f"Format of the export file. Options are: {', '.join(kAvailableFormats)}. If not set, "
+        "inferred from --export_to file extension",
+        choices=kAvailableFormats,
+        default=None,
+        metavar="<format id>",
+    )
+
+    g_export.add_argument(
+        "--export_light",
+        help="If set, uses light theme instead of dark",
+        action="store_true",
+        default=False,
+    )
+
+    return parser

benchstats/common.py

@@ -0,0 +1,153 @@
+"""Describes a base class for a source of statistics capable of producing (from a file or whatnot)
+data in format suitable for calling compare::compareStats()
+"""
+
+import os
+import re
+import rich.console
+from collections.abc import Iterable
+import enum
+
+kAvailableFormats = ("txt", "svg", "html")
+
+
+class ParserBase:
+    def __init__(self, source_id, filter, metrics, debug_log=None) -> None:
+        """Expected source constructor.
+        - source_id is an identifier of a data source that the class knows how to process and turn
+            into inputs of compare::compareStats(). Data type is derived class implementation
+            dependent.
+            Typically it's a string referencing a file to read and parse.
+        - filter is an object describing how to filter data source when not everything from it
+            is needed.
+            Typically, it's a string with a regular expression to match against a benchmark name.
+        - metrics is a list of string representing metrics to extract for each benchmark repetition.
+        - debug_log is a flag to enable/disable logging, or a logger object like LoggingConsole to
+            send logs to. When the parser is called by benchstats CLI handler, it's always an
+            instance of LoggingConsole, so support for other variants (None or bool) are needed only
+            for built-in parsers to alleviate use of the package components in a source code.
+        """
+        pass  # derived class knows what to do
+
+    def getStats(self) -> dict[str, dict[str, Iterable[float]]]:
+        """Return a dict that describes benchmarks and its measured statistics so it and can be
+        directly passed as any of the first two arguments to compare::compareStats(). See that
+        method for the details.
+        """
+        raise RuntimeError("DERIVED CLASS MUST IMPLEMENT METHOD")
+
+
+def detectExportFormat(export_to, export_fmt):
+    assert (export_to is None and export_fmt is None) or (
+        isinstance(export_to, str) and len(export_to) > 0
+    )
+    assert export_fmt is None or export_fmt in kAvailableFormats
+
+    if export_to is not None and export_fmt is None:
+        root, ext = os.path.splitext(export_to)
+        assert ext in [
+            "." + e for e in kAvailableFormats
+        ], f"Unrecognized export file extension '{ext}' of a file in --export_to parameter"
+        export_fmt = ext[1:]
+
+    return export_fmt
+
+
+class LoggingConsole(rich.console.Console):
+
+    @enum.verify(enum.CONTINUOUS)
+    class LogLevel(enum.IntEnum):
+        Debug = (0,)
+        Info = (1,)
+        Warning = (2,)
+        Error = (3,)
+        Failure = (4,)
+        Critical = 5
+
+    def __init__(self, log_level: LogLevel = LogLevel.Debug, **kwargs):
+        assert isinstance(log_level, LoggingConsole.LogLevel)
+        self.log_level = log_level
+        if "emoji" not in kwargs:
+            kwargs["emoji"] = False
+        if "highlight" not in kwargs:
+            kwargs["highlight"] = False
+        super().__init__(**kwargs)
+
+    def _do_log(self, color: str, lvl: str, *args, **kwargs):
+        if "sep" in kwargs:
+            sep = kwargs["sep"] if len(kwargs["sep"]) > 0 else " "
+        else:
+            sep = " "
+            kwargs["sep"] = sep
+        return super().print(f"\[[{color}]{lvl:4s}[/{color}]]{sep}", *args, **kwargs)
+
+    def debug(self, *args, **kwargs):
+        if self.log_level > LoggingConsole.LogLevel.Debug:
+            return None
+        return self._do_log("gray", "dbg", *args, **kwargs)
+
+    def info(self, *args, **kwargs):
+        if self.log_level > LoggingConsole.LogLevel.Info:
+            return None
+        return self._do_log("white", "info", *args, **kwargs)
+
+    def warning(self, *args, **kwargs):
+        if self.log_level > LoggingConsole.LogLevel.Warning:
+            return None
+        return self._do_log("yellow", "warn", *args, **kwargs)
+
+    def error(self, *args, **kwargs):
+        if self.log_level > LoggingConsole.LogLevel.Error:
+            return None
+        return self._do_log("orange", "Err", *args, **kwargs)
+
+    def failure(self, *args, **kwargs):
+        if self.log_level > LoggingConsole.LogLevel.Failure:
+            return None
+        return self._do_log("red", "FAIL", *args, **kwargs)
+
+    def critical(self, *args, **kwargs):
+        if self.log_level > LoggingConsole.LogLevel.Critical:
+            return None
+        return self._do_log("magenta", "CRIT", *args, **kwargs)
+
+
+def bmNamesTransform(
+    stats: dict[str, dict[str, Iterable[float]]],
+    re_from: str | None,
+    re_to: str | None,
+    set_idx: int,
+    console: LoggingConsole,
+) -> dict[str, dict[str, Iterable[float]]]:
+    assert isinstance(stats, dict) and isinstance(set_idx, int)
+    assert isinstance(console, LoggingConsole)
+    if re_from is None:
+        # not nice that it glues func args with CLI args, but ok to simplify err handling
+        assert (
+            re_to is None
+        ), f"--to{set_idx} can only be used when there's a corresponding --from{set_idx}"
+        return stats
+    assert isinstance(re_from, str) and len(re_from) > 0
+    if re_to is None:
+        re_to = ""
+    else:
+        assert isinstance(re_to, str)
+
+    r = re.compile(re_from)
+
+    old_bms = stats.keys()
+    new_bms = [r.subn(re_to, bm_name)[0] for bm_name in old_bms]
+    unique_len = len(frozenset(new_bms))
+    if unique_len != len(new_bms):
+        console.warning(
+            f"--from{set_idx} -> --to{set_idx} regexp '{re_from}'->'{re_to}' is a narrowing conversion "
+            f"generates only {unique_len} unique name from {len(new_bms)}. This most likely"
+            " isn't what you want, since the order of narrowing is not specified and you might end"
+            " up having a wrong data for some new benchmark name."
+        )
+        console.debug("Old names are:", ", ".join(old_bms))
+        console.debug("New names are:", ", ".join(new_bms))
+    else:
+        assert unique_len == len(stats)
+
+    return {r.subn(re_to, bm_name)[0]: bm_metrics for bm_name, bm_metrics in stats.items()}