k4bench 0.0.13__py3-none-any.whl

k4bench/__init__.py

@@ -0,0 +1,8 @@
+"""k4Bench — performance benchmarking for DD4hep-based simulations and reconstruction in Key4hep."""
+
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("k4bench")
+except PackageNotFoundError:
+    __version__ = "unknown"

k4bench/analysis/__init__.py

@@ -0,0 +1,19 @@
+"""Post-processing and visualisation for k4bench results."""
+
+from k4bench.analysis.loader import load_event_timing, load_region_timing, load_results
+from k4bench.analysis.plots import (
+    plot_event_memory,
+    plot_event_timing,
+    plot_region_timing,
+    plot_run_overview,
+)
+
+__all__ = [
+    "load_results",
+    "load_event_timing",
+    "load_region_timing",
+    "plot_run_overview",
+    "plot_event_timing",
+    "plot_event_memory",
+    "plot_region_timing",
+]

k4bench/analysis/loader.py

@@ -0,0 +1,258 @@
+"""Load benchmark results and per-event timing data for analysis."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pandas as pd
+
+
+
+def load_results(log_dir: str | Path, labels: list[str] | None = None) -> pd.DataFrame:
+    """Load benchmark results from a log directory into a DataFrame.
+
+    Each ``{label}_results.csv`` file written by ``k4bench`` is loaded and
+    concatenated into a single DataFrame.
+
+    Parameters
+    ----------
+    log_dir : str or Path
+        Directory containing ``*_results.csv`` files.
+    labels : list[str] or None
+        Load only these run labels.  Loads all ``*_results.csv`` files when ``None``.
+
+    Returns
+    -------
+    pd.DataFrame
+        One row per run. Float columns are cast to ``float64``;
+        integer columns that may contain NaN use nullable ``Int64``.
+    """
+    log_dir = Path(log_dir)
+    _suffix = "_results.csv"
+
+    if labels is not None:
+        candidates = [(log_dir / f"{lbl}{_suffix}", lbl) for lbl in labels]
+        missing = [lbl for path, lbl in candidates if not path.exists()]
+        if missing:
+            raise ValueError(f"Missing result files for labels: {missing}")
+        paths = [path for path, _ in candidates if path.exists()]
+    else:
+        paths = sorted(log_dir.glob(f"*{_suffix}"))
+
+    if not paths:
+        raise ValueError(f"No *_results.csv files found in '{log_dir}'.")
+
+    df = pd.concat([pd.read_csv(p) for p in paths], ignore_index=True)
+
+    float_cols = [
+        "wall_time_s", "user_cpu_s", "sys_cpu_s",
+        "peak_rss_mb", "output_size_mb", "events_per_sec",
+    ]
+    int_cols = [
+        "returncode", "n_events",
+        "major_page_faults", "voluntary_ctx_switches", "involuntary_ctx_switches",
+    ]
+
+    for col in float_cols:
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors="coerce")
+
+    for col in int_cols:
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors="coerce").astype("Int64")
+
+    return df
+
+
+def load_event_timing(
+    log_dir: str | Path,
+    labels: list[str] | None = None,
+) -> dict[str, pd.DataFrame]:
+    """Load per-event timing JSON files from a log directory.
+
+    Each ``{label}_events.json`` file written by the k4BenchTimingAction
+    plugin is parsed into a DataFrame.
+
+    Parameters
+    ----------
+    log_dir : str or Path
+        Directory containing ``*_events.json`` files.
+    labels : list[str] or None
+        Load only these run labels. If None, all ``*_events.json`` files
+        in *log_dir* are loaded.
+
+    Returns
+    -------
+    dict[str, pd.DataFrame]
+        Maps label → DataFrame with columns
+        ``event_number``, ``event_time_s``, ``rss_begin_mb``,
+        ``rss_end_mb``, ``rss_delta_mb``.
+    """
+    log_dir = Path(log_dir)
+    _suffix = "_events.json"
+
+    if labels is not None:
+        candidates = [(log_dir / f"{lbl}{_suffix}", lbl) for lbl in labels]
+    else:
+        candidates = [
+            (p, p.name[: -len(_suffix)])
+            for p in sorted(log_dir.glob(f"*{_suffix}"))
+        ]
+
+    if labels is not None:
+        missing_files = [lbl for path, lbl in candidates if not path.exists()]
+        if missing_files:
+            raise ValueError(f"Missing event files for labels: {missing_files}")
+
+    out: dict[str, pd.DataFrame] = {}
+    for path, label in candidates:
+        if not path.exists():
+            continue
+        with path.open() as f:
+            raw = json.load(f)
+        _required = ["event_numbers", "event_times_s", "event_rss_begin_mb", "event_rss_end_mb"]
+        _missing = [k for k in _required if k not in raw]
+        if _missing:
+            raise ValueError(f"{path} missing keys: {_missing}")
+        lengths = {k: len(raw[k]) for k in _required}
+        if len(set(lengths.values())) > 1:
+            raise ValueError(f"{path} has mismatched array lengths: {lengths}")
+        df = pd.DataFrame(
+            {
+                "event_number": raw["event_numbers"],
+                "event_time_s": raw["event_times_s"],
+                "rss_begin_mb": raw["event_rss_begin_mb"],
+                "rss_end_mb":   raw["event_rss_end_mb"],
+            }
+        )
+        df["rss_delta_mb"] = df["rss_end_mb"] - df["rss_begin_mb"]
+        out[label] = df
+
+    return out
+
+
+def load_region_timing(
+    log_dir: str | Path,
+    labels: list[str] | None = None,
+) -> dict[str, dict]:
+    """Load per-region timing JSON files from a log directory.
+
+    Each ``{label}_regions.json`` file written by the k4BenchRegionTimingAction
+    plugin is parsed into structured data.
+
+    Parameters
+    ----------
+    log_dir : str or Path
+        Directory containing ``*_regions.json`` files.
+    labels : list[str] or None
+        Load only these run labels.  If ``None``, all ``*_regions.json`` files
+        in *log_dir* are loaded.
+
+    Returns
+    -------
+    dict[str, dict]
+        Maps label → dict with keys:
+
+        - ``"meta"``: dict with schema_version, timer, overhead_ns, detectors,
+          lv_counts.
+        - ``"events"``: DataFrame with columns ``event_number``,
+          ``event_wall_s``, ``event_region_sum_s``, ``event_unaccounted_s``.
+        - ``"at_location"``: DataFrame indexed by ``event_number``, one column
+          per top-level detector (seconds), time charged to where the Geant4
+          step physically occurred.
+        - ``"by_birth"``: same shape as ``at_location``, time charged to the
+          detector where the primary track was created.
+    """
+    log_dir = Path(log_dir)
+    _suffix = "_regions.json"
+
+    if labels is not None:
+        candidates = [(log_dir / f"{lbl}{_suffix}", lbl) for lbl in labels]
+    else:
+        candidates = [
+            (p, p.name[: -len(_suffix)])
+            for p in sorted(log_dir.glob(f"*{_suffix}"))
+        ]
+
+    if labels is not None:
+        missing = [lbl for path, lbl in candidates if not path.exists()]
+        if missing:
+            raise ValueError(f"Missing region files for labels: {missing}")
+
+    out: dict[str, dict] = {}
+    for path, label in candidates:
+        if not path.exists():
+            continue
+        with path.open() as f:
+            raw = json.load(f)
+
+        _required = [
+            "event_numbers", "event_wall_seconds",
+            "event_region_sum_seconds", "event_unaccounted_seconds",
+            "at_location_seconds", "by_birth_seconds",
+        ]
+        _missing = [k for k in _required if k not in raw]
+        if _missing:
+            raise ValueError(f"{path} missing keys: {_missing}")
+
+        n_ev = len(raw["event_numbers"])
+        if len(set(raw["event_numbers"])) != n_ev:
+            raise ValueError(f"{path}: event_numbers contains duplicates")
+        for k in ["event_wall_seconds", "event_region_sum_seconds",
+                  "event_unaccounted_seconds", "at_location_seconds", "by_birth_seconds"]:
+            if len(raw[k]) != n_ev:
+                raise ValueError(f"{path}: array length mismatch for '{k}'")
+
+        events_df = pd.DataFrame({
+            "event_number":       raw["event_numbers"],
+            "event_wall_s":       raw["event_wall_seconds"],
+            "event_region_sum_s": raw["event_region_sum_seconds"],
+            "event_unaccounted_s": raw["event_unaccounted_seconds"],
+        })
+
+        ev_index = pd.Index(raw["event_numbers"], name="event_number")
+        at_loc_df   = pd.DataFrame(raw["at_location_seconds"], index=ev_index).fillna(0.0)
+        by_birth_df = pd.DataFrame(raw["by_birth_seconds"],    index=ev_index).fillna(0.0)
+
+        declared = raw.get("indexed_top_level_detectors", [])
+        if declared:
+            extra_at  = [c for c in at_loc_df.columns  if c not in declared]
+            extra_by  = [c for c in by_birth_df.columns if c not in declared]
+            at_loc_df   = at_loc_df.reindex(  columns=declared + extra_at,  fill_value=0.0)
+            by_birth_df = by_birth_df.reindex( columns=declared + extra_by, fill_value=0.0)
+
+        # Step counts per detector per event (interval_counts field, optional).
+        # Columns include all detector names plus "unattributed" for steps that
+        # could not be assigned to a top-level detector element.
+        raw_steps = raw.get("interval_counts", [])
+        if raw_steps and len(raw_steps) == n_ev:
+            steps_df: pd.DataFrame | None = (
+                pd.DataFrame(raw_steps, index=ev_index).fillna(0).astype(float)
+            )
+        else:
+            steps_df = None
+
+        out[label] = {
+            "meta": {
+                "schema_version":    raw.get("schema_version", 1),
+                "attribution_method": raw.get("attribution", "dd4hep_top_level_detelement"),
+                "timer":             raw.get("timer", "unknown"),
+                "overhead_ns":       raw.get("per_step_timer_overhead_ns"),
+                "detectors":         raw.get("indexed_top_level_detectors", []),
+                "lv_counts":         raw.get("indexed_top_level_detector_lv_counts", {}),
+            },
+            "events":      events_df,
+            "at_location": at_loc_df,
+            "by_birth":    by_birth_df,
+            "steps":       steps_df,   # None when interval_counts absent in JSON
+        }
+
+    if not out:
+        if labels is not None:
+            raise ValueError(
+                f"No region files found for labels={labels} in '{log_dir}'."
+            )
+        raise ValueError(f"No *_regions.json files found in '{log_dir}'.")
+
+    return out

k4bench/analysis/plots/__init__.py

@@ -0,0 +1,28 @@
+"""Plotly plotting functions for k4bench results.
+
+All functions return a :class:`~plotly.graph_objects.Figure`.
+In Jupyter the figure renders inline automatically; call ``fig.show()``
+to display it explicitly, or ``fig.write_html("out.html")`` to export.
+
+Typical notebook usage::
+
+    from k4bench.analysis import load_results, plot_run_overview, plot_event_timing
+
+    plot_run_overview("logs/")
+    plot_event_timing("logs/")
+"""
+
+from .event import plot_event_memory, plot_event_timing
+from .overview import plot_run_overview
+from .region import plot_region_timing
+from ._theme import PALETTE
+from ._utils import _compute_core_range  # re-exported: used in tests
+
+__all__ = [
+    "plot_run_overview",
+    "plot_event_timing",
+    "plot_event_memory",
+    "plot_region_timing",
+    "PALETTE",
+    "_compute_core_range",
+]

k4bench/analysis/plots/_theme.py

@@ -0,0 +1,46 @@
+"""Visual style constants shared across all plot modules."""
+
+from __future__ import annotations
+
+_TEMPLATE = "plotly_white"
+
+_BLUE = "#1f77b4"
+_RED  = "#d62728"
+
+_PALETTE = [
+    "#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd",
+    "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf",
+]
+
+PALETTE = _PALETTE
+
+_METRIC_UNITS: dict[str, str] = {
+    "wall_time_s":    "(s)",
+    "peak_rss_mb":    "(MB)",
+    "user_cpu_s":     "(s)",
+    "sys_cpu_s":      "(s)",
+    "events_per_sec": "(ev/s)",
+    "output_size_mb": "(MB)",
+}
+
+_UNACCOUNTED_COLOR = "#999999"
+_OTHER_COLOR       = "#d0d0d0"
+
+
+def _hex_to_rgba(color: str, alpha: float) -> str:
+    """Convert a hex or rgb() color string to an rgba() string."""
+    color = color.strip()
+    if color.startswith("rgba("):
+        # Already has alpha — replace it
+        inner = color[5:].rstrip(")")
+        parts = inner.rsplit(",", 1)
+        return f"rgba({parts[0]},{alpha})"
+    if color.startswith("rgb("):
+        inner = color[4:].rstrip(")")
+        return f"rgba({inner},{alpha})"
+    # Hex (#RRGGBB or #RGB)
+    h = color.lstrip("#")
+    if len(h) == 3:
+        h = h[0]*2 + h[1]*2 + h[2]*2
+    r, g, b = int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)
+    return f"rgba({r},{g},{b},{alpha})"

k4bench/analysis/plots/_traces.py

@@ -0,0 +1,90 @@
+"""Reusable Plotly trace builders shared across plot modules."""
+
+from __future__ import annotations
+
+import numpy as np
+import plotly.graph_objects as go
+
+from ._theme import _BLUE, _PALETTE, _hex_to_rgba
+
+
+def _histogram_traces(
+    arrays: dict[str, np.ndarray],
+    common_edges: np.ndarray,
+    label_list: list[str],
+    alpha: float,
+    show_legend: bool,
+    palette: list[str] | None = None,
+) -> list[go.Bar]:
+    """Return one histogram Bar trace per label using shared bin edges."""
+    centers = 0.5 * (common_edges[:-1] + common_edges[1:])
+    widths  = common_edges[1:] - common_edges[:-1]
+    _pal    = palette if palette is not None else _PALETTE
+    traces = []
+    for i, lbl in enumerate(label_list):
+        color = _BLUE if len(label_list) == 1 else _pal[i % len(_pal)]
+        counts, _ = np.histogram(arrays[lbl], bins=common_edges)
+        traces.append(go.Bar(
+            x=centers,
+            y=counts,
+            width=widths,
+            name=lbl,
+            legendgroup=lbl,
+            marker_color=_hex_to_rgba(color, alpha),
+            marker_line_width=0,
+            showlegend=show_legend,
+            hovertemplate=f"<b>{lbl}</b><br>bin centre: %{{x:.4g}}<br>count: %{{y}}<extra></extra>",
+        ))
+    return traces
+
+
+def _format_delta_cell(mean: float, sem: float, ref_mean: float, ref_sem: float) -> str:
+    """Return a formatted Δμ ± δ(Δμ) string for use as a stats-table cell.
+
+    Returns ``"—"`` for the reference run (mean == ref_mean) and
+    ``"undefined"`` when ref_mean is zero to avoid division by zero.
+    """
+    if mean == ref_mean:
+        return "—"
+    if ref_mean == 0:
+        return "undefined"
+    delta_pct = (mean - ref_mean) / ref_mean * 100
+    delta_err = (100.0 / ref_mean) * np.sqrt(sem**2 + (mean / ref_mean * ref_sem) ** 2)
+    sign = "+" if delta_pct >= 0 else ""
+    return f"{sign}{delta_pct:.2f}% ± {delta_err:.2f}%"
+
+
+def _stats_table_trace(
+    table_rows: list[list[str]],
+    ref_label: str,
+    unit_label: str,
+    label_list: list[str],
+) -> go.Table:
+    """Build a Plotly Table trace from per-run stats rows."""
+    headers = [
+        "Run",
+        f"μ ± SEM ({unit_label})",
+        f"σ ({unit_label})",
+        f"Δμ ± δ(Δμ)  [ref: {ref_label}]",
+    ]
+    n = len(table_rows)
+    row_colors = [_hex_to_rgba(_PALETTE[i % len(_PALETTE)], 0.12) for i in range(n)]
+    col_data = [list(col) for col in zip(*table_rows)] if table_rows else [[] for _ in headers]
+
+    return go.Table(
+        header=dict(
+            values=[f"<b>{h}</b>" for h in headers],
+            fill_color="#d4d4d4",
+            align="center",
+            font=dict(size=11, color="#222222"),
+            line_color="white",
+        ),
+        cells=dict(
+            values=col_data,
+            fill_color=[row_colors],
+            align="center",
+            font=dict(size=10, color="#333333"),
+            line_color="white",
+            height=24,
+        ),
+    )

k4bench/analysis/plots/_utils.py

@@ -0,0 +1,196 @@
+"""Pure-data helpers: data loading/normalisation, outlier detection, region utilities."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from ..loader import load_event_timing, load_region_timing, load_results
+
+_DEFAULT_EXCLUDE_EVENTS = [0]
+
+
+def _path_name(p: str | Path) -> str:
+    """Return the final path component, stripping trailing slashes first."""
+    return Path(str(p).rstrip("/")).name
+
+_OUTLIER_FRACTION_WARN = 0.05
+_OUTLIER_EXTREME_RATIO = 5.0
+
+
+def _compute_stats(data: np.ndarray) -> tuple[float, float, float, float]:
+    """Return (mean, std, sem, se_std) with se_std = std / sqrt(2*(n-1))."""
+    data = data[~np.isnan(data)]
+    n = len(data)
+    mean = float(data.mean()) if n > 0 else float("nan")
+    if n > 1:
+        std    = float(data.std(ddof=1))
+        sem    = std / np.sqrt(n)
+        se_std = std / np.sqrt(2 * (n - 1))
+    else:
+        std = sem = se_std = float("nan")
+    return mean, std, sem, se_std
+
+
+def _ensure_df(results: pd.DataFrame | str | Path | list) -> pd.DataFrame:
+    """Accept a DataFrame, a single log-dir path, or a list of paths."""
+    if isinstance(results, pd.DataFrame):
+        return results
+    if isinstance(results, list):
+        frames = []
+        for path in results:
+            df = load_results(path).copy()
+            prefix = _path_name(path)
+            df["label"] = df["label"].astype(str).apply(
+                lambda lbl, p=prefix: lbl if lbl.startswith(f"{p}/") else f"{p}/{lbl}"
+            )
+            frames.append(df)
+        return pd.concat(frames, ignore_index=True)
+    return load_results(results)
+
+
+def _ensure_event_data(
+    source: dict[str, pd.DataFrame] | str | Path | list,
+    labels: list[str] | None = None,
+) -> dict[str, pd.DataFrame]:
+    """Accept a pre-loaded dict, a single log-dir path, or a list of paths.
+
+    ``labels`` matching supports both exact keys and suffix matching: a key
+    ``"dir/run"`` matches the label ``"run"``.  When two labels share the same
+    suffix, pass the full prefixed key to disambiguate.
+    """
+    if isinstance(source, dict):
+        if labels is not None:
+            return {k: v for k, v in source.items() if k in labels}
+        return source
+    if isinstance(source, list):
+        out: dict[str, pd.DataFrame] = {}
+        for path in source:
+            prefix = _path_name(path)
+            for lbl, df in load_event_timing(path).items():
+                key = lbl if lbl.startswith(f"{prefix}/") else f"{prefix}/{lbl}"
+                if key in out:
+                    raise ValueError(
+                        f"Duplicate label '{key}': two source paths share the directory name "
+                        f"'{prefix}'. Rename the directories to disambiguate."
+                    )
+                out[key] = df
+        if labels is not None:
+            out = {k: v for k, v in out.items()
+                   if k in labels or any(k.endswith(f"/{w}") for w in labels)}
+        return out
+    return load_event_timing(source, labels=labels)
+
+
+def _ensure_region_data(
+    source: dict[str, dict] | str | Path | list[str | Path],
+    labels: list[str] | None = None,
+) -> dict[str, dict]:
+    """Accept a pre-loaded dict, a single log-dir path, or a list of paths."""
+    if isinstance(source, dict):
+        if labels is not None:
+            return {k: v for k, v in source.items() if k in labels}
+        return source
+    if isinstance(source, list):
+        out: dict[str, dict] = {}
+        for path in source:
+            prefix = _path_name(path)
+            for lbl, data in load_region_timing(path).items():
+                key = lbl if lbl.startswith(f"{prefix}/") else f"{prefix}/{lbl}"
+                if key in out:
+                    raise ValueError(
+                        f"Duplicate label '{key}': two source paths share the directory name "
+                        f"'{prefix}'. Rename the directories to disambiguate."
+                    )
+                out[key] = data
+        if labels is not None:
+            out = {k: v for k, v in out.items()
+                   if k in labels or any(k.endswith(f"/{w}") for w in labels)}
+        return out
+    return load_region_timing(source, labels=labels)
+
+
+def _detector_title(source: object) -> str | None:
+    """Return a display name from path(s); None for pre-loaded data."""
+    if isinstance(source, (str, Path)):
+        return _path_name(source)
+    if isinstance(source, list) and source:
+        return " vs ".join(_path_name(s) for s in source)
+    return None
+
+
+def _default_baseline(labels: list[str]) -> str:
+    return sorted(labels)[0]
+
+
+def _matches_baseline(label: str, baseline_label: str | None) -> bool:
+    if baseline_label is None:
+        return False
+    return label == baseline_label or label.endswith(f"/{baseline_label}")
+
+
+def _compute_core_range(
+    data: np.ndarray,
+    threshold: float = 3.5,
+) -> tuple[tuple[float, float], int]:
+    """Return an x-range that excludes outliers and the number of clipped points.
+
+    Uses the modified Z-score (Iglewicz & Hoaglin, 1993) with MAD as the
+    spread estimate, which is robust for skewed or heavy-tailed distributions.
+    """
+    median = float(np.median(data))
+    mad = float(np.median(np.abs(data - median)))
+
+    if mad > 0:
+        modified_z = 0.6745 * np.abs(data - median) / mad
+        mask = modified_z <= threshold
+        core = data[mask]
+        n_clipped = int((~mask).sum())
+    else:
+        core = data
+        n_clipped = 0
+
+    if len(core) == 0:
+        core = data
+        n_clipped = 0
+
+    x_min, x_max = float(core.min()), float(core.max())
+    margin = 0.05 * (x_max - x_min) if x_max > x_min else 0.01 * abs(x_max)
+    x_min = max(0.0, x_min - margin)
+    x_max = x_max + margin
+    return (x_min, x_max), n_clipped
+
+
+def _region_top_n(
+    time_df: pd.DataFrame,
+    top_n: int,
+) -> tuple[list[str], list[str]]:
+    """Return (top_dets, all_dets_sorted) by mean time descending, skipping zero-time columns."""
+    means = time_df.mean()
+    active = means[means > 0].sort_values(ascending=False)
+    all_sorted = active.index.tolist()
+    return all_sorted[:top_n], all_sorted
+
+
+def _build_stacked_arrays(
+    time_df: pd.DataFrame,
+    top_dets: list[str],
+    all_dets_sorted: list[str],
+) -> dict[str, np.ndarray]:
+    """Build per-detector time arrays; detectors outside top_dets are grouped as 'Other'."""
+    n = len(time_df)
+    arrays: dict[str, np.ndarray] = {}
+    for det in top_dets:
+        arrays[det] = time_df[det].to_numpy() if det in time_df.columns else np.zeros(n)
+
+    other_dets = [d for d in all_dets_sorted if d not in top_dets]
+    extra_dets = [d for d in time_df.columns if d not in top_dets and d not in all_dets_sorted]
+    if other_dets or extra_dets:
+        other_arr = np.zeros(n)
+        for det in other_dets + extra_dets:
+            if det in time_df.columns:
+                other_arr = other_arr + time_df[det].to_numpy()
+        arrays["Other"] = other_arr
+    return arrays