asyncviz 0.1.0__py3-none-any.whl

asyncviz/__init__.py

@@ -0,0 +1,36 @@
+from asyncviz.bootstrap import (
+    AsyncVizError,
+    AsyncVizRuntime,
+    ConfigError,
+    PortInUseError,
+    StartupError,
+    StartupTimeoutError,
+    get_runtime,
+    is_running,
+    start,
+    stop,
+)
+from asyncviz.config import AsyncVizConfig
+from asyncviz.packaging import package_version
+
+__all__ = [
+    "AsyncVizConfig",
+    "AsyncVizError",
+    "AsyncVizRuntime",
+    "ConfigError",
+    "PortInUseError",
+    "StartupError",
+    "StartupTimeoutError",
+    "get_runtime",
+    "is_running",
+    "start",
+    "stop",
+]
+
+#: Resolved at import time from installed-distribution metadata when
+#: available, falling back to the in-repo literal for editable /
+#: source-tree usage. Single source of truth — never edit this string
+#: directly; bump
+#: :data:`asyncviz.packaging.versioning._FALLBACK_VERSION` together
+#: with ``[project].version`` in ``pyproject.toml``.
+__version__ = package_version()

asyncviz/__main__.py

@@ -0,0 +1,21 @@
+"""``python -m asyncviz`` entry point.
+
+Delegates to the canonical CLI dispatcher in :mod:`asyncviz.cli`.
+Kept tiny so a future plugin system can replace ``run_cli`` without
+touching the module path users invoke from the shell.
+"""
+
+from __future__ import annotations
+
+import sys
+
+from asyncviz.cli import run_cli
+
+
+def main() -> int:
+    """Process entry point — returns the exit code without calling sys.exit."""
+    return run_cli()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

asyncviz/benchmarks/__init__.py

@@ -0,0 +1,143 @@
+"""Canonical AsyncViz benchmarking layer.
+
+Public API:
+
+* :func:`benchmark` — decorator for registering benchmarks.
+* :class:`BenchmarkRunner` — orchestrates warmup + measurement.
+* :class:`BenchmarkConfig` — runner configuration.
+* :func:`run_all` — convenience for "run every registered benchmark".
+
+Lower-level pieces live under the submodules (statistics, baselines,
+reporting, profiling).
+"""
+
+from asyncviz.benchmarks.benchmark_baselines import (
+    BaselineFile,
+    read_baseline,
+    write_baseline,
+)
+from asyncviz.benchmarks.benchmark_configuration import (
+    DEFAULT_MEASURED_ITERATIONS,
+    DEFAULT_REGRESSION_THRESHOLD,
+    DEFAULT_WARMUP_ITERATIONS,
+    BenchmarkConfig,
+    OutlierPolicy,
+    default_config,
+    quick_config,
+)
+from asyncviz.benchmarks.benchmark_diagnostics import (
+    BenchmarkDiagnostics,
+    build_benchmark_diagnostics,
+)
+from asyncviz.benchmarks.benchmark_models import (
+    BaselineComparison,
+    BenchmarkCategory,
+    BenchmarkEnvironment,
+    BenchmarkKind,
+    BenchmarkOutcome,
+    BenchmarkResult,
+    BenchmarkSample,
+    BenchmarkSpec,
+    BenchmarkStatistics,
+    BenchmarkSuiteResult,
+)
+from asyncviz.benchmarks.benchmark_observability import (
+    BenchmarkMetricsSnapshot,
+    get_benchmark_metrics,
+    get_benchmark_metrics_snapshot,
+    reset_benchmark_metrics,
+)
+from asyncviz.benchmarks.benchmark_registry import (
+    benchmark,
+    get_registry,
+    reset_registry,
+)
+from asyncviz.benchmarks.benchmark_runner import BenchmarkRunner
+from asyncviz.benchmarks.benchmark_statistics import (
+    aggregate_samples,
+    apply_outlier_policy,
+)
+from asyncviz.benchmarks.benchmark_thresholds import (
+    RegressionSummary,
+    is_regression,
+    label_for,
+    summarize_regressions,
+)
+from asyncviz.benchmarks.benchmark_tracing import (
+    BenchmarkTraceEntry,
+    BenchmarkTraceKind,
+    clear_benchmark_trace,
+    get_benchmark_trace,
+    is_benchmark_trace_enabled,
+    record_benchmark_trace,
+    set_benchmark_trace_enabled,
+)
+
+
+def run_all(
+    *,
+    config: BenchmarkConfig | None = None,
+    baselines: dict[str, int] | None = None,
+    name_prefix: str | None = None,
+    category: BenchmarkCategory | None = None,
+) -> BenchmarkSuiteResult:
+    """Run every registered benchmark + return the suite result.
+
+    Filtering: ``name_prefix`` keeps only benchmarks whose names
+    start with the prefix; ``category`` restricts to a single
+    category. Both can be combined.
+    """
+    runner = BenchmarkRunner(config=config, baselines=baselines)
+    specs = get_registry().filtered(
+        category=category,
+        name_prefix=name_prefix,
+    )
+    return runner.run_suite(specs)
+
+
+__all__ = [
+    "DEFAULT_MEASURED_ITERATIONS",
+    "DEFAULT_REGRESSION_THRESHOLD",
+    "DEFAULT_WARMUP_ITERATIONS",
+    "BaselineComparison",
+    "BaselineFile",
+    "BenchmarkCategory",
+    "BenchmarkConfig",
+    "BenchmarkDiagnostics",
+    "BenchmarkEnvironment",
+    "BenchmarkKind",
+    "BenchmarkMetricsSnapshot",
+    "BenchmarkOutcome",
+    "BenchmarkResult",
+    "BenchmarkRunner",
+    "BenchmarkSample",
+    "BenchmarkSpec",
+    "BenchmarkStatistics",
+    "BenchmarkSuiteResult",
+    "BenchmarkTraceEntry",
+    "BenchmarkTraceKind",
+    "OutlierPolicy",
+    "RegressionSummary",
+    "aggregate_samples",
+    "apply_outlier_policy",
+    "benchmark",
+    "build_benchmark_diagnostics",
+    "clear_benchmark_trace",
+    "default_config",
+    "get_benchmark_metrics",
+    "get_benchmark_metrics_snapshot",
+    "get_benchmark_trace",
+    "get_registry",
+    "is_benchmark_trace_enabled",
+    "is_regression",
+    "label_for",
+    "quick_config",
+    "read_baseline",
+    "record_benchmark_trace",
+    "reset_benchmark_metrics",
+    "reset_registry",
+    "run_all",
+    "set_benchmark_trace_enabled",
+    "summarize_regressions",
+    "write_baseline",
+]

asyncviz/benchmarks/__main__.py

@@ -0,0 +1,161 @@
+"""``python -m asyncviz.benchmarks`` — run the registered suite.
+
+Usage::
+
+    python -m asyncviz.benchmarks
+    python -m asyncviz.benchmarks --quick
+    python -m asyncviz.benchmarks --category replay
+    python -m asyncviz.benchmarks --name-prefix instrumentation.
+    python -m asyncviz.benchmarks --json /tmp/bench.json --markdown /tmp/bench.md
+    python -m asyncviz.benchmarks --baseline /tmp/baseline.json
+    python -m asyncviz.benchmarks --write-baseline /tmp/baseline.json
+    python -m asyncviz.benchmarks --track-allocations
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+# Importing the benchmark categories registers their entries.
+import asyncviz.benchmarks.instrumentation
+import asyncviz.benchmarks.memory
+import asyncviz.benchmarks.replay
+import asyncviz.benchmarks.stress
+import asyncviz.benchmarks.websocket  # noqa: F401
+from asyncviz.benchmarks import (
+    BenchmarkConfig,
+    default_config,
+    quick_config,
+    read_baseline,
+    run_all,
+    set_benchmark_trace_enabled,
+    summarize_regressions,
+    write_baseline,
+)
+from asyncviz.benchmarks.reporting import (
+    emit_ci_summary,
+    write_json_report,
+    write_markdown_report,
+)
+
+
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="asyncviz.benchmarks",
+        description="Run AsyncViz benchmarks + emit reports.",
+    )
+    parser.add_argument(
+        "--quick",
+        action="store_true",
+        help="Use the quick benchmark config",
+    )
+    parser.add_argument("--category", default=None, help="Filter by category")
+    parser.add_argument(
+        "--name-prefix",
+        default=None,
+        help="Filter by name prefix",
+    )
+    parser.add_argument(
+        "--baseline",
+        type=Path,
+        default=None,
+        help="Compare against this baseline file",
+    )
+    parser.add_argument(
+        "--write-baseline",
+        type=Path,
+        default=None,
+        help="Write a new baseline after the run",
+    )
+    parser.add_argument(
+        "--json",
+        type=Path,
+        default=None,
+        help="Write full JSON report to this path",
+    )
+    parser.add_argument(
+        "--markdown",
+        type=Path,
+        default=None,
+        help="Write markdown summary to this path",
+    )
+    parser.add_argument(
+        "--trace",
+        action="store_true",
+        help="Enable benchmark tracing during the run",
+    )
+    parser.add_argument(
+        "--track-allocations",
+        action="store_true",
+        help="Capture per-sample tracemalloc deltas (slow)",
+    )
+    parser.add_argument(
+        "--fail-on-regression",
+        action="store_true",
+        help="Exit nonzero if any regression is detected vs baseline",
+    )
+    return parser.parse_args(argv)
+
+
+def _resolve_config(args: argparse.Namespace) -> BenchmarkConfig:
+    base = quick_config() if args.quick else default_config()
+    return BenchmarkConfig(
+        warmup_iterations=base.warmup_iterations,
+        measured_iterations=base.measured_iterations,
+        min_samples=base.min_samples,
+        outlier_policy=base.outlier_policy,
+        mad_threshold=base.mad_threshold,
+        iqr_factor=base.iqr_factor,
+        regression_threshold=base.regression_threshold,
+        report_percentiles=base.report_percentiles,
+        disable_gc_during_run=base.disable_gc_during_run,
+        isolate_per_benchmark=base.isolate_per_benchmark,
+        track_allocations=args.track_allocations,
+        deterministic_seed=base.deterministic_seed,
+        fail_on_regression=args.fail_on_regression,
+        extras=dict(base.extras),
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_args(argv)
+    if args.trace:
+        set_benchmark_trace_enabled(True)
+
+    config = _resolve_config(args)
+    baselines = None
+    if args.baseline is not None:
+        bf = read_baseline(args.baseline)
+        if bf is not None:
+            baselines = bf.p95_ns_by_name
+
+    suite = run_all(
+        config=config,
+        baselines=baselines,
+        name_prefix=args.name_prefix,
+        category=args.category,  # type: ignore[arg-type]
+    )
+
+    for line in emit_ci_summary(suite):
+        print(line)
+
+    if args.json is not None:
+        write_json_report(args.json, suite)
+        print(f"json report → {args.json}")
+    if args.markdown is not None:
+        write_markdown_report(args.markdown, suite)
+        print(f"markdown report → {args.markdown}")
+    if args.write_baseline is not None:
+        write_baseline(args.write_baseline, suite)
+        print(f"baseline → {args.write_baseline}")
+
+    summary = summarize_regressions(suite)
+    if args.fail_on_regression and summary.regressed > 0:
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

asyncviz/benchmarks/benchmark_baselines.py

@@ -0,0 +1,82 @@
+"""Baseline persistence + comparison.
+
+A baseline is a JSON file mapping benchmark names → p95 ns. The
+runner consumes this via :meth:`BenchmarkRunner.set_baselines`; the
+CLI writes one out after a clean run so subsequent runs have
+something to compare against.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from asyncviz.benchmarks.benchmark_models import BenchmarkSuiteResult
+
+BASELINE_SCHEMA_VERSION = 1
+
+
+@dataclass(frozen=True, slots=True)
+class BaselineFile:
+    """Parsed view of a baseline JSON."""
+
+    schema_version: int
+    captured_at_wall_ns: int
+    asyncviz_version: str
+    p95_ns_by_name: dict[str, int]
+    metadata: dict[str, str] = field(default_factory=dict)
+
+    def get(self, name: str) -> int | None:
+        return self.p95_ns_by_name.get(name)
+
+
+def write_baseline(
+    path: Path,
+    suite: BenchmarkSuiteResult,
+    *,
+    metadata: dict[str, str] | None = None,
+) -> BaselineFile:
+    """Build a baseline file from a suite result + persist it."""
+    p95_map = {
+        result.outcome.spec_name: result.outcome.statistics.p95_ns
+        for result in suite.results
+        if result.outcome.statistics is not None and result.outcome.status == "ok"
+    }
+    baseline = BaselineFile(
+        schema_version=BASELINE_SCHEMA_VERSION,
+        captured_at_wall_ns=time.time_ns(),
+        asyncviz_version=suite.environment.asyncviz_version,
+        p95_ns_by_name=p95_map,
+        metadata=dict(metadata or {}),
+    )
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "schema_version": baseline.schema_version,
+        "captured_at_wall_ns": baseline.captured_at_wall_ns,
+        "asyncviz_version": baseline.asyncviz_version,
+        "p95_ns_by_name": baseline.p95_ns_by_name,
+        "metadata": baseline.metadata,
+    }
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
+    return baseline
+
+
+def read_baseline(path: Path) -> BaselineFile | None:
+    """Read a baseline file. Returns ``None`` if missing."""
+    if not path.exists():
+        return None
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    schema = int(raw.get("schema_version", 0))
+    if schema != BASELINE_SCHEMA_VERSION:
+        raise ValueError(
+            f"baseline at {path} has schema_version={schema} (expected {BASELINE_SCHEMA_VERSION})",
+        )
+    return BaselineFile(
+        schema_version=schema,
+        captured_at_wall_ns=int(raw.get("captured_at_wall_ns", 0)),
+        asyncviz_version=str(raw.get("asyncviz_version", "")),
+        p95_ns_by_name={str(k): int(v) for k, v in raw.get("p95_ns_by_name", {}).items()},
+        metadata={str(k): str(v) for k, v in raw.get("metadata", {}).items()},
+    )

asyncviz/benchmarks/benchmark_configuration.py

@@ -0,0 +1,105 @@
+"""Benchmark configuration.
+
+Centralizes every knob the runner cares about: warmup discipline,
+sample counts, statistical rigor (outlier policy, minimum-sample
+floor), default regression thresholds. Tests construct purpose-built
+configs; the production CLI uses :func:`default_config`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Final, Literal
+
+OutlierPolicy = Literal["none", "mad", "iqr"]
+"""How outliers are excluded from the reported statistics.
+
+* ``none`` — keep every sample.
+* ``mad`` (default) — drop samples beyond a configured number of
+  median-absolute-deviations from the median. Stable under skewed
+  distributions where mean/stdev would mislead.
+* ``iqr`` — Tukey's IQR fence.
+"""
+
+DEFAULT_WARMUP_ITERATIONS: Final[int] = 50
+DEFAULT_MEASURED_ITERATIONS: Final[int] = 500
+DEFAULT_MIN_SAMPLES: Final[int] = 25
+"""Hard floor — a benchmark with fewer surviving samples than this
+is flagged as ``insufficient`` rather than reported with bogus
+statistics."""
+
+DEFAULT_REGRESSION_THRESHOLD: Final[float] = 0.15
+"""Default ±15% p95 swing before a regression is declared."""
+
+DEFAULT_REPORT_PERCENTILES: Final[tuple[float, ...]] = (50.0, 95.0, 99.0)
+
+
+@dataclass(frozen=True, slots=True)
+class BenchmarkConfig:
+    """Immutable benchmark-runner configuration."""
+
+    warmup_iterations: int = DEFAULT_WARMUP_ITERATIONS
+    measured_iterations: int = DEFAULT_MEASURED_ITERATIONS
+    min_samples: int = DEFAULT_MIN_SAMPLES
+
+    outlier_policy: OutlierPolicy = "mad"
+    mad_threshold: float = 3.5
+    """Number of MADs from the median to keep a sample."""
+
+    iqr_factor: float = 1.5
+
+    regression_threshold: float = DEFAULT_REGRESSION_THRESHOLD
+
+    report_percentiles: tuple[float, ...] = DEFAULT_REPORT_PERCENTILES
+
+    disable_gc_during_run: bool = True
+    """Disable Python's cyclic GC while measuring so allocation
+    spikes from cycle collection don't pollute timing samples."""
+
+    isolate_per_benchmark: bool = True
+    """Force a full GC + counter reset between benchmarks so one
+    workload's tail allocations don't affect the next."""
+
+    track_allocations: bool = False
+    """When True, every sample also captures
+    :func:`tracemalloc.take_snapshot` deltas — slows the run."""
+
+    deterministic_seed: int = 0xA5_BE_F7
+    """Seed used for the synthetic workloads' random number streams."""
+
+    fail_on_regression: bool = False
+    """When True, the suite returns a nonzero exit code on regression.
+    Off by default — informative, not enforcing — until baselines
+    are established."""
+
+    extras: dict[str, str] = field(default_factory=dict)
+    """Free-form notes for downstream tooling."""
+
+    def __post_init__(self) -> None:
+        if self.warmup_iterations < 0:
+            raise ValueError("warmup_iterations must be >= 0")
+        if self.measured_iterations < 1:
+            raise ValueError("measured_iterations must be >= 1")
+        if self.min_samples < 1:
+            raise ValueError("min_samples must be >= 1")
+        if not (0.0 < self.regression_threshold < 10.0):
+            raise ValueError(
+                f"regression_threshold {self.regression_threshold} must be in (0, 10)",
+            )
+        if self.outlier_policy == "mad" and self.mad_threshold <= 0:
+            raise ValueError("mad_threshold must be > 0")
+        if self.outlier_policy == "iqr" and self.iqr_factor <= 0:
+            raise ValueError("iqr_factor must be > 0")
+
+
+def default_config() -> BenchmarkConfig:
+    return BenchmarkConfig()
+
+
+def quick_config() -> BenchmarkConfig:
+    """Faster config for smoke tests + CI sanity runs."""
+    return BenchmarkConfig(
+        warmup_iterations=10,
+        measured_iterations=50,
+        min_samples=10,
+    )

asyncviz/benchmarks/benchmark_diagnostics.py

@@ -0,0 +1,44 @@
+"""Diagnostics builder for the benchmark layer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from asyncviz.benchmarks.benchmark_models import BenchmarkSuiteResult
+from asyncviz.benchmarks.benchmark_observability import (
+    BenchmarkMetricsSnapshot,
+    get_benchmark_metrics_snapshot,
+)
+from asyncviz.benchmarks.benchmark_thresholds import (
+    RegressionSummary,
+    summarize_regressions,
+)
+from asyncviz.benchmarks.benchmark_tracing import (
+    BenchmarkTraceEntry,
+    get_benchmark_trace,
+    is_benchmark_trace_enabled,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class BenchmarkDiagnostics:
+    metrics: BenchmarkMetricsSnapshot
+    regression: RegressionSummary | None
+    trace_enabled: bool
+    recent_trace: tuple[BenchmarkTraceEntry, ...]
+
+
+def build_benchmark_diagnostics(
+    suite: BenchmarkSuiteResult | None = None,
+    *,
+    trace_limit: int = 32,
+) -> BenchmarkDiagnostics:
+    trace = get_benchmark_trace()
+    if trace_limit > 0:
+        trace = trace[-trace_limit:]
+    return BenchmarkDiagnostics(
+        metrics=get_benchmark_metrics_snapshot(),
+        regression=summarize_regressions(suite) if suite is not None else None,
+        trace_enabled=is_benchmark_trace_enabled(),
+        recent_trace=trace,
+    )