zeno-mobile-runner 0.1.2

package/scripts/benchmark_gate.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import math
+import statistics
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Gate benchmark results by pass rate and duration thresholds.")
+    parser.add_argument("--results", required=True, help="Path to benchmark results.jsonl.")
+    parser.add_argument("--min-pass-rate", type=float, default=None, help="Minimum pass rate percentage, for example 100.")
+    parser.add_argument("--max-failures", type=int, default=None, help="Maximum allowed failed runs.")
+    parser.add_argument("--max-mean-ms", type=int, default=None, help="Maximum allowed mean duration in ms.")
+    parser.add_argument("--max-p95-ms", type=int, default=None, help="Maximum allowed p95 duration in ms.")
+    return parser.parse_args()
+
+
+def is_pass(row):
+    if row.get("status") != "ok":
+        return False
+    trace_status = row.get("traceStatus")
+    return trace_status in (None, "passed")
+
+
+def p95(durations):
+    if not durations:
+        return 0
+    ordered = sorted(durations)
+    index = max(0, math.ceil(len(ordered) * 0.95) - 1)
+    return ordered[index]
+
+
+def read_rows(path):
+    rows = []
+    with Path(path).open(encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise SystemExit(f"{path}:{line_number}: invalid json: {exc}") from exc
+            if not isinstance(row, dict):
+                raise SystemExit(f"{path}:{line_number}: expected object row")
+            rows.append(row)
+    return rows
+
+
+def summarize(tool, rows):
+    durations = [int(row.get("durationMs", 0)) for row in rows]
+    failures = [row for row in rows if not is_pass(row)]
+    passed = len(rows) - len(failures)
+    pass_rate = (passed / len(rows) * 100.0) if rows else 0.0
+    mean_ms = round(statistics.mean(durations)) if durations else 0
+    p95_ms = p95(durations)
+    return {
+        "tool": tool,
+        "runs": len(rows),
+        "passed": passed,
+        "failures": len(failures),
+        "passRate": pass_rate,
+        "meanMs": mean_ms,
+        "p95Ms": p95_ms,
+        "failureRows": failures,
+    }
+
+
+def format_summary(summary):
+    return (
+        f"{summary['tool']}: runs={summary['runs']} "
+        f"passRate={summary['passRate']:.2f}% failures={summary['failures']} "
+        f"meanMs={summary['meanMs']} p95Ms={summary['p95Ms']}"
+    )
+
+
+def violations(summary, args):
+    problems = []
+    if args.min_pass_rate is not None and summary["passRate"] < args.min_pass_rate:
+        problems.append(f"passRate {summary['passRate']:.2f}% < {args.min_pass_rate:.2f}%")
+    if args.max_failures is not None and summary["failures"] > args.max_failures:
+        problems.append(f"failures {summary['failures']} > {args.max_failures}")
+    if args.max_mean_ms is not None and summary["meanMs"] > args.max_mean_ms:
+        problems.append(f"meanMs {summary['meanMs']} > {args.max_mean_ms}")
+    if args.max_p95_ms is not None and summary["p95Ms"] > args.max_p95_ms:
+        problems.append(f"p95Ms {summary['p95Ms']} > {args.max_p95_ms}")
+    return problems
+
+
+def main():
+    args = parse_args()
+    rows = read_rows(args.results)
+    if not rows:
+        print(f"no benchmark rows found: {args.results}", file=sys.stderr)
+        return 2
+
+    by_tool = defaultdict(list)
+    for row in rows:
+        by_tool[str(row.get("tool", "unknown"))].append(row)
+
+    failed = False
+    for tool in sorted(by_tool):
+        summary = summarize(tool, by_tool[tool])
+        print(format_summary(summary))
+        problems = violations(summary, args)
+        for problem in problems:
+            print(f"gate failed for {tool}: {problem}", file=sys.stderr)
+        failed = failed or bool(problems)
+
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/benchmark_result_row.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+import argparse
+import json
+from pathlib import Path
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Build one benchmark results JSON row.")
+    parser.add_argument("--tool", required=True)
+    parser.add_argument("--run", required=True, type=int)
+    parser.add_argument("--command-status", required=True, type=int)
+    parser.add_argument("--duration-ms", required=True, type=int)
+    parser.add_argument("--trace-dir", required=True)
+    parser.add_argument("--platform")
+    parser.add_argument("--device")
+    parser.add_argument("--app-id")
+    parser.add_argument("--scenario")
+    parser.add_argument("--app-build")
+    return parser.parse_args()
+
+
+def read_zmr_trace(trace_dir):
+    events_path = Path(trace_dir) / "events.jsonl"
+    if not events_path.exists():
+        return {}
+
+    last_step_error = {}
+    last_scenario_end = {}
+
+    with events_path.open(encoding="utf-8") as events:
+        for line in events:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                event = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            payload = event.get("payload")
+            if not isinstance(payload, dict):
+                payload = {}
+
+            if event.get("kind") == "step.error":
+                last_step_error = payload
+            elif event.get("kind") == "scenario.end":
+                last_scenario_end = payload
+
+    trace = {}
+    if "status" in last_scenario_end:
+        trace["traceStatus"] = last_scenario_end["status"]
+    if "error" in last_scenario_end:
+        trace["traceError"] = last_scenario_end["error"]
+    elif "error" in last_step_error:
+        trace["traceError"] = last_step_error["error"]
+    if "failedStepIndex" in last_scenario_end:
+        trace["failedStepIndex"] = last_scenario_end["failedStepIndex"]
+    elif "index" in last_step_error:
+        trace["failedStepIndex"] = last_step_error["index"]
+    return trace
+
+
+def main():
+    args = parse_args()
+    row = {
+        "tool": args.tool,
+        "run": args.run,
+        "status": "ok" if args.command_status == 0 else "failed",
+        "durationMs": args.duration_ms,
+        "traceDir": args.trace_dir,
+    }
+    metadata = {
+        "platform": args.platform,
+        "device": args.device,
+        "appId": args.app_id,
+        "scenario": args.scenario,
+        "appBuild": args.app_build,
+    }
+    row.update({key: value for key, value in metadata.items() if value})
+
+    if args.tool == "zmr":
+        row.update(read_zmr_trace(args.trace_dir))
+
+    print(json.dumps(row, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()

package/scripts/compare-benchmarks.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import math
+import shlex
+import statistics
+import sys
+import time
+from collections import defaultdict
+from pathlib import Path
+
+CONTEXT_FIELDS = ("platform", "device", "appId", "scenario", "appBuild")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Compare benchmark JSONL rows for two runner labels.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=(
+            "--evidence-out requires --min-candidate-pass-rate, "
+            "--max-candidate-failures, --min-mean-speedup, and "
+            "--min-p95-speedup so market-claim evidence includes explicit gates."
+        ),
+    )
+    parser.add_argument("--results", required=True, help="Path to a benchmark results.jsonl file.")
+    parser.add_argument("--candidate", default="zmr", help="Candidate tool label. Default: zmr.")
+    parser.add_argument("--baseline", required=True, help="Baseline tool label to compare against.")
+    parser.add_argument("--format", choices=("markdown", "json"), default="markdown", help="Output format.")
+    parser.add_argument("--out", help="Optional output file. Defaults to stdout.")
+    parser.add_argument("--min-candidate-pass-rate", type=float, help="Minimum candidate pass rate percentage.")
+    parser.add_argument("--max-candidate-failures", type=int, help="Maximum allowed candidate failures.")
+    parser.add_argument("--min-mean-speedup", type=float, help="Minimum required mean speedup versus baseline.")
+    parser.add_argument("--min-p95-speedup", type=float, help="Minimum required p95 speedup versus baseline.")
+    parser.add_argument("--evidence-out", help="Optional JSONL file to append a market-claim readiness evidence row.")
+    args = parser.parse_args()
+    for name in (
+        "min_candidate_pass_rate",
+        "max_candidate_failures",
+        "min_mean_speedup",
+        "min_p95_speedup",
+    ):
+        value = getattr(args, name)
+        if value is not None and value < 0:
+            parser.error(f"--{name.replace('_', '-')} must be non-negative")
+    if args.evidence_out:
+        missing_gate_args = [
+            f"--{name.replace('_', '-')}"
+            for name in (
+                "min_candidate_pass_rate",
+                "max_candidate_failures",
+                "min_mean_speedup",
+                "min_p95_speedup",
+            )
+            if getattr(args, name) is None
+        ]
+        if missing_gate_args:
+            parser.error(
+                "; ".join(f"{name} is required with --evidence-out" for name in missing_gate_args)
+            )
+    return args
+
+
+def is_pass(row):
+    if row.get("status") != "ok":
+        return False
+    trace_status = row.get("traceStatus")
+    return trace_status in (None, "passed")
+
+
+def p95(durations):
+    if not durations:
+        return 0
+    ordered = sorted(durations)
+    index = max(0, math.ceil(len(ordered) * 0.95) - 1)
+    return ordered[index]
+
+
+def read_rows(path):
+    rows = []
+    with Path(path).open(encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise SystemExit(f"{path}:{line_number}: invalid json: {exc}") from exc
+            if not isinstance(row, dict):
+                raise SystemExit(f"{path}:{line_number}: expected object row")
+            rows.append(row)
+    return rows
+
+
+def summarize(tool, rows):
+    durations = [int(row.get("durationMs", 0)) for row in rows]
+    failures = [row for row in rows if not is_pass(row)]
+    passed = len(rows) - len(failures)
+    pass_rate = (passed / len(rows) * 100.0) if rows else 0.0
+    mean_ms = round(statistics.mean(durations)) if durations else 0
+    return {
+        "tool": tool,
+        "runs": len(rows),
+        "passed": passed,
+        "failures": len(failures),
+        "passRate": pass_rate,
+        "meanMs": mean_ms,
+        "p95Ms": p95(durations),
+    }
+
+
+def ratio(baseline_value, candidate_value):
+    if baseline_value <= 0 or candidate_value <= 0:
+        return None
+    return baseline_value / candidate_value
+
+
+def percent_delta(candidate_value, baseline_value):
+    if baseline_value <= 0:
+        return None
+    return ((candidate_value - baseline_value) / baseline_value) * 100.0
+
+
+def comparison(candidate, baseline):
+    return {
+        "candidate": candidate,
+        "baseline": baseline,
+        "meanSpeedup": ratio(baseline["meanMs"], candidate["meanMs"]),
+        "p95Speedup": ratio(baseline["p95Ms"], candidate["p95Ms"]),
+        "meanDeltaPct": percent_delta(candidate["meanMs"], baseline["meanMs"]),
+        "p95DeltaPct": percent_delta(candidate["p95Ms"], baseline["p95Ms"]),
+    }
+
+
+def benchmark_context(candidate_rows, baseline_rows):
+    rows = candidate_rows + baseline_rows
+    context = {}
+    problems = []
+    for field in CONTEXT_FIELDS:
+        values = [str(row.get(field, "")).strip() for row in rows]
+        concrete = [value for value in values if value]
+        unique = sorted(set(concrete))
+        if len(concrete) != len(values):
+            problems.append(f"{field} missing")
+        elif len(unique) != 1:
+            problems.append(f"{field} mismatch: {', '.join(unique)}")
+        else:
+            context[field] = unique[0]
+    return {
+        "sameContext": not problems,
+        "context": context,
+        "contextProblems": problems,
+    }
+
+
+def format_ratio(value):
+    return "n/a" if value is None else f"{value:.2f}x"
+
+
+def format_pct(value):
+    return "n/a" if value is None else f"{value:+.1f}%"
+
+
+def markdown_report(data):
+    candidate = data["candidate"]
+    baseline = data["baseline"]
+    lines = [
+        "# Benchmark Comparison",
+        "",
+        "| Tool | Runs | Pass rate | Failures | Mean ms | P95 ms |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+        f"| {candidate['tool']} | {candidate['runs']} | {candidate['passRate']:.2f}% | {candidate['failures']} | {candidate['meanMs']} | {candidate['p95Ms']} |",
+        f"| {baseline['tool']} | {baseline['runs']} | {baseline['passRate']:.2f}% | {baseline['failures']} | {baseline['meanMs']} | {baseline['p95Ms']} |",
+        "",
+        f"- Mean speedup: {format_ratio(data['meanSpeedup'])} ({format_pct(data['meanDeltaPct'])} candidate vs baseline)",
+        f"- P95 speedup: {format_ratio(data['p95Speedup'])} ({format_pct(data['p95DeltaPct'])} candidate vs baseline)",
+        f"- Same benchmark context: {'yes' if data.get('sameContext') else 'no'}",
+        "",
+        "Interpretation: negative deltas mean the candidate was faster for that metric. Compare only runs collected on the same host, device state, app build, and scenario.",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def gate_failures(data, args):
+    failures = []
+    candidate = data["candidate"]
+    baseline = data["baseline"]
+    if args.evidence_out and candidate["runs"] < 20:
+        failures.append(f"candidateRuns {candidate['runs']} below minimum 20")
+    if args.evidence_out and baseline["runs"] < 20:
+        failures.append(f"baselineRuns {baseline['runs']} below minimum 20")
+    if args.min_candidate_pass_rate is not None and candidate["passRate"] < args.min_candidate_pass_rate:
+        failures.append(
+            f"candidate passRate {candidate['passRate']:.2f}% below minimum {args.min_candidate_pass_rate:.2f}%"
+        )
+    if args.max_candidate_failures is not None and candidate["failures"] > args.max_candidate_failures:
+        failures.append(
+            f"candidate failures={candidate['failures']} above maximum {args.max_candidate_failures}"
+        )
+    if args.min_mean_speedup is not None:
+        speedup = data["meanSpeedup"]
+        if speedup is None or speedup < args.min_mean_speedup:
+            actual = "n/a" if speedup is None else f"{speedup:.2f}x"
+            failures.append(f"meanSpeedup {actual} below minimum {args.min_mean_speedup:.2f}x")
+    if args.min_p95_speedup is not None:
+        speedup = data["p95Speedup"]
+        if speedup is None or speedup < args.min_p95_speedup:
+            actual = "n/a" if speedup is None else f"{speedup:.2f}x"
+            failures.append(f"p95Speedup {actual} below minimum {args.min_p95_speedup:.2f}x")
+    if args.evidence_out and not data.get("sameContext"):
+        details = "; ".join(data.get("contextProblems", [])) or "missing context"
+        failures.append(f"same benchmark context evidence required ({details})")
+    return failures
+
+
+def write_evidence(args, data, failures, duration_ms):
+    if not args.evidence_out:
+        return
+    path = Path(args.evidence_out)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    row = {
+        "name": "competitive benchmark comparison",
+        "status": "failed" if failures else "passed",
+        "durationMs": duration_ms,
+        "command": " ".join(shlex.quote(part) for part in sys.argv),
+        "candidate": args.candidate,
+        "baseline": args.baseline,
+        "results": args.results,
+        "minCandidatePassRate": args.min_candidate_pass_rate,
+        "maxCandidateFailures": args.max_candidate_failures,
+        "minMeanSpeedup": args.min_mean_speedup,
+        "minP95Speedup": args.min_p95_speedup,
+        "candidateRuns": data["candidate"]["runs"],
+        "baselineRuns": data["baseline"]["runs"],
+        "candidatePassRate": data["candidate"]["passRate"],
+        "candidateFailures": data["candidate"]["failures"],
+        "meanSpeedup": data["meanSpeedup"],
+        "p95Speedup": data["p95Speedup"],
+        "sameContext": data["sameContext"],
+        "context": data["context"],
+    }
+    if failures:
+        row["error"] = "; ".join(failures)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n")
+
+
+def main():
+    started = time.monotonic()
+    args = parse_args()
+    rows = read_rows(args.results)
+    by_tool = defaultdict(list)
+    for row in rows:
+        by_tool[str(row.get("tool", "unknown"))].append(row)
+
+    missing = [tool for tool in (args.candidate, args.baseline) if tool not in by_tool]
+    if missing:
+        print(f"missing benchmark rows for: {', '.join(missing)}", file=sys.stderr)
+        return 2
+
+    data = comparison(
+        summarize(args.candidate, by_tool[args.candidate]),
+        summarize(args.baseline, by_tool[args.baseline]),
+    )
+    data.update(benchmark_context(by_tool[args.candidate], by_tool[args.baseline]))
+
+    if args.format == "json":
+        output = json.dumps(data, sort_keys=True) + "\n"
+    else:
+        output = markdown_report(data)
+
+    if args.out:
+        Path(args.out).write_text(output, encoding="utf-8")
+    else:
+        sys.stdout.write(output)
+
+    failures = gate_failures(data, args)
+    duration_ms = round((time.monotonic() - started) * 1000)
+    write_evidence(args, data, failures, duration_ms)
+    if failures:
+        for failure in failures:
+            print(f"benchmark comparison gate failed: {failure}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())