pi-dev-team 0.2.0

package/skills/mutation-testing/scripts/xunit_v3_feature_detector.py

@@ -0,0 +1,341 @@
+#!/usr/bin/env python3
+"""Detect xunit.v3-only constructs that break the v2 mutation-shim compile,
+and classify each so a human can decide how to handle it (#1160, part of #1156).
+
+The v2 shim is the only path that gives the mutant-kill loop per-test coverage
+on an xunit.v3 suite (validated in #669). Its weakness is compile fragility:
+any xunit-v3-only API breaks the shim's compile before Stryker runs a mutant.
+This module turns that opaque C# compile error into a classified list the
+mutation-kill agent presents at an **always-ask, never-auto-drop** human gate.
+
+Two classification axes:
+
+- ``compile_ability`` — ``clean-translatable`` (a mechanical v2 equivalent
+  exists) or ``no-v2-equivalent`` (the construct cannot compile under v2).
+- ``coverage_impact`` — ``neutral`` (excluding the test from the shim changes
+  no coverage, e.g. ``[Fact(Explicit = true)]`` tests are skipped by default)
+  or ``bearing`` (the test runs and covers code, so excluding it inflates the
+  loop's survivor set — the operator must be told which lines go dark).
+
+Pure and stdlib-only. The interactive gate and the actual shim-exclusion wiring
+(#1159) consume this module's structured output; this module never edits a file.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections.abc import Iterable, Sequence
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+CLEAN_TRANSLATABLE = "clean-translatable"
+NO_V2_EQUIVALENT = "no-v2-equivalent"
+COVERAGE_NEUTRAL = "neutral"
+COVERAGE_BEARING = "bearing"
+
+
+@dataclass(frozen=True)
+class Construct:
+    name: str
+    pattern: re.Pattern[str]
+    compile_ability: str
+    coverage_impact: str
+    note: str
+
+
+# Each entry's classification is grounded in xunit.v3's documented surface —
+# xunit.net/docs/getting-started/v3/whats-new (Explicit property, Assert.Skip*,
+# TestContext, IAsyncLifetime→ValueTask, TheoryDataRow) — validated against real
+# xunit behavior in #1156. C# attribute/keyword casing is fixed, so patterns are
+# case-sensitive (no re.IGNORECASE).
+_CONSTRUCTS: list[Construct] = [
+    Construct(
+        name="fact-explicit",
+        # [Fact(Explicit = true)] / [Theory(Explicit=true)] — the property does
+        # not exist on v2's FactAttribute.
+        #
+        # `(?:[^\]"]|"[^"]*")*` consumes either a non-quote character or a WHOLE
+        # quoted span, never a lone `"`. So the property match can never land
+        # inside a string (a DisplayName="Explicit = true" is still not a hit),
+        # but it can step over a completed one — the previous `[^\]"]*` could
+        # only refuse to enter a quote, which silently hid every form where a
+        # string-valued property came first: `[Fact(DisplayName = "x",
+        # Explicit = true)]`. The same construct is allowed to share its bracket
+        # (`[Trait("a","b"), Fact(Explicit = true)]`) rather than having to own
+        # it, matching the autofixture construct's reasoning below.
+        pattern=re.compile(
+            r"\[(?:[^\]\"]|\"[^\"]*\")*?\b(?:Fact|Theory)\s*\("
+            r"(?:[^\]\"]|\"[^\"]*\")*?\bExplicit\s*=\s*true"
+        ),
+        compile_ability=NO_V2_EQUIVALENT,
+        coverage_impact=COVERAGE_NEUTRAL,
+        note="Explicit tests are skipped unless the run opts in (-explicit / "
+        "xunit.runner.json explicit=on), so under a normal mutation run they "
+        "contribute zero coverage — excluding them from the shim changes nothing. "
+        "Neutral ONLY while the run does not enable explicit tests.",
+    ),
+    Construct(
+        name="assert-skip",
+        # Assert.Skip / Assert.SkipWhen / Assert.SkipUnless — dynamic skip is v3-only.
+        pattern=re.compile(r"\bAssert\.(?:Skip|SkipWhen|SkipUnless)\s*\("),
+        compile_ability=NO_V2_EQUIVALENT,
+        coverage_impact=COVERAGE_BEARING,
+        note="Dynamic skip has no v2 equivalent (v2 skip is a static attribute). "
+        "The test runs and covers code; excluding it hides that coverage.",
+    ),
+    Construct(
+        name="test-context",
+        pattern=re.compile(r"\b(?:TestContext\.Current|ITestContext)\b"),
+        compile_ability=NO_V2_EQUIVALENT,
+        coverage_impact=COVERAGE_BEARING,
+        note="TestContext/ITestContext is v3-only with no v2 equivalent; the "
+        "test runs and covers code.",
+    ),
+    Construct(
+        name="async-lifetime-valuetask",
+        # v3's IAsyncLifetime uses ValueTask InitializeAsync/DisposeAsync;
+        # v2 uses Task. Mechanically translatable but changes signatures.
+        pattern=re.compile(
+            r"\bValueTask\s+(?:InitializeAsync|DisposeAsync)\s*\("
+        ),
+        compile_ability=CLEAN_TRANSLATABLE,
+        coverage_impact=COVERAGE_BEARING,
+        note="v3's IAsyncLifetime returns ValueTask (and extends IAsyncDisposable); "
+        "v2's returns Task. Translatable for the common CompletedTask case but not "
+        "purely a return-type swap — the interface surface differs and cached "
+        "ValueTask / IValueTaskSource bodies don't map trivially. Coverage-bearing "
+        "(runs real setup/teardown).",
+    ),
+    Construct(
+        name="theory-data-row",
+        pattern=re.compile(r"\bTheoryDataRow\b"),
+        compile_ability=CLEAN_TRANSLATABLE,
+        coverage_impact=COVERAGE_BEARING,
+        note="v3's TheoryDataRow maps to v2's TheoryData<> for value-only rows; "
+        "rows carrying per-row metadata (Skip/Explicit/Traits/TestDisplayName) "
+        "lose it in translation. Coverage-bearing (the test runs and covers code).",
+    ),
+    Construct(
+        name="autofixture-auto-data",
+        # AutoFixture.Xunit3's attribute family. The bare-name alternations are
+        # word-bounded so `MemberAutoData`/`InlineAutoData` each count once
+        # instead of also matching a nested `AutoData` — an inflated blocker
+        # count is an operator-visible defect at the gate (#1791). Matching the
+        # attribute NAME rather than requiring a leading `[` is deliberate:
+        # `[Theory, AutoData]` is the common form and has no bracket of its own.
+        pattern=re.compile(
+            r"\bAutoFixture\.Xunit3\b|\bAutoData\b|\bInlineAutoData\b"
+            r"|\bAutoMoqData\b|\bMemberAutoData\b"
+        ),
+        compile_ability=NO_V2_EQUIVALENT,
+        coverage_impact=COVERAGE_BEARING,
+        note="AutoFixture.Xunit3's [AutoData]/[InlineAutoData]/[AutoMoqData]/"
+        "[MemberAutoData] target xunit.v3's attribute surface; the generated shim "
+        "mirrors the package reference verbatim, so the linked sources fail to "
+        "compile under xunit.v2. Porting means a manual `new Fixture()` in the "
+        "body or [Theory]+[InlineData] rows — heavy if pervasive. Coverage-bearing "
+        "(these tests run and cover code). A file that names AutoFixture.Xunit2 "
+        "and not .Xunit3 is NOT flagged (see _suppressed_constructs): the "
+        "attribute names are identical between the two packages, and the v2 one "
+        "already compiles in the shim.",
+    ),
+    Construct(
+        name="assert-multiple",
+        pattern=re.compile(r"\bAssert\.Multiple\s*\("),
+        compile_ability=NO_V2_EQUIVALENT,
+        coverage_impact=COVERAGE_BEARING,
+        note="Assert.Multiple (grouped assertion reporting) is v3-only; v2 has no "
+        "equivalent, so the call must become sequential asserts. The test runs and "
+        "covers code.",
+    ),
+    Construct(
+        name="assert-equivalent",
+        pattern=re.compile(r"\bAssert\.Equivalent\s*\("),
+        compile_ability=NO_V2_EQUIVALENT,
+        coverage_impact=COVERAGE_BEARING,
+        note="Assert.Equivalent (structural equivalence) is v3-only; under v2 it "
+        "must become an explicit member-by-member or collection assertion. The "
+        "test runs and covers code.",
+    ),
+]
+
+
+@dataclass(frozen=True)
+class Finding:
+    file: str
+    line: int
+    construct: str
+    compile_ability: str
+    coverage_impact: str
+    snippet: str
+
+
+#: Constructs whose pattern alone cannot decide, keyed to the file-level signal
+#: that resolves them. See :func:`_suppressed_constructs`.
+_V2_AUTOFIXTURE = "AutoFixture.Xunit2"
+_V3_AUTOFIXTURE = "AutoFixture.Xunit3"
+
+
+def _suppressed_constructs(text: str) -> frozenset[str]:
+    """Constructs to skip for this file because a file-level signal clears them.
+
+    ``[AutoData]`` and friends are spelled IDENTICALLY by AutoFixture.Xunit2 and
+    AutoFixture.Xunit3, so the attribute name alone cannot tell a shim blocker
+    from code that already compiles under v2. When a file declares the v2
+    package and not the v3 one, it is not a blocker — flagging it would stall
+    the operator gate on nothing and, if they chose ``exclude``, drop
+    v2-compatible tests out of measurement.
+
+    Absence of both signals is NOT treated as v2: the reference may be a global
+    using or live in the ``.csproj``, so the construct is still reported and the
+    operator decides. Blocking and asking is the safe default here; silently
+    proceeding into a shim that cannot compile is not.
+    """
+    if _V2_AUTOFIXTURE in text and _V3_AUTOFIXTURE not in text:
+        return frozenset({"autofixture-auto-data"})
+    return frozenset()
+
+
+def scan_text(path: str, text: str) -> list[Finding]:
+    """Return the v3-only constructs found in ``text``, attributed to ``path``.
+
+    Line-oriented so each finding carries a 1-based line number and the source
+    snippet the operator will see at the gate.
+    """
+    findings: list[Finding] = []
+    suppressed = _suppressed_constructs(text)
+    for lineno, line in enumerate(text.splitlines(), start=1):
+        # Skip fully commented-out lines so a `// [Fact(Explicit = true)]` in
+        # dead code isn't flagged as a shim-breaker. A trailing comment on a
+        # real code line is left alone (the code before `//` still matters).
+        if line.lstrip().startswith("//"):
+            continue
+        for construct in _CONSTRUCTS:
+            if construct.name in suppressed:
+                continue
+            if construct.pattern.search(line):
+                findings.append(
+                    Finding(
+                        file=path,
+                        line=lineno,
+                        construct=construct.name,
+                        compile_ability=construct.compile_ability,
+                        coverage_impact=construct.coverage_impact,
+                        snippet=line.strip(),
+                    )
+                )
+    return findings
+
+
+def scan_file(path: Path) -> list[Finding]:
+    """Scan a single .cs file. Missing/unreadable files yield no findings."""
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except (OSError, FileNotFoundError):
+        return []
+    return scan_text(str(path), text)
+
+
+def scan_files(paths: Iterable[Path]) -> list[Finding]:
+    findings: list[Finding] = []
+    for p in paths:
+        findings.extend(scan_file(p))
+    return findings
+
+
+def files_with_findings(findings: Sequence[Finding]) -> list[str]:
+    """Distinct files carrying at least one shim-breaking construct — the
+    candidate exclusion set the human gate offers (dedup, stable order)."""
+    seen: list[str] = []
+    for f in findings:
+        if f.file not in seen:
+            seen.append(f.file)
+    return seen
+
+
+@dataclass(frozen=True)
+class Summary:
+    total: int
+    files: int
+    neutral: int
+    bearing: int
+    recommendation: str
+
+
+def summarize(findings: Sequence[Finding]) -> Summary:
+    """Roll findings up into counts plus an advisory sizing recommendation.
+
+    The recommendation is advisory only — the gate is always-ask (#1160). It
+    never auto-excludes; it tells the operator which way the blast radius leans.
+    """
+    neutral = sum(1 for f in findings if f.coverage_impact == COVERAGE_NEUTRAL)
+    bearing = sum(1 for f in findings if f.coverage_impact == COVERAGE_BEARING)
+    files = len(files_with_findings(findings))
+
+    if not findings:
+        rec = (
+            "No shim-breaking xunit.v3 constructs found — the v2 shim path is "
+            "viable as-is."
+        )
+    elif bearing == 0:
+        rec = (
+            f"{neutral} coverage-neutral hit(s) across {files} file(s) "
+            "(Explicit tests skipped by default). Excluding them from the shim "
+            "is safe PROVIDED the run does not enable explicit tests "
+            "(-explicit / xunit.runner.json explicit=on) — confirm that, then "
+            "proceeding on the shim path is reasonable."
+        )
+    else:
+        rec = (
+            f"{bearing} coverage-bearing hit(s) across {files} file(s): "
+            "excluding these from the shim blinds the loop to their coverage. "
+            "If few and acceptable, exclude and proceed; if many, skip the loop "
+            "and run a single-pass advisory with coverage-analysis off."
+        )
+    return Summary(
+        total=len(findings),
+        files=files,
+        neutral=neutral,
+        bearing=bearing,
+        recommendation=rec,
+    )
+
+
+def _cli(argv: Sequence[str]) -> int:
+    parser = argparse.ArgumentParser(
+        description="Detect and classify xunit.v3-only, shim-breaking test constructs."
+    )
+    parser.add_argument("paths", nargs="+", help="test .cs files to scan")
+    parser.add_argument(
+        "--json", action="store_true", help="emit JSON instead of a table"
+    )
+    args = parser.parse_args(list(argv))
+
+    findings = scan_files(Path(p) for p in args.paths)
+    summary = summarize(findings)
+
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "findings": [asdict(f) for f in findings],
+                    "summary": asdict(summary),
+                },
+                indent=2,
+            )
+        )
+    else:
+        for f in findings:
+            print(
+                f"{f.file}:{f.line}  {f.construct}  "
+                f"[{f.compile_ability} / coverage-{f.coverage_impact}]  {f.snippet}"
+            )
+        print(f"\n{summary.recommendation}")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(_cli(sys.argv[1:]))

package/skills/performance-benchmark/SKILL.md

@@ -0,0 +1,174 @@
+---
+name: performance-benchmark
+description: >-
+  Capture runtime performance metrics (Core Web Vitals, resource sizes, load
+  times) against defined budgets. Compare to baselines, flag regressions, and
+  maintain trend history. Complements the code-level performance-review agent
+  with actual runtime measurement.
+role: worker
+user-invocable: false
+---
+
+# Performance Benchmark Skill
+
+Measures **runtime performance** (what the user experiences) — complements the static `performance-review` agent. Uses Playwright + Chromium to load pages, collect timing via the Performance API, measure resource sizes, and compare against baselines and budgets.
+
+## Prerequisites
+
+Installed by `/project-init` for frontend projects, or install manually:
+
+```bash
+npx playwright install chromium
+```
+
+A running dev server (or accessible URL) for the pages being benchmarked.
+
+## Metrics collected
+
+### Core Web Vitals
+
+| Metric | API | Budget Default | Measures |
+|---|---|---|---|
+| **LCP** (Largest Contentful Paint) | `PerformanceObserver('largest-contentful-paint')` | ≤ 2500ms | When main content is visible |
+| **FID** (First Input Delay) | `PerformanceObserver('first-input')` | ≤ 100ms | First-interaction responsiveness |
+| **CLS** (Cumulative Layout Shift) | `PerformanceObserver('layout-shift')` | ≤ 0.1 | Visual stability during load |
+| **INP** (Interaction to Next Paint) | `PerformanceObserver('event')` | ≤ 200ms | Responsiveness across lifecycle |
+
+### Navigation Timing
+
+| Metric | API | Measures |
+|---|---|---|
+| **TTFB** | `performance.timing.responseStart - navigationStart` | Server response time |
+| **FCP** (First Contentful Paint) | `PerformanceObserver('paint')` | When first content appears |
+| **DOM Interactive** | `performance.timing.domInteractive` | When DOM is parseable |
+| **Load Complete** | `performance.timing.loadEventEnd` | Page fully loaded |
+
+### Resource metrics
+
+| Metric | Collection | Budget Default |
+|---|---|---|
+| **Total transfer size** | `performance.getEntriesByType('resource')` sum | ≤ 500KB |
+| **JS bundle size** | Resources with `.js` extension | ≤ 200KB |
+| **CSS bundle size** | Resources with `.css` extension | ≤ 50KB |
+| **Image payload** | Resources with image MIME types | ≤ 300KB |
+| **Request count** | Resource entry count | ≤ 50 |
+| **Largest resource** | Max single transfer size | ≤ 150KB |
+
+## Collection procedure
+
+Run a Playwright script that:
+
+1. Launches headless Chromium with consistent viewport (1280×720) and CPU throttling (4x for mobile, 1x for desktop)
+2. Navigates with `waitUntil: 'networkidle'`
+3. Injects a Performance Observer for Web Vitals
+4. Waits 2s after load for metrics to stabilize
+5. Collects all `performance.getEntriesByType('resource')` entries
+6. Returns structured JSON
+
+Full script template: `references/benchmark-script.md`.
+
+**Reliability**: run each page **3 times**, take the **median**. Use `--disable-gpu` and `--disable-extensions`. Clear cookies and use a fresh context per run. No network throttling by default; `--3g` flag for mobile simulation.
+
+## Modes
+
+### Baseline (`--baseline`)
+
+Capture current metrics and save as the baseline:
+
+```
+benchmarks/<slug>/baseline.json
+```
+
+Commit baselines so the team shares a reference point.
+
+### Compare (default)
+
+Run and compare against the saved baseline:
+
+| Change | Status |
+|---|---|
+| Metric worsened > 10% | `fail` (regression) |
+| Metric worsened 5–10% | `warn` (degradation) |
+| Metric improved > 10% | noted in report |
+| Within ±5% | `pass` (stable) |
+
+### Budget (`--budget`)
+
+Check absolute budgets from `performance-budget.json` at the project root:
+
+```json
+{
+  "budgets": [
+    {
+      "path": "/",
+      "metrics": {
+        "LCP": 2500,
+        "CLS": 0.1,
+        "totalTransferSize": 500000,
+        "jsSize": 200000
+      }
+    },
+    {
+      "path": "/dashboard",
+      "metrics": { "LCP": 3000, "totalTransferSize": 800000 }
+    }
+  ]
+}
+```
+
+If no budget file, fall back to defaults above.
+
+### Trend (`--trend`)
+
+Read `benchmarks/<slug>/history.jsonl` and produce a trend summary across the last N runs.
+
+## Output format (JSON)
+
+```json
+{
+  "url": "http://localhost:3000/dashboard",
+  "timestamp": "2026-04-10T14:30:00Z",
+  "runs": 3,
+  "device": "desktop",
+  "metrics": {
+    "LCP": {"median": 1850, "p95": 2100, "unit": "ms"},
+    "FCP": {"median": 920, "p95": 1050, "unit": "ms"},
+    "CLS": {"median": 0.05, "p95": 0.08, "unit": "score"},
+    "INP": {"median": 120, "p95": 180, "unit": "ms"},
+    "TTFB": {"median": 210, "p95": 280, "unit": "ms"},
+    "loadComplete": {"median": 2400, "p95": 2800, "unit": "ms"}
+  },
+  "resources": {
+    "totalTransferSize": 342000,
+    "jsSize": 156000,
+    "cssSize": 28000,
+    "imageSize": 98000,
+    "requestCount": 34,
+    "largestResource": {"url": "/assets/main.js", "size": 95000}
+  },
+  "comparison": {
+    "baseline": "2026-04-08T10:00:00Z",
+    "regressions": [
+      {"metric": "LCP", "baseline": 1600, "current": 1850, "change": "+15.6%", "severity": "fail"}
+    ],
+    "improvements": [],
+    "stable": ["FCP", "CLS", "INP", "TTFB"]
+  },
+  "budget": { "status": "pass", "violations": [] },
+  "status": "pass|warn|fail"
+}
+```
+
+Human-readable report template: [`examples/report-format.md`](examples/report-format.md).
+
+## File storage
+
+```
+benchmarks/
+├── <page-slug>/
+│   ├── baseline.json        # Committed — shared baseline
+│   └── history.jsonl        # Committed — trend data (append-only)
+└── performance-budget.json  # Committed — budget definitions
+```
+
+Page slug derived from the URL path: `/dashboard` → `dashboard`, `/` → `index`.

package/skills/performance-benchmark/examples/report-format.md

@@ -0,0 +1,43 @@
+# Report Format Example
+
+Human-readable benchmark report (emitted by `/benchmark` in prose mode).
+
+```markdown
+# Performance Benchmark: <URL>
+
+**Date**: <timestamp>
+**Device**: desktop | mobile
+**Runs**: 3 (median reported)
+
+## Core Web Vitals
+
+| Metric | Value | Budget | Baseline | Change | Status |
+|--------|-------|--------|----------|--------|--------|
+| LCP    | 1850ms | ≤2500ms | 1600ms | +15.6% | FAIL |
+| FCP    | 920ms  | —      | 900ms  | +2.2%  | PASS |
+| CLS    | 0.05   | ≤0.1   | 0.04   | +25%   | PASS |
+| INP    | 120ms  | ≤200ms | 115ms  | +4.3%  | PASS |
+
+## Resource Budget
+
+| Resource | Size | Budget | Status |
+|----------|------|--------|--------|
+| Total    | 342KB | ≤500KB | PASS |
+| JS       | 156KB | ≤200KB | PASS |
+| CSS      | 28KB  | ≤50KB  | PASS |
+| Images   | 98KB  | ≤300KB | PASS |
+| Requests | 34    | ≤50    | PASS |
+
+## Slowest Resources
+
+| Resource | Size | Time |
+|----------|------|------|
+| /assets/main.js | 95KB | 450ms |
+| ...
+
+## Regressions
+
+- **LCP**: +15.6% (1600ms → 1850ms) — investigate main.js growth
+
+## Verdict: WARN (1 regression detected)
+```