pi-dev-team 0.2.0

package/scripts/gherkin_effectiveness_rollup.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""gherkin_effectiveness_rollup.py — roll up gherkin-derive scenario data
+against the coverage/mutation measurements the plugin already computes,
+so there is a signal on whether BDD-derived tests catch more real defects
+than ad hoc ones (issue #1296).
+
+Today `/gherkin-derive` writes a surface inventory (`gherkin.md`) and
+`/gherkin-public` writes a scenario→Story map (`gherkin-bindings.json`), while
+`/coverage-baseline`, `/coverage-delta`, and the mutation baseline/history
+files separately track coverage and mutation-survivor counts. Nothing
+correlates the two. This script reads whatever of those files exist for one
+workflow run and emits one JSONL record per discovered scenario to
+`.claude/metrics/gherkin-derive-effectiveness.jsonl` (append-only, per the
+`performance-metrics` skill's convention).
+
+**Attribution granularity is honest, not exact.** No file in this plugin
+today maps an individual scenario to the specific test(s) that exercise it
+at the mutation/coverage level — `gherkin-bindings.json` maps a scenario to a
+*Story* (which may bind several tests), and coverage/mutation deltas are
+measured per-file or repo-wide, not per-scenario. This script therefore
+reports the best available repo/workflow-level coverage and mutation deltas
+alongside each scenario's provenance and binding mode, and does NOT claim a
+delta is caused solely by that scenario's test. Treat the emitted
+`coverage_delta` / `mutation_delta` fields as workflow-level context for the
+scenario, not as isolated per-scenario attribution.
+
+The surface-inventory table format is not fixed by `/gherkin-derive` today,
+so parsing is deliberately lenient: any markdown table under the file is
+scanned for columns whose header contains "surface", "source", "provenance",
+or "mode" (case-insensitive); a file with no such table yields a warning on
+stderr and zero scenario records, rather than crashing.
+
+Stdlib-only. (ADR 0014/0015).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+_HOOKS_LIB_DIR = Path(__file__).resolve().parent.parent / "hooks" / "lib"
+if str(_HOOKS_LIB_DIR) not in sys.path:
+    sys.path.insert(0, str(_HOOKS_LIB_DIR))
+
+import artifact_paths
+from atomic_state import append_line_locked
+
+_TABLE_ROW_RE = re.compile(r"^\s*\|(.+)\|\s*$")
+_SEPARATOR_CELL_RE = re.compile(r"^\s*:?-{2,}:?\s*$")
+
+# Header keywords this parser recognizes, mapped to the output field name.
+_COLUMN_KEYWORDS: dict[str, str] = {
+    "surface": "surface",
+    "source": "discovery_source",
+    "provenance": "provenance",
+    "mode": "binding_mode",
+    "file": "files",
+}
+
+
+class RollupWarning(Exception):
+    """Non-fatal condition worth surfacing on stderr; callers keep going."""
+
+
+def _split_row(line: str) -> list[str]:
+    return [cell.strip() for cell in line.strip().strip("|").split("|")]
+
+
+def parse_surface_table(text: str) -> list[dict[str, str]]:
+    """Return one dict per data row of the first markdown table whose header
+    matches at least one column in `_COLUMN_KEYWORDS`. Returns [] if no such
+    table is found."""
+    lines = text.splitlines()
+    i = 0
+    while i < len(lines) - 1:
+        header_match = _TABLE_ROW_RE.match(lines[i])
+        sep_match = _TABLE_ROW_RE.match(lines[i + 1]) if header_match else None
+        if header_match and sep_match:
+            sep_cells = _split_row(lines[i + 1])
+            if all(_SEPARATOR_CELL_RE.match(c) for c in sep_cells if c):
+                header_cells = [c.lower() for c in _split_row(lines[i])]
+                field_for_col = {}
+                for idx, cell in enumerate(header_cells):
+                    for keyword, field in _COLUMN_KEYWORDS.items():
+                        if keyword in cell:
+                            field_for_col[idx] = field
+                            break
+                if field_for_col:
+                    rows: list[dict[str, str]] = []
+                    j = i + 2
+                    while j < len(lines) and _TABLE_ROW_RE.match(lines[j]):
+                        cells = _split_row(lines[j])
+                        row = {
+                            field: cells[idx]
+                            for idx, field in field_for_col.items()
+                            if idx < len(cells)
+                        }
+                        if row:
+                            rows.append(row)
+                        j += 1
+                    return rows
+        i += 1
+    return []
+
+
+def load_scenarios(gherkin_md: Path | None) -> list[dict[str, str]]:
+    if gherkin_md is None:
+        return []
+    if not gherkin_md.is_file():
+        raise RollupWarning(f"surface inventory not found: {gherkin_md}")
+    rows = parse_surface_table(gherkin_md.read_text(encoding="utf-8"))
+    if not rows:
+        raise RollupWarning(f"no recognizable surface table in: {gherkin_md}")
+    return rows
+
+
+def load_json(path: Path | None) -> dict | None:
+    if path is None:
+        return None
+    if not path.is_file():
+        return None
+    with path.open(encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+def load_bindings(bindings_json: Path | None) -> dict[str, object]:
+    data = load_json(bindings_json)
+    return data if isinstance(data, dict) else {}
+
+
+def scenario_key(surface: str) -> str:
+    """Best-effort key to look up a surface in `gherkin-bindings.json`, which
+    is keyed `<feature-file>::<scenario-name>`. The inventory table has no
+    fixed scenario-name column, so this matches by surface/feature-file
+    prefix rather than requiring an exact key."""
+    return surface.strip()
+
+
+def coverage_delta(baseline: dict | None, current: dict | None) -> dict | None:
+    if not baseline or not current:
+        return None
+    delta = {}
+    for field in ("line_pct", "branch_pct"):
+        b, c = baseline.get(field), current.get(field)
+        if isinstance(b, (int, float)) and isinstance(c, (int, float)):
+            delta[field] = round(c - b, 2)
+    return delta or None
+
+
+def mutation_delta(baseline: dict | None, current: dict | None) -> dict | None:
+    if not baseline or not current:
+        return None
+    b, c = baseline.get("survivors_after"), current.get("survivors_after")
+    if isinstance(b, (int, float)) and isinstance(c, (int, float)):
+        return {"survivors_after_delta": c - b}
+    return None
+
+
+def build_records(
+    scenarios: list[dict[str, str]],
+    bindings: dict[str, object],
+    cov_delta: dict | None,
+    mut_delta: dict | None,
+) -> list[dict]:
+    records = []
+    for row in scenarios:
+        surface = row.get("surface", "")
+        record = {
+            "surface": surface or None,
+            "discovery_source": row.get("discovery_source"),
+            "provenance": row.get("provenance"),
+            "binding_mode": row.get("binding_mode"),
+            "bound_story": bindings.get(scenario_key(surface)),
+            "coverage_delta": cov_delta,
+            "mutation_delta": mut_delta,
+        }
+        records.append(record)
+    return records
+
+
+def append_jsonl(records: list[dict], out_path: Path) -> int:
+    """Append each record as one JSON line, serialized via
+    `atomic_state.append_line_locked` (#1901) so two concurrent rollup runs
+    writing to the same stream cannot interleave their bytes into one
+    corrupted line — matching the pattern #1896 applied to the plugin's
+    other `.claude/metrics/*.jsonl` appenders. `fail_open=False` re-raises a
+    write error after the lock releases rather than swallowing it, matching
+    this function's previous behavior (a bare `open()`/`write()` that
+    propagated any OSError to the caller)."""
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    for record in records:
+        line = json.dumps(record, sort_keys=True) + "\n"
+        append_line_locked(out_path, line, fail_open=False)
+    return len(records)
+
+
+def main(argv: list[str] | None = None) -> int:
+    # Resolved per-invocation (not at import time): shells out to git via
+    # artifact_paths.metrics_dir(), and freezing that at module-import time
+    # would pin the resolved root to whatever cwd happened to be active
+    # during import (e.g. test collection), not the invocation's cwd.
+    default_out = artifact_paths.metrics_dir() / "gherkin-derive-effectiveness.jsonl"
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--gherkin-md", type=Path, help="surface inventory markdown (gherkin.md)")
+    parser.add_argument("--bindings-json", type=Path, help="gherkin-bindings.json, if present")
+    parser.add_argument("--baseline-coverage", type=Path, help="baseline-coverage.json")
+    parser.add_argument("--current-coverage", type=Path, help="latest coverage measurement (same shape as baseline-coverage.json)")
+    parser.add_argument("--baseline-mutation", type=Path, help="baseline-mutation.json")
+    parser.add_argument("--current-mutation", type=Path, help="latest mutation measurement (same shape as baseline-mutation.json)")
+    parser.add_argument("--out", type=Path, default=default_out, help=f"output JSONL path (default: {default_out})")
+    args = parser.parse_args(argv)
+
+    try:
+        scenarios = load_scenarios(args.gherkin_md)
+    except RollupWarning as exc:
+        print(f"gherkin_effectiveness_rollup: {exc}", file=sys.stderr)
+        scenarios = []
+
+    if not scenarios:
+        print("gherkin_effectiveness_rollup: no scenarios to roll up; nothing written", file=sys.stderr)
+        return 0
+
+    bindings = load_bindings(args.bindings_json)
+    cov_delta = coverage_delta(load_json(args.baseline_coverage), load_json(args.current_coverage))
+    mut_delta = mutation_delta(load_json(args.baseline_mutation), load_json(args.current_mutation))
+
+    records = build_records(scenarios, bindings, cov_delta, mut_delta)
+    count = append_jsonl(records, args.out)
+    print(f"gherkin_effectiveness_rollup: wrote {count} record(s) to {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/gherkin_failure_path_gate.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""gherkin_failure_path_gate.py — flag Feature: blocks with no failure-path
+scenario (issue #1420).
+
+Mirrors `gherkin_stub_gate.py`'s shape: scans `.feature` files under one or
+more directories, parses each `Feature:` block's scenarios, and flags any
+block whose scenarios (title + step text, case-insensitive) match none of a
+keyword list. This is a deliberately simple, best-effort heuristic, not a
+semantic classifier — both false negatives (a genuine failure scenario
+phrased without any listed keyword) and false positives (a happy-path
+scenario coincidentally containing a listed substring, e.g. "does not exceed
+the limit") are possible and accepted; `--keyword`/`--extra-keyword` let a
+repo tune the list.
+
+Stdlib-only. (ADR 0014/0015).
+
+Usage:
+    python3 gherkin_failure_path_gate.py --dir <dir> [--dir <dir> ...] [--json]
+    python3 gherkin_failure_path_gate.py --dir features --extra-keyword declined
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+_HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(_HERE / "lib"))
+
+from _gherkin_text import FEATURE_PREFIX as _FEATURE_PREFIX
+from _gherkin_text import SCENARIO_OUTLINE_PREFIX as _SCENARIO_OUTLINE_PREFIX
+from _gherkin_text import SCENARIO_PREFIX as _SCENARIO_PREFIX
+from _gherkin_text import safe_for_terminal as _safe_for_terminal
+from _gherkin_text import stripped as _stripped
+from _gherkin_text import trim_trailing_tag_run as _trim_trailing_tag_run
+from _vendored_tree import find_files as _find_files
+
+# Entries are literal substrings, not stems — "exceeds" does not match
+# "exceed" — matching gherkin_feature_merge.py's `is_stale` in spirit (a
+# deliberately simple, best-effort check, not a semantic classifier). A
+# cleanup pass "fixing" what looks like a stray verb-form mismatch would
+# silently change gate behavior: the documented false-positive test
+# (test_extra_keyword_can_produce_a_documented_false_positive) specifically
+# relies on "exceed" being absent from this list by default. "exceeds" is
+# the intended default per issue #1420 — the false positive it enables (a
+# scenario using "exceed" instead) is accepted and demonstrated by that
+# same test via --extra-keyword, not an oversight.
+DEFAULT_KEYWORDS = (
+    "invalid",
+    "error",
+    "fail",
+    "unauthorized",
+    "not found",
+    "denied",
+    "timeout",
+    "exceeds",
+    "missing",
+    "malformed",
+)
+
+
+def find_feature_files(directories: list) -> list:
+    """Return every `.feature` file under `directories`, pruning vendored
+    trees, sorted for deterministic output."""
+    return _find_files(directories, lambda path: path.suffix == ".feature")
+
+
+def parse_features(text: str, path) -> list:
+    """Return one entry per `Feature:` block: {file, feature_title, line,
+    scenario_titles, scenario_text} — scenario_text is every non-header line
+    in the block, used for keyword matching against step text as well as
+    titles. `file` is set here from `path` (a caller no longer patches it in
+    afterward) so a feature dict is never partially constructed — the
+    "title" key is also named `feature_title` here to match every other
+    consumer of this concept (`gherkin_feature_merge.py`'s `--feature-title`,
+    and this module's own finding dict below)."""
+    lines = text.splitlines(keepends=True)
+    features = []
+    header_indices = [
+        i for i, line in enumerate(lines) if _stripped(line).strip().startswith(_FEATURE_PREFIX)
+    ]
+    for idx, header_index in enumerate(header_indices):
+        title = _stripped(lines[header_index]).strip()[len(_FEATURE_PREFIX) :].strip()
+        end_index = header_indices[idx + 1] if idx + 1 < len(header_indices) else len(lines)
+        # A `@tag`/blank-line run immediately preceding the next Feature:
+        # header belongs to that next block, not this one. Without this, a
+        # tag like `@error-handling` on the *following* Feature block leaks
+        # into this block's scenario_text, and a keyword match against that
+        # leaked tag can make this block pass the gate even though it has no
+        # failure-path scenario of its own.
+        end_index = _trim_trailing_tag_run(lines, header_index, end_index)
+        body = lines[header_index + 1 : end_index]
+        scenario_titles = []
+        for line in body:
+            stripped = _stripped(line).strip()
+            if stripped.startswith(_SCENARIO_OUTLINE_PREFIX):
+                scenario_titles.append(stripped[len(_SCENARIO_OUTLINE_PREFIX) :].strip())
+            elif stripped.startswith(_SCENARIO_PREFIX):
+                scenario_titles.append(stripped[len(_SCENARIO_PREFIX) :].strip())
+        features.append(
+            {
+                "file": str(path),
+                "feature_title": title,
+                "line": header_index + 1,
+                "scenario_titles": scenario_titles,
+                "scenario_text": "".join(body),
+            }
+        )
+    return features
+
+
+def find_missing_failure_path(features: list, keywords) -> list:
+    """Return {file, line, feature_title} for every feature with zero
+    scenarios (title + step text, case-insensitive) matching `keywords`."""
+    findings = []
+    lowered_keywords = [k.lower() for k in keywords]
+    for feature in features:
+        haystack = (feature["scenario_text"] + " " + " ".join(feature["scenario_titles"])).lower()
+        if not any(keyword in haystack for keyword in lowered_keywords):
+            findings.append(
+                {
+                    "file": feature["file"],
+                    "line": feature["line"],
+                    "feature_title": feature["feature_title"],
+                }
+            )
+    return findings
+
+
+def main(argv: list | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="gherkin_failure_path_gate.py",
+        description="Flag Feature: blocks with no failure-path scenario.",
+    )
+    parser.add_argument("--dir", dest="dirs", action="append", type=Path, required=True)
+    parser.add_argument("--json", action="store_true")
+    parser.add_argument(
+        "--keyword",
+        dest="keywords",
+        action="append",
+        help=(
+            "Discard the default keyword list and use only the --keyword "
+            "value(s) given (repeatable) — this REPLACES the defaults, it "
+            "does not add to them. To add one term on top of the defaults, "
+            "use --extra-keyword instead."
+        ),
+    )
+    parser.add_argument(
+        "--extra-keyword",
+        dest="extra_keywords",
+        action="append",
+        default=[],
+        help="Add to the default keyword list (repeatable).",
+    )
+    args = parser.parse_args(argv)
+
+    keywords = list(args.keywords) if args.keywords else list(DEFAULT_KEYWORDS)
+    keywords.extend(args.extra_keywords)
+
+    files = find_feature_files(args.dirs)
+
+    if not files:
+        # A --dir that doesn't exist (or resolves to zero .feature files)
+        # used to fall through to "OK: 0 Feature block(s) scanned, all have
+        # a failure-path scenario" below — an affirmative all-clear for zero
+        # evidence. --dir is not always typed by a human: gherkin-derive
+        # composes it from a probe of the target repo, so a gate that can't
+        # find anything to scan must say so distinctly, not pass.
+        dirs_display = ", ".join(str(d) for d in args.dirs)
+        message = f"no .feature files found under {dirs_display} — gate did not run"
+        if args.json:
+            print(json.dumps({"scanned": [], "findings": [], "warning": message}, indent=2))
+        else:
+            # --dir is composed by gherkin-derive from a probe of the target
+            # repo, not always typed by a human — the same untrusted-path
+            # class as a scanned file's path, sanitized the same way.
+            print(f"WARN: no .feature files found under {_safe_for_terminal(dirs_display)} — gate did not run")
+        return 2
+
+    all_features = []
+    for path in files:
+        text = path.read_text(encoding="utf-8", errors="replace")
+        all_features.extend(parse_features(text, path))
+
+    findings = find_missing_failure_path(all_features, keywords)
+
+    if args.json:
+        print(json.dumps({"scanned": [str(f) for f in files], "findings": findings}, indent=2))
+        return 1 if findings else 0
+
+    if findings:
+        print(f"FAIL: {len(findings)} Feature block(s) missing a failure-path scenario:")
+        for entry in findings:
+            path = _safe_for_terminal(entry["file"])
+            title = _safe_for_terminal(entry["feature_title"])
+            print(f"  - {path}:{entry['line']} — {title}")
+        return 1
+
+    print(f"OK: {len(all_features)} Feature block(s) scanned, all have a failure-path scenario.")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())