qa-engineer 0.9.0

package/shared/diagnostics/lib/qa_diagnostics/timeline.py

@@ -0,0 +1,71 @@
+"""Deterministic timeline reconstruction.
+
+Reconstructs the ordered sequence of a run — start, browser launch, navigation,
+requests and responses, console errors, assertions, failure, cleanup — from the
+execution result and the analysis findings. It records only stages for which
+there is evidence; it never invents an event to fill the shape. Reusable by
+qa-debug and qa-report.
+"""
+
+# Canonical phase order, used to sort events that share (or lack) a timestamp.
+PHASE_ORDER = [
+    "execution-start", "browser-launch", "navigation", "request", "response",
+    "console-error", "assertion", "failure", "cleanup", "execution-finish",
+]
+_PHASE_INDEX = {phase: i for i, phase in enumerate(PHASE_ORDER)}
+
+# Evidence type -> the timeline phase it contributes.
+_EVIDENCE_PHASE = {
+    "network": "response",
+    "console": "console-error",
+    "trace": "navigation",
+    "junit": "assertion",
+    "report": "assertion",
+}
+
+
+def build_timeline(execution_result, findings=None):
+    """Build an ordered timeline. Returns a list of event dicts:
+    {order, phase, detail, source, timestamp?}. Deterministic: same inputs,
+    same timeline. Only evidenced stages appear."""
+    events = []
+    execution = (execution_result or {}).get("execution", {})
+
+    started = execution.get("startedAt")
+    if started or execution_result:
+        events.append(_event("execution-start", "Run started", "execution-result", started))
+
+    # Contribute a stage per evidence entry we actually have.
+    for finding in findings or []:
+        for ev in finding.get("evidence", []):
+            phase = _EVIDENCE_PHASE.get(ev.get("type"))
+            if phase:
+                events.append(_event(phase, ev.get("description", ""), ev.get("source", "analysis"), None))
+
+    # A failure stage per failed test recorded in the result.
+    for test in (execution_result or {}).get("executed", []):
+        if test.get("status") == "failed":
+            events.append(_event("failure", f"Test failed: {test.get('title', '')}",
+                                 test.get("file", "execution-result"), None))
+
+    finished = execution.get("finishedAt")
+    if finished:
+        events.append(_event("cleanup", "Run cleaned up", "execution-result", None))
+        events.append(_event("execution-finish", "Run finished", "execution-result", finished))
+
+    # Deterministic order: by canonical phase first (it encodes the logical
+    # sequence), then by timestamp within a phase, then by insertion order. Phase
+    # order is primary because per-event wall-clock times are often unavailable.
+    for i, event in enumerate(events):
+        event["_seq"] = i
+    events.sort(key=lambda e: (_PHASE_INDEX.get(e["phase"], 99), e["timestamp"] or "", e["_seq"]))
+    for order, event in enumerate(events):
+        event["order"] = order
+        del event["_seq"]
+        if event["timestamp"] is None:
+            del event["timestamp"]
+    return events
+
+
+def _event(phase, detail, source, timestamp):
+    return {"phase": phase, "detail": detail, "source": source, "timestamp": timestamp}

package/shared/diagnostics/schemas/internal/analysis-result.schema.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:qa-pack:internal:analysis-result:1",
+  "title": "Internal analysis_result consumed by diagnostics",
+  "type": "object",
+  "required": ["findings"],
+  "additionalProperties": true,
+  "properties": {
+    "findings": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["classification", "reason"],
+        "additionalProperties": true,
+        "properties": {
+          "classification": { "type": "string" },
+          "reason": { "type": "string" },
+          "confidence": { "type": "number" },
+          "httpStatus": { "type": ["integer", "null"] },
+          "retries": { "type": "integer" },
+          "finalStatus": { "type": "string" },
+          "evidence": { "type": "array" },
+          "affectedTests": { "type": "array", "items": { "type": "string" } }
+        }
+      }
+    }
+  }
+}

package/shared/diagnostics/schemas/internal/diagnosis.schema.json

@@ -0,0 +1,55 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:qa-pack:internal:diagnosis:1",
+  "title": "Internal diagnosis (Analysis → Diagnostics output)",
+  "type": "object",
+  "required": ["entries", "timeline", "recommendations"],
+  "additionalProperties": false,
+  "properties": {
+    "entries": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["rootCause", "priority", "affectedTests"],
+        "additionalProperties": true,
+        "properties": {
+          "rootCause": {
+            "type": "object",
+            "required": ["classification", "confidence"],
+            "properties": {
+              "classification": { "type": "string", "minLength": 1 },
+              "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
+              "reason": { "type": "string" }
+            }
+          },
+          "priority": {
+            "type": "object",
+            "required": ["priority"],
+            "properties": {
+              "priority": { "type": "string", "enum": ["P1", "P2", "P3"] }
+            }
+          },
+          "affectedTests": {
+            "type": "array",
+            "items": { "type": "string" }
+          }
+        }
+      }
+    },
+    "timeline": { "type": "array" },
+    "recommendations": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["action", "priority"],
+        "additionalProperties": true,
+        "properties": {
+          "action": { "type": "string" },
+          "priority": { "type": "string" },
+          "owner": { "type": "string" },
+          "classification": { "type": "string" }
+        }
+      }
+    }
+  }
+}

package/shared/diagnostics/schemas/internal/execution-result-min.schema.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:qa-pack:internal:execution-result-min:1",
+  "title": "Minimal execution_result fields diagnostics and evals require",
+  "type": "object",
+  "additionalProperties": true,
+  "properties": {
+    "tests": {
+      "type": "object",
+      "additionalProperties": true,
+      "properties": {
+        "passed": { "type": "integer" },
+        "failed": { "type": "integer" },
+        "flaky": { "type": "integer" },
+        "skipped": { "type": "integer" }
+      }
+    },
+    "executed": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["status"],
+        "additionalProperties": true,
+        "properties": {
+          "title": { "type": "string" },
+          "status": {
+            "type": "string",
+            "enum": ["passed", "failed", "flaky", "skipped", "timedOut"]
+          }
+        }
+      }
+    }
+  }
+}

package/shared/frameworks/cypress/lib/cypress_analysis.py

@@ -0,0 +1,27 @@
+"""Cypress analysis adapter.
+
+Cypress emits JUnit XML (via a JUnit reporter), so Cypress normalization is the
+framework-agnostic JUnit parser — the adapter only points at the artifact and
+tags provenance. Like the Selenium adapter, its thinness is the proof of the
+multi-framework boundary: the contract and parsing are shared; only the artifact
+location differs.
+"""
+
+from qa_analysis import junit, taxonomy
+
+FRAMEWORK = "cypress"
+
+# Where Cypress projects conventionally write JUnit results (see cypress-artifacts.md).
+RESULT_GLOBS = ["**/results/*.xml", "**/cypress/results/*.xml", "**/junit*.xml"]
+
+
+def normalize(junit_path):
+    """Normalize a Cypress run into the shared result shape via the agnostic
+    JUnit parser. Identical output shape to any other framework."""
+    return junit.parse_junit(junit_path)
+
+
+def classify_failure(message, http_status=None):
+    """Classify a Cypress failure using the shared taxonomy — no Cypress-specific
+    classifier, because failure classes are framework-agnostic."""
+    return taxonomy.classify(message, http_status=http_status)

package/shared/frameworks/playwright/lib/playwright_analysis.py

@@ -0,0 +1,167 @@
+"""Playwright-specific analyzers: trace and JSON report.
+
+These are the Playwright *adapter* for the analysis platform. They depend on
+the framework-agnostic qa_analysis core for the evidence model, redaction, and
+the failure taxonomy; they add only what is Playwright-specific — the shape of
+a Playwright trace.zip and of Playwright's JSON reporter. Everything they emit
+is in the shared normalized shape, so nothing downstream knows it was Playwright.
+
+The adapter carries its own CLI (`python -m playwright_analysis report|trace
+<path>`) so a skill can reach it the same way it reaches every other
+deterministic tool. Framework knowledge stays inside the adapter: the core CLI
+never grows a `--framework` flag ([ADR-0013](../../../../docs/architecture/ADR-0013-framework-boundary.md)).
+
+Usage:
+  python -m playwright_analysis report <results.json>
+  python -m playwright_analysis trace <trace.zip>
+"""
+
+import argparse
+import json
+import sys
+import zipfile
+
+from qa_analysis.junit import MalformedArtifact
+from qa_analysis.redaction import redact_text
+from qa_analysis import taxonomy
+
+
+def parse_report(path):
+    """Parse Playwright's JSON reporter output into the normalized result shape.
+
+    Mirrors qa_analysis.junit's output ({tests, executed}) so a Playwright run
+    normalizes identically whether it emitted JSON or JUnit. Raises
+    MalformedArtifact on unreadable input.
+    """
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            data = json.load(handle)
+    except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc:
+        raise MalformedArtifact(f"could not parse Playwright report at {path}: {exc}") from exc
+
+    executed = []
+    for suite in data.get("suites", []):
+        _walk_suite(suite, executed)
+
+    counts = {
+        "total": len(executed),
+        "passed": sum(1 for e in executed if e["status"] == "passed"),
+        "failed": sum(1 for e in executed if e["status"] == "failed"),
+        "skipped": sum(1 for e in executed if e["status"] == "skipped"),
+    }
+    return {"tests": counts, "executed": executed}
+
+
+def _walk_suite(suite, executed):
+    for spec in suite.get("specs", []):
+        for test in spec.get("tests", []):
+            results = test.get("results", [])
+            status = _status_of(results)
+            message = ""
+            if status == "failed" and results:
+                errors = results[-1].get("errors") or []
+                if errors:
+                    message = redact_text((errors[0].get("message") or "").strip())
+            entry = {
+                "title": spec.get("title", ""),
+                "file": suite.get("file", ""),
+                "status": status,
+                "durationMs": results[-1].get("duration", 0) if results else 0,
+                "retries": max(0, len(results) - 1),
+            }
+            if message:
+                entry["message"] = message
+            executed.append(entry)
+    for child in suite.get("suites", []):
+        _walk_suite(child, executed)
+
+
+def _status_of(results):
+    if not results:
+        return "skipped"
+    final = results[-1].get("status")
+    if final in ("passed", "expected"):
+        return "flaky" if len(results) > 1 else "passed"
+    if final in ("skipped",):
+        return "skipped"
+    return "failed"
+
+
+def analyze_trace(path):
+    """Extract a deterministic summary from a Playwright trace.zip.
+
+    A trace is a zip of newline-delimited JSON event files. This lists the
+    actions, surfaces the last error, and counts console and network events —
+    enough for a diagnostic skill to reason over, without decoding the full
+    binary. Raises MalformedArtifact if the file is not a valid trace zip.
+    """
+    if not zipfile.is_zipfile(path):
+        raise MalformedArtifact(f"not a valid trace zip: {path}")
+
+    actions, console, network, errors = [], 0, 0, []
+    try:
+        with zipfile.ZipFile(path) as archive:
+            names = [n for n in archive.namelist() if n.endswith(".trace") or n.endswith(".jsonl") or "trace" in n]
+            for name in names or archive.namelist():
+                with archive.open(name) as member:
+                    for raw in member.read().decode("utf-8", "replace").splitlines():
+                        raw = raw.strip()
+                        if not raw or not raw.startswith("{"):
+                            continue
+                        try:
+                            event = json.loads(raw)
+                        except json.JSONDecodeError:
+                            continue
+                        kind = event.get("type") or event.get("kind")
+                        if kind in ("action", "before"):
+                            actions.append(redact_text(str(event.get("apiName") or event.get("method") or "")))
+                        elif kind == "console":
+                            console += 1
+                        elif kind in ("resource", "network", "http"):
+                            network += 1
+                        if event.get("error") or kind == "error":
+                            msg = event.get("error", {}).get("message") if isinstance(event.get("error"), dict) else event.get("message")
+                            if msg:
+                                errors.append(redact_text(str(msg)))
+    except (OSError, KeyError) as exc:
+        raise MalformedArtifact(f"could not read trace {path}: {exc}") from exc
+
+    last_error = errors[-1] if errors else ""
+    classification, confidence, reason = taxonomy.classify(last_error) if last_error else (taxonomy.UNKNOWN, 0.2, "No error found in trace.")
+    return {
+        "actions": [a for a in actions if a],
+        "consoleEvents": console,
+        "networkEvents": network,
+        "errors": errors,
+        "classification": classification,
+        "confidence": confidence,
+        "reason": reason,
+    }
+
+
+def main(argv=None):
+    """CLI: JSON to stdout, exit 0 on success, 2 on a malformed artifact."""
+    parser = argparse.ArgumentParser(
+        prog="playwright_analysis",
+        description="Playwright adapter: normalize a JSON report or summarize a trace",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    p = sub.add_parser("report", help="normalize Playwright's JSON reporter output")
+    p.add_argument("path")
+    p = sub.add_parser("trace", help="summarize a Playwright trace.zip")
+    p.add_argument("path")
+    args = parser.parse_args(argv)
+
+    try:
+        result = parse_report(args.path) if args.command == "report" else analyze_trace(args.path)
+    except MalformedArtifact as exc:
+        json.dump({"error": "malformed-artifact", "detail": str(exc)}, sys.stdout, indent=2, sort_keys=True)
+        sys.stdout.write("\n")
+        return 2
+    json.dump(result, sys.stdout, indent=2, sort_keys=True)
+    sys.stdout.write("\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/shared/frameworks/registry.json

@@ -0,0 +1,150 @@
+{
+  "$schema": "./registry.schema.json",
+  "version": 1,
+  "frameworks": [
+    {
+      "id": "playwright",
+      "displayName": "Playwright",
+      "supportLevel": "Production",
+      "adapterDir": "shared/frameworks/playwright",
+      "liveExecution": true,
+      "liveGeneration": true,
+      "detection": {
+        "configFiles": ["playwright.config.ts", "playwright.config.js", "playwright.config.mjs", "playwright.config.cjs"],
+        "configPrefixes": ["playwright.config."],
+        "dependencies": ["@playwright/test", "playwright"],
+        "directories": []
+      },
+      "capabilities": {
+        "detection": "Full",
+        "execution": "Full",
+        "generation": "Full",
+        "artifactMapping": "Full",
+        "normalization": "Full",
+        "analysis": "Full",
+        "diagnostics": "Full",
+        "reporting": "Full"
+      }
+    },
+    {
+      "id": "selenium",
+      "displayName": "Selenium",
+      "supportLevel": "Beta",
+      "adapterDir": "shared/frameworks/selenium",
+      "liveExecution": false,
+      "liveGeneration": false,
+      "detection": {
+        "configFiles": [],
+        "configPrefixes": [],
+        "dependencies": ["selenium-webdriver", "selenium"],
+        "directories": []
+      },
+      "capabilities": {
+        "detection": "Full",
+        "execution": "Adapter-complete",
+        "generation": "Adapter-complete",
+        "artifactMapping": "Full",
+        "normalization": "Full",
+        "analysis": "Full",
+        "diagnostics": "Full",
+        "reporting": "Full"
+      }
+    },
+    {
+      "id": "cypress",
+      "displayName": "Cypress",
+      "supportLevel": "Beta",
+      "adapterDir": "shared/frameworks/cypress",
+      "liveExecution": false,
+      "liveGeneration": false,
+      "detection": {
+        "configFiles": ["cypress.config.js", "cypress.config.ts", "cypress.config.mjs"],
+        "configPrefixes": [],
+        "dependencies": ["cypress"],
+        "directories": ["cypress"]
+      },
+      "capabilities": {
+        "detection": "Full",
+        "execution": "Adapter-complete",
+        "generation": "Adapter-complete",
+        "artifactMapping": "Full",
+        "normalization": "Full",
+        "analysis": "Full",
+        "diagnostics": "Full",
+        "reporting": "Full"
+      }
+    },
+    {
+      "id": "webdriverio",
+      "displayName": "WebdriverIO",
+      "supportLevel": "Beta",
+      "adapterDir": "shared/frameworks/webdriverio",
+      "liveExecution": false,
+      "liveGeneration": false,
+      "detection": {
+        "configFiles": ["wdio.conf.js", "wdio.conf.ts", "wdio.conf.mjs"],
+        "configPrefixes": [],
+        "dependencies": ["webdriverio", "@wdio/cli"],
+        "directories": []
+      },
+      "capabilities": {
+        "detection": "Full",
+        "execution": "Adapter-complete",
+        "generation": "Adapter-complete",
+        "artifactMapping": "Full",
+        "normalization": "Full",
+        "analysis": "Full",
+        "diagnostics": "Full",
+        "reporting": "Full"
+      }
+    },
+    {
+      "id": "robot",
+      "displayName": "Robot Framework",
+      "supportLevel": "Planning",
+      "adapterDir": null,
+      "liveExecution": false,
+      "liveGeneration": false,
+      "detection": {
+        "configFiles": [],
+        "configPrefixes": [],
+        "dependencies": [],
+        "directories": []
+      },
+      "capabilities": {
+        "detection": "Planning",
+        "execution": "—",
+        "generation": "—",
+        "artifactMapping": "—",
+        "normalization": "—",
+        "analysis": "—",
+        "diagnostics": "—",
+        "reporting": "—"
+      }
+    },
+    {
+      "id": "appium",
+      "displayName": "Appium",
+      "supportLevel": "Planning",
+      "adapterDir": null,
+      "liveExecution": false,
+      "liveGeneration": false,
+      "detection": {
+        "configFiles": [],
+        "configPrefixes": [],
+        "dependencies": [],
+        "directories": []
+      },
+      "capabilities": {
+        "detection": "Planning",
+        "execution": "Planning",
+        "generation": "—",
+        "artifactMapping": "—",
+        "normalization": "—",
+        "analysis": "—",
+        "diagnostics": "—",
+        "reporting": "—"
+      }
+    }
+  ]
+}

package/shared/frameworks/registry.mjs

@@ -0,0 +1,37 @@
+// Load the canonical framework registry. All detection and matrix checks
+// derive from shared/frameworks/registry.json — do not duplicate framework lists.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+let cached = null;
+
+export function registryPath() {
+  return path.join(here, 'registry.json');
+}
+
+export function loadFrameworkRegistry() {
+  if (cached) return cached;
+  const raw = JSON.parse(fs.readFileSync(registryPath(), 'utf8'));
+  cached = raw;
+  return raw;
+}
+
+export function listFrameworks() {
+  return loadFrameworkRegistry().frameworks;
+}
+
+export function getFramework(id) {
+  return listFrameworks().find((f) => f.id === id) ?? null;
+}
+
+export function liveExecutionFrameworks() {
+  return listFrameworks().filter((f) => f.liveExecution).map((f) => f.id);
+}
+
+export function supportedFrameworks() {
+  return listFrameworks().filter((f) => f.supportLevel !== 'Planning');
+}

package/shared/frameworks/registry.schema.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:qa-pack:framework-registry:1",
+  "type": "object",
+  "required": ["version", "frameworks"],
+  "additionalProperties": false,
+  "properties": {
+    "version": { "type": "integer", "minimum": 1 },
+    "frameworks": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": [
+          "id",
+          "displayName",
+          "supportLevel",
+          "adapterDir",
+          "liveExecution",
+          "liveGeneration",
+          "detection",
+          "capabilities"
+        ],
+        "additionalProperties": false,
+        "properties": {
+          "id": { "type": "string", "pattern": "^[a-z][a-z0-9-]*$" },
+          "displayName": { "type": "string", "minLength": 1 },
+          "supportLevel": {
+            "type": "string",
+            "enum": ["Production", "Beta", "Experimental", "Planning"]
+          },
+          "adapterDir": { "type": ["string", "null"] },
+          "liveExecution": { "type": "boolean" },
+          "liveGeneration": { "type": "boolean" },
+          "detection": {
+            "type": "object",
+            "required": ["configFiles", "configPrefixes", "dependencies", "directories"],
+            "additionalProperties": false,
+            "properties": {
+              "configFiles": { "type": "array", "items": { "type": "string" } },
+              "configPrefixes": { "type": "array", "items": { "type": "string" } },
+              "dependencies": { "type": "array", "items": { "type": "string" } },
+              "directories": { "type": "array", "items": { "type": "string" } }
+            }
+          },
+          "capabilities": {
+            "type": "object",
+            "required": [
+              "detection",
+              "execution",
+              "generation",
+              "artifactMapping",
+              "normalization",
+              "analysis",
+              "diagnostics",
+              "reporting"
+            ],
+            "additionalProperties": false,
+            "properties": {
+              "detection": { "type": "string" },
+              "execution": { "type": "string" },
+              "generation": { "type": "string" },
+              "artifactMapping": { "type": "string" },
+              "normalization": { "type": "string" },
+              "analysis": { "type": "string" },
+              "diagnostics": { "type": "string" },
+              "reporting": { "type": "string" }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/shared/frameworks/selenium/lib/selenium_analysis.py

@@ -0,0 +1,28 @@
+"""Selenium analysis adapter.
+
+Selenium's runners (JUnit, TestNG, pytest) emit JUnit XML, so Selenium
+normalization *is* the framework-agnostic JUnit parser — the adapter only points
+at the right artifact and tags provenance. This is the proof of the pack's
+multi-framework claim: the second framework's analysis is a few lines, because
+the contract and the parsing are shared, and only the artifact location differs.
+"""
+
+from qa_analysis import junit, taxonomy
+
+FRAMEWORK = "selenium"
+
+# Where Selenium projects conventionally write results (documented in
+# selenium-artifacts.md); discovery uses these, the parser is shared.
+RESULT_GLOBS = ["**/target/surefire-reports/*.xml", "**/test-results/*.xml", "**/junit*.xml"]
+
+
+def normalize(junit_path):
+    """Normalize a Selenium run into the shared result shape via the agnostic
+    JUnit parser. Identical output shape to any other framework's normalization."""
+    return junit.parse_junit(junit_path)
+
+
+def classify_failure(message, http_status=None):
+    """Classify a Selenium failure using the shared taxonomy — no Selenium-specific
+    classifier, because failure classes are framework-agnostic."""
+    return taxonomy.classify(message, http_status=http_status)