qa-engineer 0.9.2 → 0.11.0

package/shared/diagnostics/lib/qa_diagnostics/root_cause.py

@@ -1,89 +0,0 @@
-"""Deterministic root-cause analysis.
-
-Turns a failure signal into a classified root cause with the four things every
-classification must carry: a taxonomy class (with confidence and evidence-backed
-reason), a recommended action, and an owner. It reuses the analysis platform's
-failure taxonomy; it adds the ownership and recommendation mappings and the
-metadata-driven classes (flaky) the taxonomy cannot infer from a message alone.
-
-No unsupported conclusions: a signal that matches no rule is `unknown`.
-"""
-
-from qa_analysis import taxonomy
-
-# classification -> the party that typically owns the fix.
-OWNERSHIP = {
-    taxonomy.ASSERTION: "test-author-or-product",
-    taxonomy.LOCATOR: "test-author",
-    taxonomy.TIMEOUT: "test-author-or-environment",
-    taxonomy.NETWORK: "backend-or-infrastructure",
-    taxonomy.AUTH: "auth-or-test-setup",
-    taxonomy.AUTHORIZATION: "permissions-or-test-account",
-    taxonomy.ENVIRONMENT: "environment-owner",
-    taxonomy.CONFIGURATION: "config-owner",
-    taxonomy.INFRASTRUCTURE: "ci-or-infrastructure",
-    taxonomy.TEST_DATA: "test-data-owner",
-    taxonomy.APPLICATION_BUG: "product",
-    taxonomy.FRAMEWORK: "framework-or-driver",
-    taxonomy.FLAKY: "test-author",
-    taxonomy.UNKNOWN: "needs-triage",
-}
-
-# classification -> the safe recommended action (implements the analysis
-# platform's recommendation-guidelines; never recommends forcing a pass).
-RECOMMENDATION = {
-    taxonomy.ASSERTION: "Confirm whether the app or the expectation is wrong; fix whichever is genuinely incorrect.",
-    taxonomy.LOCATOR: "Inspect the current DOM and update the locator to target the same element.",
-    taxonomy.TIMEOUT: "Investigate the slowness; raise a wait only if the operation is legitimately slower.",
-    taxonomy.NETWORK: "Check the upstream service and the request; retry only if the failure is genuinely transient.",
-    taxonomy.AUTH: "Fix the credentials or auth setup; do not weaken the authentication check.",
-    taxonomy.AUTHORIZATION: "Grant the test account the needed permission or use an authorized role; do not bypass the check.",
-    taxonomy.ENVIRONMENT: "Fix the environment (base URL, service availability); the test is likely fine.",
-    taxonomy.CONFIGURATION: "Correct the configuration; do not work around it in the test.",
-    taxonomy.INFRASTRUCTURE: "Escalate to CI or infrastructure owners; add resources, do not shrink the suite.",
-    taxonomy.TEST_DATA: "Repair or reseed the data; do not delete the assertion that caught the gap.",
-    taxonomy.APPLICATION_BUG: "File a bug against the product; do NOT modify the test to pass.",
-    taxonomy.FRAMEWORK: "Update or pin the framework/driver; report upstream if it is a genuine defect.",
-    taxonomy.FLAKY: "Stabilize the test (fix the race or synchronization); quarantine only with a tracking issue.",
-    taxonomy.UNKNOWN: "Investigate further; the evidence was insufficient to classify.",
-}
-
-
-def analyze(signal):
-    """Classify a failure signal into a root cause.
-
-    signal is a dict that may contain: message, httpStatus, retries, finalStatus,
-    evidence (a list of evidence refs). Returns a dict:
-    {classification, confidence, reason, ownership, recommendation, evidence}.
-    """
-    message = signal.get("message", "")
-    http_status = signal.get("httpStatus")
-    retries = signal.get("retries", 0) or 0
-    final_status = signal.get("finalStatus")
-
-    provided = signal.get("classification")
-
-    # Flakiness is a metadata signal, not a message pattern: a test that needed a
-    # retry to pass, or is explicitly flagged flaky, is nondeterministic.
-    if final_status == "flaky" or (retries > 0 and final_status in ("passed", "flaky")):
-        classification, confidence, reason = (
-            taxonomy.FLAKY, 0.8,
-            "The test passed only after a retry, indicating nondeterministic behavior.",
-        )
-    elif provided in taxonomy.CLASSES:
-        # The analysis platform already classified this deterministically; trust it
-        # rather than re-deriving from the message.
-        classification = provided
-        confidence = signal.get("confidence", 0.8)
-        reason = signal.get("reason") or f"Classified {provided} by the analysis platform."
-    else:
-        classification, confidence, reason = taxonomy.classify(message, http_status=http_status)
-
-    return {
-        "classification": classification,
-        "confidence": confidence,
-        "reason": reason,
-        "ownership": OWNERSHIP.get(classification, "needs-triage"),
-        "recommendation": RECOMMENDATION.get(classification, RECOMMENDATION[taxonomy.UNKNOWN]),
-        "evidence": signal.get("evidence", []),
-    }

package/shared/diagnostics/lib/qa_diagnostics/timeline.py

@@ -1,71 +0,0 @@
-"""Deterministic timeline reconstruction.
-
-Reconstructs the ordered sequence of a run — start, browser launch, navigation,
-requests and responses, console errors, assertions, failure, cleanup — from the
-execution result and the analysis findings. It records only stages for which
-there is evidence; it never invents an event to fill the shape. Reusable by
-qa-debug and qa-report.
-"""
-
-# Canonical phase order, used to sort events that share (or lack) a timestamp.
-PHASE_ORDER = [
-    "execution-start", "browser-launch", "navigation", "request", "response",
-    "console-error", "assertion", "failure", "cleanup", "execution-finish",
-]
-_PHASE_INDEX = {phase: i for i, phase in enumerate(PHASE_ORDER)}
-
-# Evidence type -> the timeline phase it contributes.
-_EVIDENCE_PHASE = {
-    "network": "response",
-    "console": "console-error",
-    "trace": "navigation",
-    "junit": "assertion",
-    "report": "assertion",
-}
-
-
-def build_timeline(execution_result, findings=None):
-    """Build an ordered timeline. Returns a list of event dicts:
-    {order, phase, detail, source, timestamp?}. Deterministic: same inputs,
-    same timeline. Only evidenced stages appear."""
-    events = []
-    execution = (execution_result or {}).get("execution", {})
-
-    started = execution.get("startedAt")
-    if started or execution_result:
-        events.append(_event("execution-start", "Run started", "execution-result", started))
-
-    # Contribute a stage per evidence entry we actually have.
-    for finding in findings or []:
-        for ev in finding.get("evidence", []):
-            phase = _EVIDENCE_PHASE.get(ev.get("type"))
-            if phase:
-                events.append(_event(phase, ev.get("description", ""), ev.get("source", "analysis"), None))
-
-    # A failure stage per failed test recorded in the result.
-    for test in (execution_result or {}).get("executed", []):
-        if test.get("status") == "failed":
-            events.append(_event("failure", f"Test failed: {test.get('title', '')}",
-                                 test.get("file", "execution-result"), None))
-
-    finished = execution.get("finishedAt")
-    if finished:
-        events.append(_event("cleanup", "Run cleaned up", "execution-result", None))
-        events.append(_event("execution-finish", "Run finished", "execution-result", finished))
-
-    # Deterministic order: by canonical phase first (it encodes the logical
-    # sequence), then by timestamp within a phase, then by insertion order. Phase
-    # order is primary because per-event wall-clock times are often unavailable.
-    for i, event in enumerate(events):
-        event["_seq"] = i
-    events.sort(key=lambda e: (_PHASE_INDEX.get(e["phase"], 99), e["timestamp"] or "", e["_seq"]))
-    for order, event in enumerate(events):
-        event["order"] = order
-        del event["_seq"]
-        if event["timestamp"] is None:
-            del event["timestamp"]
-    return events
-
-
-def _event(phase, detail, source, timestamp):
-    return {"phase": phase, "detail": detail, "source": source, "timestamp": timestamp}

package/shared/frameworks/cypress/lib/cypress_analysis.py

@@ -1,27 +0,0 @@
-"""Cypress analysis adapter.
-
-Cypress emits JUnit XML (via a JUnit reporter), so Cypress normalization is the
-framework-agnostic JUnit parser — the adapter only points at the artifact and
-tags provenance. Like the Selenium adapter, its thinness is the proof of the
-multi-framework boundary: the contract and parsing are shared; only the artifact
-location differs.
-"""
-
-from qa_analysis import junit, taxonomy
-
-FRAMEWORK = "cypress"
-
-# Where Cypress projects conventionally write JUnit results (see cypress-artifacts.md).
-RESULT_GLOBS = ["**/results/*.xml", "**/cypress/results/*.xml", "**/junit*.xml"]
-
-
-def normalize(junit_path):
-    """Normalize a Cypress run into the shared result shape via the agnostic
-    JUnit parser. Identical output shape to any other framework."""
-    return junit.parse_junit(junit_path)
-
-
-def classify_failure(message, http_status=None):
-    """Classify a Cypress failure using the shared taxonomy — no Cypress-specific
-    classifier, because failure classes are framework-agnostic."""
-    return taxonomy.classify(message, http_status=http_status)

package/shared/frameworks/playwright/lib/playwright_analysis.py

@@ -1,167 +0,0 @@
-"""Playwright-specific analyzers: trace and JSON report.
-
-These are the Playwright *adapter* for the analysis platform. They depend on
-the framework-agnostic qa_analysis core for the evidence model, redaction, and
-the failure taxonomy; they add only what is Playwright-specific — the shape of
-a Playwright trace.zip and of Playwright's JSON reporter. Everything they emit
-is in the shared normalized shape, so nothing downstream knows it was Playwright.
-
-The adapter carries its own CLI (`python -m playwright_analysis report|trace
-<path>`) so a skill can reach it the same way it reaches every other
-deterministic tool. Framework knowledge stays inside the adapter: the core CLI
-never grows a `--framework` flag ([ADR-0013](../../../../docs/architecture/ADR-0013-framework-boundary.md)).
-
-Usage:
-  python -m playwright_analysis report <results.json>
-  python -m playwright_analysis trace <trace.zip>
-"""
-
-import argparse
-import json
-import sys
-import zipfile
-
-from qa_analysis.junit import MalformedArtifact
-from qa_analysis.redaction import redact_text
-from qa_analysis import taxonomy
-
-
-def parse_report(path):
-    """Parse Playwright's JSON reporter output into the normalized result shape.
-
-    Mirrors qa_analysis.junit's output ({tests, executed}) so a Playwright run
-    normalizes identically whether it emitted JSON or JUnit. Raises
-    MalformedArtifact on unreadable input.
-    """
-    try:
-        with open(path, "r", encoding="utf-8") as handle:
-            data = json.load(handle)
-    except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc:
-        raise MalformedArtifact(f"could not parse Playwright report at {path}: {exc}") from exc
-
-    executed = []
-    for suite in data.get("suites", []):
-        _walk_suite(suite, executed)
-
-    counts = {
-        "total": len(executed),
-        "passed": sum(1 for e in executed if e["status"] == "passed"),
-        "failed": sum(1 for e in executed if e["status"] == "failed"),
-        "skipped": sum(1 for e in executed if e["status"] == "skipped"),
-    }
-    return {"tests": counts, "executed": executed}
-
-
-def _walk_suite(suite, executed):
-    for spec in suite.get("specs", []):
-        for test in spec.get("tests", []):
-            results = test.get("results", [])
-            status = _status_of(results)
-            message = ""
-            if status == "failed" and results:
-                errors = results[-1].get("errors") or []
-                if errors:
-                    message = redact_text((errors[0].get("message") or "").strip())
-            entry = {
-                "title": spec.get("title", ""),
-                "file": suite.get("file", ""),
-                "status": status,
-                "durationMs": results[-1].get("duration", 0) if results else 0,
-                "retries": max(0, len(results) - 1),
-            }
-            if message:
-                entry["message"] = message
-            executed.append(entry)
-    for child in suite.get("suites", []):
-        _walk_suite(child, executed)
-
-
-def _status_of(results):
-    if not results:
-        return "skipped"
-    final = results[-1].get("status")
-    if final in ("passed", "expected"):
-        return "flaky" if len(results) > 1 else "passed"
-    if final in ("skipped",):
-        return "skipped"
-    return "failed"
-
-
-def analyze_trace(path):
-    """Extract a deterministic summary from a Playwright trace.zip.
-
-    A trace is a zip of newline-delimited JSON event files. This lists the
-    actions, surfaces the last error, and counts console and network events —
-    enough for a diagnostic skill to reason over, without decoding the full
-    binary. Raises MalformedArtifact if the file is not a valid trace zip.
-    """
-    if not zipfile.is_zipfile(path):
-        raise MalformedArtifact(f"not a valid trace zip: {path}")
-
-    actions, console, network, errors = [], 0, 0, []
-    try:
-        with zipfile.ZipFile(path) as archive:
-            names = [n for n in archive.namelist() if n.endswith(".trace") or n.endswith(".jsonl") or "trace" in n]
-            for name in names or archive.namelist():
-                with archive.open(name) as member:
-                    for raw in member.read().decode("utf-8", "replace").splitlines():
-                        raw = raw.strip()
-                        if not raw or not raw.startswith("{"):
-                            continue
-                        try:
-                            event = json.loads(raw)
-                        except json.JSONDecodeError:
-                            continue
-                        kind = event.get("type") or event.get("kind")
-                        if kind in ("action", "before"):
-                            actions.append(redact_text(str(event.get("apiName") or event.get("method") or "")))
-                        elif kind == "console":
-                            console += 1
-                        elif kind in ("resource", "network", "http"):
-                            network += 1
-                        if event.get("error") or kind == "error":
-                            msg = event.get("error", {}).get("message") if isinstance(event.get("error"), dict) else event.get("message")
-                            if msg:
-                                errors.append(redact_text(str(msg)))
-    except (OSError, KeyError) as exc:
-        raise MalformedArtifact(f"could not read trace {path}: {exc}") from exc
-
-    last_error = errors[-1] if errors else ""
-    classification, confidence, reason = taxonomy.classify(last_error) if last_error else (taxonomy.UNKNOWN, 0.2, "No error found in trace.")
-    return {
-        "actions": [a for a in actions if a],
-        "consoleEvents": console,
-        "networkEvents": network,
-        "errors": errors,
-        "classification": classification,
-        "confidence": confidence,
-        "reason": reason,
-    }
-
-
-def main(argv=None):
-    """CLI: JSON to stdout, exit 0 on success, 2 on a malformed artifact."""
-    parser = argparse.ArgumentParser(
-        prog="playwright_analysis",
-        description="Playwright adapter: normalize a JSON report or summarize a trace",
-    )
-    sub = parser.add_subparsers(dest="command", required=True)
-    p = sub.add_parser("report", help="normalize Playwright's JSON reporter output")
-    p.add_argument("path")
-    p = sub.add_parser("trace", help="summarize a Playwright trace.zip")
-    p.add_argument("path")
-    args = parser.parse_args(argv)
-
-    try:
-        result = parse_report(args.path) if args.command == "report" else analyze_trace(args.path)
-    except MalformedArtifact as exc:
-        json.dump({"error": "malformed-artifact", "detail": str(exc)}, sys.stdout, indent=2, sort_keys=True)
-        sys.stdout.write("\n")
-        return 2
-    json.dump(result, sys.stdout, indent=2, sort_keys=True)
-    sys.stdout.write("\n")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/shared/frameworks/selenium/lib/selenium_analysis.py

@@ -1,28 +0,0 @@
-"""Selenium analysis adapter.
-
-Selenium's runners (JUnit, TestNG, pytest) emit JUnit XML, so Selenium
-normalization *is* the framework-agnostic JUnit parser — the adapter only points
-at the right artifact and tags provenance. This is the proof of the pack's
-multi-framework claim: the second framework's analysis is a few lines, because
-the contract and the parsing are shared, and only the artifact location differs.
-"""
-
-from qa_analysis import junit, taxonomy
-
-FRAMEWORK = "selenium"
-
-# Where Selenium projects conventionally write results (documented in
-# selenium-artifacts.md); discovery uses these, the parser is shared.
-RESULT_GLOBS = ["**/target/surefire-reports/*.xml", "**/test-results/*.xml", "**/junit*.xml"]
-
-
-def normalize(junit_path):
-    """Normalize a Selenium run into the shared result shape via the agnostic
-    JUnit parser. Identical output shape to any other framework's normalization."""
-    return junit.parse_junit(junit_path)
-
-
-def classify_failure(message, http_status=None):
-    """Classify a Selenium failure using the shared taxonomy — no Selenium-specific
-    classifier, because failure classes are framework-agnostic."""
-    return taxonomy.classify(message, http_status=http_status)

package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py

@@ -1,24 +0,0 @@
-"""WebdriverIO analysis adapter.
-
-WebdriverIO emits JUnit XML (via @wdio/junit-reporter), so its normalization is
-the framework-agnostic JUnit parser — the adapter points at the artifact and
-tags provenance. Thin by design: the fourth framework confirms the boundary.
-"""
-
-from qa_analysis import junit, taxonomy
-
-FRAMEWORK = "webdriverio"
-
-# Where WebdriverIO projects conventionally write JUnit results.
-RESULT_GLOBS = ["**/junit*.xml", "**/results/*.xml", "**/test-results/*.xml"]
-
-
-def normalize(junit_path):
-    """Normalize a WebdriverIO run into the shared result shape via the agnostic
-    JUnit parser. Identical output shape to any other framework."""
-    return junit.parse_junit(junit_path)
-
-
-def classify_failure(message, http_status=None):
-    """Classify a WebdriverIO failure using the shared taxonomy."""
-    return taxonomy.classify(message, http_status=http_status)

package/shared/tooling/qa_tool.py

@@ -1,127 +0,0 @@
-#!/usr/bin/env python3
-"""The single entry point for a skill's bundled deterministic tooling.
-
-Every skill that carries an engine also carries a copy of this file at
-`<skill>/scripts/qa_tool.py`, beside the `lib/` directory it dispatches into.
-
-## Why this exists
-
-The invocation used to be a shell recipe:
-
-    QA_LIB="$(ls -d .agents/skills/qa-run/scripts/lib ... | head -1)"
-    PYTHONPATH="$QA_LIB" python3 -m qa_analysis.cli junit report.xml
-
-Three parts of that are POSIX-only — command substitution, `ls -d | head -1`, and
-the `VAR=value command` prefix — so on Windows PowerShell every deterministic call
-failed. Failing calls do not stop a skill; they push it onto its documented manual
-fallback. The result was that Windows users silently got the *guessing* behaviour
-this project exists to replace, while believing they had the tooling.
-
-So the shell does no work at all now. This launcher finds its own `lib/`
-directory from `__file__` and puts it on `sys.path` itself:
-
-    python3 .agents/skills/qa-run/scripts/qa_tool.py analysis junit report.xml
-
-That line is identical in bash, zsh, PowerShell, and cmd.exe. On Windows, use
-`python` if `python3` is not on PATH — that is the only remaining difference.
-
-## Usage
-
-    qa_tool.py analysis    <subcommand> [args]   -> qa_analysis.cli
-    qa_tool.py diagnostics <subcommand> [args]   -> qa_diagnostics.cli
-    qa_tool.py playwright  <subcommand> [args]   -> playwright_analysis
-    qa_tool.py --list                            what this skill bundles
-
-Exit codes pass through from the underlying tool unchanged: 0 success, 1 an
-invalid contract, 2 unreadable input or a payload that failed its contract.
-"""
-
-import os
-import pathlib
-import sys
-
-_HERE = pathlib.Path(__file__).resolve().parent
-_LIB = _HERE / "lib"
-
-# Tool name -> (module path, callable). Resolved lazily so a skill that bundles
-# only part of the toolkit still runs the parts it has.
-_TOOLS = {
-    "analysis": ("qa_analysis.cli", "main"),
-    "diagnostics": ("qa_diagnostics.cli", "main"),
-    "playwright": ("playwright_analysis", "main"),
-}
-
-_USAGE = """usage: qa_tool.py <tool> <subcommand> [args]
-
-  analysis     parse artifacts, classify errors, validate contracts, diff-guard
-  diagnostics  root cause, timeline, priority, repair plans
-  playwright   normalize a Playwright report or summarize a trace
-
-  --list       show which tools this skill bundles
-
-examples:
-  python3 qa_tool.py analysis junit test-results/results.xml
-  python3 qa_tool.py analysis diff-guard change.diff
-  python3 qa_tool.py diagnostics report --execution-result qa-artifacts/run.json
-  python3 qa_tool.py playwright report test-results/results.json
-"""
-
-
-def _available():
-    """Tools whose module is importable from this skill's bundled lib/."""
-    found = []
-    for name, (module, _) in _TOOLS.items():
-        top = module.split(".")[0]
-        if (_LIB / top).is_dir() or (_LIB / f"{top}.py").is_file():
-            found.append(name)
-    return found
-
-
-def main(argv=None):
-    argv = list(sys.argv[1:] if argv is None else argv)
-
-    if not _LIB.is_dir():
-        sys.stderr.write(
-            f"qa_tool: no bundled library at {_LIB}\n"
-            "The skill's deterministic engine is missing. Run: qa repair --project .\n"
-        )
-        return 2
-
-    # The bundled lib is authoritative: prepend so a same-named package installed
-    # elsewhere on the machine cannot shadow the version that shipped with this skill.
-    sys.path.insert(0, str(_LIB))
-
-    if not argv or argv[0] in ("-h", "--help", "help"):
-        sys.stdout.write(_USAGE)
-        return 0
-
-    if argv[0] == "--list":
-        available = _available()
-        sys.stdout.write(
-            "bundled tools: " + (", ".join(available) if available else "none") + "\n"
-        )
-        return 0 if available else 2
-
-    tool = argv[0]
-    if tool not in _TOOLS:
-        sys.stderr.write(f"qa_tool: unknown tool {tool!r}\n\n{_USAGE}")
-        return 2
-
-    module_name, entry = _TOOLS[tool]
-    try:
-        module = __import__(module_name, fromlist=[entry])
-    except ImportError as exc:
-        available = _available()
-        sys.stderr.write(
-            f"qa_tool: {tool} is not bundled in this skill ({exc}).\n"
-            f"This skill bundles: {', '.join(available) if available else 'nothing'}\n"
-        )
-        return 2
-
-    return getattr(module, entry)(argv[1:])
-
-
-if __name__ == "__main__":
-    # os.environ is untouched: the launcher never mutates the caller's environment.
-    del os
-    sys.exit(main())