qa-engineer 0.9.0

package/shared/analysis/lib/qa_analysis/evidence.py

@@ -0,0 +1,128 @@
+"""The evidence and finding model shared by every analyzer.
+
+Every finding an analyzer produces carries the same structure, so downstream
+skills (qa-debug, qa-report, qa-fix) consume one shape regardless of which
+analyzer or framework produced it. Text fields are redacted at construction.
+"""
+
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timezone
+
+from .redaction import redact_text
+
+EVIDENCE_TYPES = {
+    "trace", "har", "junit", "report", "console", "network", "stdout",
+    "stderr", "screenshot", "video", "log", "file", "diff",
+}
+
+
+def utc_now():
+    """ISO 8601 UTC timestamp. Isolated so tests can monkeypatch it."""
+    return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass
+class Evidence:
+    """One observation supporting a finding. Excerpts are redacted."""
+    type: str
+    description: str
+    source: str
+    excerpt: str = ""
+
+    def __post_init__(self):
+        if self.type not in EVIDENCE_TYPES:
+            raise ValueError(f"unknown evidence type: {self.type}")
+        self.excerpt = redact_text(self.excerpt)
+
+    def to_dict(self):
+        data = {"type": self.type, "description": self.description, "source": self.source}
+        if self.excerpt:
+            data["excerpt"] = self.excerpt
+        return data
+
+
+@dataclass
+class Finding:
+    """A single diagnostic conclusion, traceable to a specific artifact.
+
+    Carries everything the evidence model requires: the artifact and location
+    it came from, when, why, the supporting evidence, a calibrated confidence,
+    the affected tests, related artifacts, and recommended actions.
+    """
+    classification: str
+    reason: str
+    artifact: str
+    location: str
+    confidence: float = None
+    timestamp: str = field(default_factory=utc_now)
+    evidence: list = field(default_factory=list)
+    affected_tests: list = field(default_factory=list)
+    related_artifacts: list = field(default_factory=list)
+    recommendations: list = field(default_factory=list)
+
+    def to_dict(self):
+        data = {
+            "classification": self.classification,
+            "reason": self.reason,
+            "artifact": self.artifact,
+            "location": self.location,
+            "timestamp": self.timestamp,
+            "evidence": [e.to_dict() if isinstance(e, Evidence) else e for e in self.evidence],
+            "affectedTests": self.affected_tests,
+            "relatedArtifacts": self.related_artifacts,
+            "recommendations": self.recommendations,
+        }
+        if self.confidence is not None:
+            data["confidence"] = self.confidence
+        return data
+
+
+@dataclass
+class AnalyzerOutput:
+    """The envelope an analyzer emits: findings plus the artifacts it examined.
+
+    A downstream skill wraps this in its own output contract; on its own it is
+    the deterministic, machine-readable result of one analysis.
+    """
+    analyzer: str
+    findings: list = field(default_factory=list)
+    artifacts: list = field(default_factory=list)
+    warnings: list = field(default_factory=list)
+    generated_at: str = field(default_factory=utc_now)
+
+    def to_dict(self):
+        return {
+            "analyzer": self.analyzer,
+            "generatedAt": self.generated_at,
+            "findings": [f.to_dict() if isinstance(f, Finding) else f for f in self.findings],
+            "artifacts": [a.to_dict() if hasattr(a, "to_dict") else a for a in self.artifacts],
+            "warnings": self.warnings,
+        }
+
+
+@dataclass
+class Artifact:
+    """A discovered artifact, in the common model shared with the execution engine."""
+    type: str
+    location: str
+    framework: str = "unknown"
+    ownership: str = "qa-analysis"
+    timestamp: str = field(default_factory=utc_now)
+    media_type: str = ""
+    test_ref: str = ""
+    present: bool = True
+
+    def to_dict(self):
+        data = {
+            "type": self.type,
+            "location": self.location,
+            "framework": self.framework,
+            "timestamp": self.timestamp,
+            "ownership": self.ownership,
+            "present": self.present,
+        }
+        if self.media_type:
+            data["mediaType"] = self.media_type
+        if self.test_ref:
+            data["testRef"] = self.test_ref
+        return data

package/shared/analysis/lib/qa_analysis/har.py

@@ -0,0 +1,58 @@
+"""HAR (HTTP Archive) parser.
+
+Framework-agnostic: a HAR is a standard JSON format, whoever produced it.
+Extracts request outcomes, flags failures and slow calls, and redacts headers
+and credentialed URLs before anything is exposed. A malformed HAR raises.
+"""
+
+import json
+
+from .redaction import redact_text, redact_headers
+from .junit import MalformedArtifact
+
+
+def parse_har(path, slow_ms=1000):
+    """Parse a HAR file into a redacted network summary.
+
+    Returns {entries, failures, slow, redacted: True}, where each entry is
+    {method, url, status, durationMs, headersRedacted}. Failures are entries
+    with status >= 400 or status 0 (no response). Raises MalformedArtifact on
+    unreadable input.
+    """
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            data = json.load(handle)
+    except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc:
+        raise MalformedArtifact(f"could not parse HAR at {path}: {exc}") from exc
+
+    try:
+        raw_entries = data["log"]["entries"]
+    except (KeyError, TypeError) as exc:
+        raise MalformedArtifact(f"not a HAR document at {path}") from exc
+
+    entries = []
+    for item in raw_entries:
+        request = item.get("request", {})
+        response = item.get("response", {})
+        status = _int(response.get("status"))
+        entry = {
+            "method": request.get("method", ""),
+            # Redaction strips any credentials embedded in the URL.
+            "url": redact_text(request.get("url", "")),
+            "status": status,
+            "durationMs": int(round(float(item.get("time", 0) or 0))),
+            "requestHeaders": redact_headers(request.get("headers", [])),
+            "responseHeaders": redact_headers(response.get("headers", [])),
+        }
+        entries.append(entry)
+
+    failures = [e for e in entries if e["status"] == 0 or e["status"] >= 400]
+    slow = [e for e in entries if e["durationMs"] >= slow_ms]
+    return {"entries": entries, "failures": failures, "slow": slow, "redacted": True}
+
+
+def _int(value, default=0):
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default

package/shared/analysis/lib/qa_analysis/junit.py

@@ -0,0 +1,80 @@
+"""JUnit XML parser.
+
+Framework-agnostic: Playwright, Selenium, Cypress, WebdriverIO, and most unit
+runners emit JUnit XML, so this one parser normalizes them all into the pack's
+per-test result shape. This is the concrete proof that different frameworks
+share one contract — only where the file lives differs, not how it is read.
+
+Parses deterministically; a malformed document raises rather than guessing.
+"""
+
+import xml.etree.ElementTree as ET
+
+from .redaction import redact_text
+
+
+class MalformedArtifact(ValueError):
+    """Raised when an artifact cannot be parsed. Never swallowed into a guess."""
+
+
+def _int(value, default=0):
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def parse_junit(path):
+    """Parse a JUnit XML file into a normalized result.
+
+    Returns a dict: {tests: {total, passed, failed, skipped}, executed: [...]}.
+    Each executed entry: {title, file, status, durationMs, message?}.
+    Raises MalformedArtifact if the XML is unreadable or not JUnit-shaped.
+    """
+    try:
+        tree = ET.parse(path)
+    except (ET.ParseError, OSError) as exc:
+        raise MalformedArtifact(f"could not parse JUnit XML at {path}: {exc}") from exc
+
+    root = tree.getroot()
+    # Accept either a <testsuites> root or a single <testsuite> root.
+    if root.tag == "testsuites":
+        suites = root.findall("testsuite")
+    elif root.tag == "testsuite":
+        suites = [root]
+    else:
+        raise MalformedArtifact(f"not a JUnit document (root <{root.tag}>) at {path}")
+
+    executed = []
+    for suite in suites:
+        for case in suite.findall("testcase"):
+            failure = case.find("failure")
+            error = case.find("error")
+            skipped = case.find("skipped")
+            if failure is not None or error is not None:
+                status = "failed"
+                node = failure if failure is not None else error
+                message = redact_text((node.get("message") or node.text or "").strip())
+            elif skipped is not None:
+                status = "skipped"
+                message = redact_text((skipped.get("message") or "").strip())
+            else:
+                status = "passed"
+                message = ""
+            entry = {
+                "title": case.get("name", ""),
+                "file": case.get("classname", ""),
+                "status": status,
+                "durationMs": int(round(float(case.get("time", "0") or "0") * 1000)),
+            }
+            if message:
+                entry["message"] = message
+            executed.append(entry)
+
+    counts = {
+        "total": len(executed),
+        "passed": sum(1 for e in executed if e["status"] == "passed"),
+        "failed": sum(1 for e in executed if e["status"] == "failed"),
+        "skipped": sum(1 for e in executed if e["status"] == "skipped"),
+    }
+    return {"tests": counts, "executed": executed}

package/shared/analysis/lib/qa_analysis/redaction.py

@@ -0,0 +1,99 @@
+"""Credential, secret, token, and PII redaction.
+
+Every analyzer runs artifact text through redaction before it appears in a
+finding, a report, or stdout. Redaction happens as evidence is captured, not
+after — a secret must never reach a model's context or a log. Standard library
+only; deterministic.
+"""
+
+import re
+
+# Ordered (name, pattern, replacement). Order matters: match structured,
+# high-signal secrets (JWTs, provider keys) before generic fallbacks.
+_RULES = [
+    ("jwt", re.compile(r"eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}")),
+    ("aws-access-key", re.compile(r"AKIA[0-9A-Z]{16}")),
+    ("github-token", re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}")),
+    ("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")),
+    ("openai-key", re.compile(r"sk-[A-Za-z0-9]{20,}")),
+    ("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/-]+=*")),
+    # Sensitive header lines: keep the header name, mask the value.
+    ("auth-header", re.compile(r"(?im)^(\s*(?:authorization|proxy-authorization)\s*[:=]\s*).+$")),
+    ("cookie-header", re.compile(r"(?im)^(\s*(?:set-cookie|cookie)\s*[:=]\s*).+$")),
+    # Secret-like assignments: key=value / "key": "value".
+    ("assigned-secret", re.compile(
+        r'(?i)(\b(?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|client[_-]?secret)\b\s*[:=]\s*["\']?)'
+        r'([^\s"\'&]{4,})')),
+    # Credentials in URLs and query strings.
+    ("url-credential", re.compile(r"(?i)(://[^:/@\s]+:)([^@/\s]+)(@)")),
+    ("query-secret", re.compile(
+        r"(?i)([?&](?:password|passwd|pwd|token|secret|api[_-]?key|access[_-]?key)=)([^&\s#]+)")),
+    # PII: email addresses.
+    ("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")),
+]
+
+# Rules whose replacement preserves a leading captured group (the label/prefix).
+_PREFIX_PRESERVING = {
+    "auth-header": r"\1[REDACTED:auth-header]",
+    "cookie-header": r"\1[REDACTED:cookie-header]",
+    "assigned-secret": r"\1[REDACTED:secret]",
+    "url-credential": r"\1[REDACTED:credential]\3",
+    "query-secret": r"\1[REDACTED:secret]",
+}
+
+
+def redact_text(text):
+    """Return text with every recognized secret or PII value masked."""
+    if not text:
+        return text
+    result = text
+    for name, pattern in _RULES:
+        if name in _PREFIX_PRESERVING:
+            result = pattern.sub(_PREFIX_PRESERVING[name], result)
+        else:
+            result = pattern.sub(f"[REDACTED:{name}]", result)
+    return result
+
+
+def detect_secrets(text):
+    """Return a list of {type, start, end} for secrets found — never the values.
+
+    Used to decide whether an artifact is safe to expose, without surfacing the
+    secret itself.
+    """
+    findings = []
+    if not text:
+        return findings
+    for name, pattern in _RULES:
+        for match in pattern.finditer(text):
+            findings.append({"type": name, "start": match.start(), "end": match.end()})
+    findings.sort(key=lambda f: f["start"])
+    return findings
+
+
+_SENSITIVE_HEADERS = {
+    "authorization", "proxy-authorization", "cookie", "set-cookie",
+    "x-api-key", "x-auth-token", "api-key", "x-csrf-token",
+}
+
+
+def redact_headers(headers):
+    """Mask the values of sensitive HTTP headers.
+
+    Accepts a dict, or a list of {"name","value"} entries (HAR shape), and
+    returns the same shape with sensitive values replaced.
+    """
+    def mask(name, value):
+        return "[REDACTED:header]" if name.strip().lower() in _SENSITIVE_HEADERS else redact_text(value)
+
+    if isinstance(headers, dict):
+        return {k: mask(k, str(v)) for k, v in headers.items()}
+    if isinstance(headers, list):
+        out = []
+        for entry in headers:
+            if isinstance(entry, dict) and "name" in entry:
+                out.append({**entry, "value": mask(str(entry["name"]), str(entry.get("value", "")))})
+            else:
+                out.append(entry)
+        return out
+    return headers

package/shared/analysis/lib/qa_analysis/taxonomy.py

@@ -0,0 +1,98 @@
+"""The canonical QA failure taxonomy and its deterministic classifier.
+
+Classification maps observed signals to one of a closed set of failure classes.
+It is rule-based and conservative: when no rule matches with sufficient signal,
+the result is UNKNOWN. Unknown is preferable to incorrect.
+"""
+
+import re
+
+# The closed set of failure classifications. Every value implies a different
+# owner and next action; if two would lead to the same action, they are merged.
+ASSERTION = "assertion-failure"
+LOCATOR = "locator-failure"
+TIMEOUT = "timeout"
+NETWORK = "network"
+AUTH = "authentication"
+AUTHORIZATION = "authorization"
+ENVIRONMENT = "environment"
+CONFIGURATION = "configuration"
+INFRASTRUCTURE = "infrastructure"
+TEST_DATA = "test-data"
+APPLICATION_BUG = "application-bug"
+FRAMEWORK = "framework-failure"
+FLAKY = "flaky"
+UNKNOWN = "unknown"
+
+CLASSES = {
+    ASSERTION, LOCATOR, TIMEOUT, NETWORK, AUTH, AUTHORIZATION, ENVIRONMENT,
+    CONFIGURATION, INFRASTRUCTURE, TEST_DATA, APPLICATION_BUG, FRAMEWORK,
+    FLAKY, UNKNOWN,
+}
+
+# Ordered rules: (classification, message-pattern, confidence, reason).
+#
+# Order encodes priority, and the first two rules exist because modern runners
+# print a timeout budget in *every* assertion failure. Playwright's message for a
+# plain text mismatch ends with "Timeout: 5000ms" even though nothing timed out,
+# so a naive timeout rule captures assertion and locator failures alike and sends
+# the reader to raise a timeout — the one action guaranteed not to help. The
+# patterns below are derived from real captured runner output; see
+# tests/test_analysis.py::RealRunnerMessageTests, which pins the exact strings.
+#
+# The discriminators, from that output:
+#   element missing  -> "Error: element(s) not found"   (no Received: value)
+#   value mismatch   -> "Expected: X" + "Received: Y"   (element was resolved)
+#   real timeout     -> a timeout with neither of the above
+_RULES = [
+    (LOCATOR, re.compile(r"(?i)(no such element|element(?:\(s\))?\s+not\s+(?:found|visible|attached)|locator.*(?:resolved to 0|not found)|waiting for (?:locator|selector)|unable to locate element)"), 0.8,
+     "Error indicates the target element could not be found or resolved."),
+    # A concrete expected-vs-received comparison is an assertion result, not a
+    # time budget: the runner resolved the target and compared values.
+    (ASSERTION, re.compile(r"(?is)expected:.*received:"), 0.8,
+     "Error shows a concrete expected-versus-received comparison, so the assertion did not hold."),
+    (TIMEOUT, re.compile(r"(?i)(timeout|timed out|exceeded .*ms|deadline exceeded)"), 0.75,
+     "Error indicates an operation exceeded its time budget."),
+    (AUTH, re.compile(r"(?i)(401 unauthorized|authentication failed|invalid (?:credentials|token)|login failed|not authenticated)"), 0.85,
+     "Error indicates an authentication failure — the identity could not be established."),
+    (AUTHORIZATION, re.compile(r"(?i)(403 forbidden|permission denied|access denied|not authorized|forbidden\b|insufficient (?:permission|privilege))"), 0.85,
+     "Error indicates an authorization failure — the identity lacked permission."),
+    (NETWORK, re.compile(r"(?i)(ECONNREFUSED| ENOTFOUND|net::ERR|connection (?:refused|reset)|5\d\d (?:internal server error|bad gateway|service unavailable)|fetch failed)"), 0.8,
+     "Error indicates a network or upstream service failure."),
+    (ASSERTION, re.compile(r"(?i)(assertion|expect(?:ed)?\b|toBe|toEqual|toHaveText|toBeVisible|AssertionError|expected .* (?:to|but))"), 0.7,
+     "Error indicates an assertion did not hold."),
+    (TEST_DATA, re.compile(r"(?i)(duplicate key|constraint violation|no rows|seed data|fixture .*(?:missing|not found)|invalid test data)"), 0.7,
+     "Error indicates a test-data problem."),
+    (CONFIGURATION, re.compile(r"(?i)(cannot find module|config(?:uration)? (?:error|not found)|missing (?:config|environment variable)|unknown option)"), 0.7,
+     "Error indicates a configuration problem."),
+    (ENVIRONMENT, re.compile(r"(?i)(base ?url|env(?:ironment)? .*(?:not set|missing)|ECONNREFUSED .*localhost|dev server)"), 0.6,
+     "Error indicates an environment problem such as a missing base URL or unreachable local service."),
+    (INFRASTRUCTURE, re.compile(r"(?i)(out of memory|OOM|disk (?:full|space)|browser .*(?:crash|closed unexpectedly)|worker .*(?:died|crashed))"), 0.7,
+     "Error indicates an infrastructure problem such as a crash or resource exhaustion."),
+    (FRAMEWORK, re.compile(r"(?i)(internal error|unexpected error in (?:playwright|selenium|cypress|webdriver)|driver .*mismatch)"), 0.6,
+     "Error indicates a fault in the test framework or driver itself."),
+]
+
+
+def classify(message, http_status=None):
+    """Classify a failure from its error message and optional HTTP status.
+
+    Returns (classification, confidence, reason). Never guesses: an
+    unrecognized signal yields UNKNOWN at low confidence with an honest reason.
+    """
+    text = message or ""
+
+    # A concrete HTTP status is stronger evidence than message text.
+    if http_status is not None:
+        if http_status == 401:
+            return (AUTH, 0.9, "HTTP 401 indicates an authentication failure.")
+        if http_status == 403:
+            return (AUTHORIZATION, 0.9, "HTTP 403 indicates an authorization failure.")
+        if 500 <= http_status <= 599:
+            return (NETWORK, 0.85, f"HTTP {http_status} indicates an upstream server failure.")
+
+    for classification, pattern, confidence, reason in _RULES:
+        if pattern.search(text):
+            return (classification, confidence, reason)
+
+    return (UNKNOWN, 0.2, "No classification rule matched the available signals; manual review needed.")

package/shared/analysis/schemas/context.schema.json

@@ -0,0 +1,82 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:qa-pack:schema:context:1",
+  "title": "QA project context frontmatter",
+  "description": "The machine-readable frontmatter of .qa/context.md, produced by qa-init and read by every skill. Fields are always present; an undetermined value is null, never omitted. See docs/architecture/context-contract.md for semantics.",
+  "type": "object",
+  "additionalProperties": true,
+  "required": [
+    "schemaVersion", "generatedBy", "generatedAt", "repository", "language",
+    "packageManager", "testFramework", "browserAutomation", "apiStyles", "ci",
+    "conventions", "confidence"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "generatedBy": { "type": "string", "minLength": 1 },
+    "generatedAt": { "type": "string", "format": "date-time" },
+    "repository": {
+      "type": "object",
+      "required": ["root", "monorepo", "packages"],
+      "properties": {
+        "root": { "type": "string" },
+        "monorepo": { "type": "boolean" },
+        "packages": { "type": "array" }
+      }
+    },
+    "language": {
+      "type": "object",
+      "required": ["primary"],
+      "properties": {
+        "primary": { "type": ["string", "null"] },
+        "others": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "runtime": { "type": "object" },
+    "packageManager": {
+      "type": ["string", "null"],
+      "enum": ["npm", "pnpm", "yarn", "maven", "gradle", "pip", "poetry", "nuget", null]
+    },
+    "buildTool": { "type": ["string", "null"] },
+    "testFramework": {
+      "type": "object",
+      "required": ["unit", "e2e", "bdd"],
+      "properties": {
+        "unit": { "type": ["string", "null"] },
+        "e2e": { "type": ["string", "null"], "enum": ["playwright", "selenium", "cypress", "webdriverio", null] },
+        "bdd": { "type": ["string", "null"] }
+      }
+    },
+    "browserAutomation": {
+      "type": "object",
+      "required": ["tool", "mcp"],
+      "properties": {
+        "tool": { "type": ["string", "null"] },
+        "mcp": { "type": "boolean" }
+      }
+    },
+    "apiStyles": {
+      "type": "array",
+      "items": { "type": "string", "enum": ["rest", "graphql", "websocket"] }
+    },
+    "ci": {
+      "type": "object",
+      "required": ["provider", "workflows"],
+      "properties": {
+        "provider": { "type": ["string", "null"], "enum": ["github-actions", "jenkins", "gitlab-ci", "azure-devops", null] },
+        "workflows": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "apiStylesDetail": { "type": "object" },
+    "conventions": {
+      "type": "object",
+      "required": ["testDir", "specGlob", "configFiles"],
+      "properties": {
+        "testDir": { "type": ["string", "null"] },
+        "specGlob": { "type": ["string", "null"] },
+        "configFiles": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "existingAutomation": { "type": "boolean" },
+    "confidence": { "type": "string", "enum": ["high", "medium", "low"] }
+  }
+}

package/shared/diagnostics/lib/qa_diagnostics/__init__.py

@@ -0,0 +1,13 @@
+"""qa_diagnostics — the pack's shared diagnostic engine.
+
+One place where failure reasoning lives: root-cause analysis, timeline
+reconstruction, finding prioritization, and repair planning. The three
+diagnostic skills (qa-debug, qa-fix, qa-report) consume this engine and differ
+only in presentation — the reasoning is not duplicated across them.
+
+Standard library only. Builds on qa_analysis (taxonomy, evidence, diff guard);
+adds no framework-specific logic. Deterministic: the same inputs yield the same
+diagnosis every time.
+"""
+
+__version__ = "0.1.0"