qa-engineer 0.10.0 → 0.11.0

package/skills/qa-explore/contracts/explore-result.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "urn:qa-pack:contract:qa-explore:explore-result:1",
   "title": "qa-explore explore result",
-  "description": "Full-spectrum product QA against a live URL: findings with severity and proof, optional test-case coverage and DB validation. classification is the overall explore outcome.",
+  "description": "Full-spectrum product QA against a live URL: findings with severity and proof, optional test-case coverage and DB validation. classification is the overall explore outcome. Version 1.1 adds the evidence and analysis blocks the enterprise report renders — artifacts, executive verdict, scores, pages, timeline, network, performance, accessibility, security, console — all optional, so a 1.0 producer stays valid.",
   "type": "object",
   "additionalProperties": false,
   "required": [
@@ -42,6 +42,21 @@
     },
     "url": { "type": "string", "minLength": 1 },
     "reportVersion": { "type": "string" },
+    "title": {
+      "type": "string",
+      "minLength": 1,
+      "description": "The product or application name, for the report masthead. The URL is the fallback when this is absent."
+    },
+    "environment": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Which deployment was exercised — production, staging, local. A finding's severity is read differently depending on where it was found."
+    },
+    "durationMs": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Wall-clock length of the whole run."
+    },
     "browserAdapter": {
       "enum": [
         "playwright-mcp",
@@ -52,6 +67,49 @@
         "unavailable"
       ]
     },
+    "authentication": {
+      "description": "How the run got past the login wall, if it had to. Absent means no authentication was involved.",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "required": { "type": "boolean" },
+        "method": {
+          "enum": [
+            "none",
+            "form",
+            "sso",
+            "oauth",
+            "saml",
+            "azure-ad",
+            "okta",
+            "auth0",
+            "google",
+            "microsoft",
+            "magic-link",
+            "jwt",
+            "cookie",
+            "session",
+            "basic",
+            "api-key",
+            "unknown"
+          ]
+        },
+        "detectedBy": {
+          "type": "string",
+          "minLength": 1,
+          "description": "The signal that proved it — a 401, a redirect to an identity provider, a login form."
+        },
+        "sessionSource": {
+          "enum": ["manual-login", "reused-session", "not-required", "failed"]
+        },
+        "profile": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Which stored role this run used — admin, manager, read-only."
+        },
+        "note": { "type": "string", "minLength": 1 }
+      }
+    },
     "dimensionsRun": {
       "type": "array",
       "items": {
@@ -62,7 +120,9 @@
           "security",
           "ui",
           "ux",
-          "data"
+          "data",
+          "accessibility",
+          "console"
         ]
       }
     },
@@ -88,6 +148,61 @@
         }
       }
     },
+    "executive": {
+      "description": "The release decision, stated once at the top for a reader who will not scroll. Everything here must be derivable from the findings below it.",
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["verdict", "headline"],
+      "properties": {
+        "verdict": {
+          "enum": ["ship", "ship-with-risks", "do-not-ship", "insufficient-data"]
+        },
+        "headline": {
+          "type": "string",
+          "minLength": 1,
+          "description": "One sentence a CEO can act on."
+        },
+        "health": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Current state of the application in two or three sentences."
+        },
+        "risks": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "description": "The reasons behind the verdict, worst first."
+        },
+        "recommendedAction": { "type": "string", "minLength": 1 },
+        "estimatedFixHours": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["low", "high"],
+          "properties": {
+            "low": { "type": "number", "minimum": 0 },
+            "high": { "type": "number", "minimum": 0 }
+          }
+        },
+        "confidence": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "How much the run trusts its own verdict. Below 0.6 the report says so in the header."
+        }
+      }
+    },
+    "scores": {
+      "description": "Zero to one hundred, higher is better. Omit any dimension the run did not measure — a fabricated score is worse than a missing one.",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "overall": { "type": "integer", "minimum": 0, "maximum": 100 },
+        "functional": { "type": "integer", "minimum": 0, "maximum": 100 },
+        "performance": { "type": "integer", "minimum": 0, "maximum": 100 },
+        "accessibility": { "type": "integer", "minimum": 0, "maximum": 100 },
+        "security": { "type": "integer", "minimum": 0, "maximum": 100 },
+        "ux": { "type": "integer", "minimum": 0, "maximum": 100 }
+      }
+    },
     "severityCounts": {
       "type": "object",
       "additionalProperties": false,
@@ -99,6 +214,258 @@
         "low": { "type": "integer", "minimum": 0 }
       }
     },
+    "artifacts": {
+      "description": "Every file this run produced, registered once. Findings and evidence entries point at these by id, so a path exists in exactly one place and the renderer can tell a missing file from a present one instead of emitting a broken image.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["id", "kind", "path"],
+        "properties": {
+          "id": {
+            "type": "string",
+            "minLength": 1,
+            "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
+          },
+          "kind": {
+            "enum": [
+              "screenshot",
+              "video",
+              "trace",
+              "har",
+              "dom",
+              "console",
+              "network",
+              "performance",
+              "accessibility",
+              "security",
+              "api",
+              "coverage",
+              "log",
+              "diff",
+              "report",
+              "file"
+            ]
+          },
+          "path": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Location of the file, relative to the directory holding this result JSON. A path that escapes that directory is rejected by the artifact manager."
+          },
+          "label": { "type": "string", "minLength": 1 },
+          "mimeType": { "type": "string", "minLength": 1 },
+          "bytes": { "type": "integer", "minimum": 0 },
+          "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+          "capturedAt": { "type": "string", "format": "date-time" },
+          "width": { "type": "integer", "minimum": 1 },
+          "height": { "type": "integer", "minimum": 1 },
+          "thumbnail": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Relative path to a smaller stand-in used for the inline preview, when one was generated."
+          },
+          "compares": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Artifact id this one should be shown against, for before/after pairs."
+          },
+          "note": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "pages": {
+      "description": "One entry per page the run visited. A multi-page run is one report, not several.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["url", "status"],
+        "properties": {
+          "url": { "type": "string", "minLength": 1 },
+          "title": { "type": "string", "minLength": 1 },
+          "status": { "enum": ["ok", "issues", "failed", "blocked", "skipped"] },
+          "httpStatus": { "type": "integer", "minimum": 100, "maximum": 599 },
+          "findingIds": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+          "loadMs": { "type": "integer", "minimum": 0 },
+          "requests": { "type": "integer", "minimum": 0 },
+          "consoleErrors": { "type": "integer", "minimum": 0 },
+          "score": { "type": "integer", "minimum": 0, "maximum": 100 },
+          "note": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "timeline": {
+      "description": "What the run did, in order, so a reader can see where the time went and what was actually exercised.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["phase", "label"],
+        "properties": {
+          "phase": {
+            "enum": [
+              "launch",
+              "authentication",
+              "navigation",
+              "functional",
+              "api",
+              "performance",
+              "security",
+              "accessibility",
+              "ui",
+              "data",
+              "analysis",
+              "reporting"
+            ]
+          },
+          "label": { "type": "string", "minLength": 1 },
+          "startedAt": { "type": "string", "format": "date-time" },
+          "durationMs": { "type": "integer", "minimum": 0 },
+          "status": { "enum": ["ok", "warned", "failed", "skipped"] },
+          "detail": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "network": {
+      "description": "What the page asked the network for, and what came back.",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "totalRequests": { "type": "integer", "minimum": 0 },
+        "failedRequests": { "type": "integer", "minimum": 0 },
+        "slowRequests": { "type": "integer", "minimum": 0 },
+        "duplicateRequests": { "type": "integer", "minimum": 0 },
+        "totalBytes": { "type": "integer", "minimum": 0 },
+        "slowThresholdMs": { "type": "integer", "minimum": 0 },
+        "endpoints": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["method", "url"],
+            "properties": {
+              "method": { "type": "string", "minLength": 1 },
+              "url": { "type": "string", "minLength": 1 },
+              "status": { "type": "integer", "minimum": 0, "maximum": 599 },
+              "durationMs": { "type": "integer", "minimum": 0 },
+              "bytes": { "type": "integer", "minimum": 0 },
+              "count": { "type": "integer", "minimum": 1 },
+              "startedMs": {
+                "type": "integer",
+                "minimum": 0,
+                "description": "Offset from the first request, for the waterfall."
+              },
+              "issue": {
+                "enum": [
+                  "failed",
+                  "slow",
+                  "duplicate",
+                  "large-payload",
+                  "uncached",
+                  "polling",
+                  "n-plus-one",
+                  "race",
+                  "retry"
+                ]
+              },
+              "findingId": { "type": "string", "minLength": 1 }
+            }
+          }
+        }
+      }
+    },
+    "performance": {
+      "description": "Field measurements only. Every number here was observed in this run; nothing is estimated.",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "lcpMs": { "type": "number", "minimum": 0 },
+        "fcpMs": { "type": "number", "minimum": 0 },
+        "ttfbMs": { "type": "number", "minimum": 0 },
+        "inpMs": { "type": "number", "minimum": 0 },
+        "cls": { "type": "number", "minimum": 0 },
+        "longTasks": { "type": "integer", "minimum": 0 },
+        "longTaskMs": { "type": "number", "minimum": 0 },
+        "domNodes": { "type": "integer", "minimum": 0 },
+        "jsBytes": { "type": "integer", "minimum": 0 },
+        "cssBytes": { "type": "integer", "minimum": 0 },
+        "imageBytes": { "type": "integer", "minimum": 0 },
+        "totalBytes": { "type": "integer", "minimum": 0 },
+        "jsHeapBytes": { "type": "integer", "minimum": 0 },
+        "note": { "type": "string", "minLength": 1 }
+      }
+    },
+    "accessibility": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "checked": { "type": "boolean" },
+        "summary": { "type": "string", "minLength": 1 },
+        "violations": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["rule", "impact", "count"],
+            "properties": {
+              "rule": { "type": "string", "minLength": 1 },
+              "impact": { "enum": ["critical", "serious", "moderate", "minor"] },
+              "count": { "type": "integer", "minimum": 1 },
+              "description": { "type": "string", "minLength": 1 },
+              "selector": { "type": "string", "minLength": 1 },
+              "findingId": { "type": "string", "minLength": 1 }
+            }
+          }
+        }
+      }
+    },
+    "security": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "checked": { "type": "boolean" },
+        "summary": { "type": "string", "minLength": 1 },
+        "checks": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["check", "status"],
+            "properties": {
+              "check": { "type": "string", "minLength": 1 },
+              "status": { "enum": ["pass", "fail", "warn", "not-checked"] },
+              "detail": { "type": "string", "minLength": 1 },
+              "findingId": { "type": "string", "minLength": 1 }
+            }
+          }
+        }
+      }
+    },
+    "console": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "errors": { "type": "integer", "minimum": 0 },
+        "warnings": { "type": "integer", "minimum": 0 },
+        "unhandledRejections": { "type": "integer", "minimum": 0 },
+        "deprecations": { "type": "integer", "minimum": 0 },
+        "entries": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["level", "message"],
+            "properties": {
+              "level": { "enum": ["error", "warning", "info", "debug"] },
+              "message": { "type": "string", "minLength": 1 },
+              "count": { "type": "integer", "minimum": 1 },
+              "source": { "type": "string", "minLength": 1 },
+              "findingId": { "type": "string", "minLength": 1 }
+            }
+          }
+        }
+      }
+    },
     "evidence": {
       "type": "array",
       "minItems": 1,
@@ -117,11 +484,22 @@
               "db",
               "file",
               "report",
-              "command"
+              "command",
+              "video",
+              "trace",
+              "performance",
+              "accessibility",
+              "security",
+              "log"
             ]
           },
           "description": { "type": "string", "minLength": 1 },
           "source": { "type": "string", "minLength": 1 },
+          "artifactId": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Preferred over source when the file is registered in artifacts[] — the renderer then knows its size, type, and whether it is actually there."
+          },
           "excerpt": { "type": "string" }
         }
       }
@@ -154,7 +532,9 @@
               "security",
               "ui",
               "ux",
-              "data"
+              "data",
+              "accessibility",
+              "console"
             ]
           },
           "title": { "type": "string", "minLength": 1 },
@@ -167,16 +547,86 @@
               "confirmed",
               "validated-user-report",
               "could-not-reproduce",
-              "partial"
+              "partial",
+              "fixed-in-run"
             ]
           },
+          "businessImpact": {
+            "type": "string",
+            "minLength": 1,
+            "description": "What it costs the business if this ships — in money, trust, compliance, or support load. Not a restatement of the defect."
+          },
+          "rootCause": {
+            "description": "The investigation, not the observation. Present only when the run actually established a cause.",
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["summary"],
+            "properties": {
+              "summary": { "type": "string", "minLength": 1 },
+              "chain": {
+                "type": "array",
+                "items": { "type": "string", "minLength": 1 },
+                "description": "The causal steps, in order: what the click did, what the request returned, why, and what the UI failed to do about it."
+              },
+              "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
+              "layer": {
+                "enum": ["frontend", "backend", "network", "data", "infrastructure", "third-party", "unknown"]
+              }
+            }
+          },
+          "regressionRisk": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["level"],
+            "properties": {
+              "level": { "enum": ["high", "medium", "low"] },
+              "note": { "type": "string", "minLength": 1 },
+              "retest": {
+                "type": "array",
+                "items": { "type": "string", "minLength": 1 },
+                "description": "What QA should re-run once this is fixed."
+              }
+            }
+          },
+          "developerNotes": { "type": "string", "minLength": 1 },
+          "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
+          "page": { "type": "string", "minLength": 1 },
+          "tags": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 }
+          },
+          "affectedApis": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 }
+          },
+          "metrics": {
+            "description": "Numbers that make the finding concrete, rendered as a small table on the card.",
+            "type": "array",
+            "items": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["label", "value"],
+              "properties": {
+                "label": { "type": "string", "minLength": 1 },
+                "value": { "type": "string", "minLength": 1 },
+                "budget": { "type": "string", "minLength": 1 },
+                "breached": { "type": "boolean" }
+              }
+            }
+          },
+          "steps": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 },
+            "description": "Reproduction as discrete steps. Preferred over repro prose when the run recorded them separately; repro stays required so a 1.0 reader keeps working."
+          },
           "evidence": {
             "type": "array",
             "minItems": 1,
             "items": {
               "type": "object",
               "additionalProperties": false,
-              "required": ["type", "source"],
+              "description": "Every entry must locate its file, by artifactId when the artifact is registered or by source when it is not. One or the other is required — an evidence entry that names neither is a claim with nothing behind it.",
+              "required": ["type"],
               "properties": {
                 "type": {
                   "enum": [
@@ -186,12 +636,27 @@
                     "dom",
                     "har",
                     "db",
-                    "command"
+                    "command",
+                    "video",
+                    "trace",
+                    "performance",
+                    "accessibility",
+                    "security",
+                    "log"
                   ]
                 },
                 "source": { "type": "string", "minLength": 1 },
+                "artifactId": { "type": "string", "minLength": 1 },
+                "caption": { "type": "string", "minLength": 1 },
                 "excerpt": { "type": "string" }
-              }
+              },
+              "allOf": [
+                {
+                  "if": { "required": ["artifactId"] },
+                  "then": {},
+                  "else": { "required": ["source"] }
+                }
+              ]
             }
           }
         }
@@ -219,7 +684,13 @@
               "status": {
                 "enum": ["pass", "fail", "blocked", "skipped"]
               },
-              "findingId": { "type": "string" }
+              "findingId": { "type": "string" },
+              "source": {
+                "enum": ["user-supplied", "derived", "exploratory"],
+                "description": "Where the case came from. A user-supplied case carries different weight than one the run invented."
+              },
+              "page": { "type": "string", "minLength": 1 },
+              "note": { "type": "string", "minLength": 1 }
             }
           }
         }
@@ -264,10 +735,45 @@
         "required": ["action", "priority"],
         "properties": {
           "action": { "type": "string", "minLength": 1 },
-          "priority": { "enum": ["high", "medium", "low"] }
+          "priority": { "enum": ["high", "medium", "low"] },
+          "rationale": { "type": "string", "minLength": 1 },
+          "owner": {
+            "enum": ["frontend", "backend", "platform", "design", "qa", "product", "security"]
+          },
+          "effort": { "enum": ["small", "medium", "large"] },
+          "findingIds": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 }
+          }
         }
       }
     },
     "metadata": { "type": "object" }
-  }
+  },
+  "allOf": [
+    {
+      "description": "A `ship` verdict cannot stand over a critical or high finding. The report's own severity legend says critical blocks release and high must be fixed before it, so a run that counts one and then recommends shipping is contradicting itself — and the contradiction is invisible to a reader who only reads the banner. `ship-with-risks` is the honest verdict for knowingly releasing over a high; there is no honest verdict for releasing over a critical.",
+      "if": {
+        "type": "object",
+        "required": ["executive"],
+        "properties": {
+          "executive": {
+            "type": "object",
+            "required": ["verdict"],
+            "properties": { "verdict": { "const": "ship" } }
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "severityCounts": {
+            "properties": {
+              "critical": { "maximum": 0 },
+              "high": { "maximum": 0 }
+            }
+          }
+        }
+      }
+    }
+  ]
 }

package/skills/qa-explore/references/api-replay.md

@@ -3,11 +3,50 @@
 
 Live API audit during a browser explore session: use the app's own network traffic as the source of truth, then replay exact URLs. Consumed by the explore skill. Complements the REST domain (which covers API *test* quality).
 
+## Count with the parser, judge with your eyes
+
+**Do not count requests, time them, or spot duplicates by reading a network panel.**
+Capture a HAR and let the engine do it:
+
+```bash
+node <SKILL_DIR>/scripts/qa-tool.mjs analysis network <capture.har> \
+  [--slow-ms 1000] [--large-bytes 524288]
+```
+
+What comes back is the report contract's `network` block, ready to drop into the result
+and validate: `totalRequests`, `failedRequests`, `slowRequests`, `duplicateRequests`,
+`totalBytes`, and one entry per endpoint carrying its method, URL, status, duration,
+size, call count, start offset, and an `issue` flag.
+
+The flags are assigned by stated rules, worst first:
+
+| Flag | Rule |
+| --- | --- |
+| `failed` | status 0, 4xx, or 5xx |
+| `slow` | duration at or over the threshold (1000 ms by default) |
+| `polling` | three or more identical calls at an even cadence — a timer, not a burst |
+| `duplicate` | the same method and URL called more than once |
+| `n-plus-one` | four or more distinct ids on one path shape inside two seconds |
+| `large-payload` | response at or over the size threshold (512 KB by default) |
+| `uncached` | a 200 for a static asset with neither `Cache-Control` nor `ETag` |
+
+URLs and headers are redacted on the way through, so a token in a query string never
+reaches the report.
+
+**What is still yours.** The parser says *what happened*; only you can say whether it
+matters. A duplicated analytics beacon and a duplicated payment request are identical in
+a HAR and wildly different findings. Read the flags, then write findings for the ones
+that affect this product — and set each finding's `affectedApis` to the endpoints the
+block already named, so the report links them.
+
+If no HAR is available, fall back to `performance.getEntriesByType('resource')` and say
+in the report that the counts were observed rather than parsed.
+
 ## Best practices
 
 - **Best practice:** prefer `performance.getEntriesByType('resource')` (or an equivalent HAR) over agent network monitors that miss XHR/fetch.
 - **Best practice:** replay the application's exact URLs and methods via in-page fetch (using the page's existing auth), not guessed GET probes against POST endpoints.
-- **Recommendation:** classify calls as irrelevant, duplicate, failing, missing-params, slow (cold vs warm), or over-fetching before writing findings.
+- **Best practice:** run `analysis network` on a HAR for the counts and flags; classify each flagged call as relevant or not, rather than counting by hand.
 - **Recommendation:** capture UI values with an ISO timestamp when comparing to API or DB responses; match within a stated freshness window.
 - **Best practice:** redact Authorization headers, cookies, tokens, and PII from every excerpt written to artifacts.