qa-engineer 0.10.0 → 0.11.0

package/packages/engine/lib/report/core/model.mjs

@@ -0,0 +1,270 @@
+// Contract artifact → view model.
+//
+// Every renderer reads this shape, never the raw artifact. One place therefore owns
+// the questions that would otherwise be answered slightly differently in eight
+// sections: which verdict applies when `executive` is absent, what counts as a
+// "covered" dimension, what the subject line of the report is, and — the one that
+// matters most — what may be *derived* versus what must be *measured*.
+//
+// ## Derivation discipline
+//
+// A report is trusted because its numbers came from somewhere. So:
+//
+//   derived  — the overall score (a fixed function of severity counts), dimension
+//              filter counts, page/finding association, totals, the fallback verdict
+//   measured — every performance number, every score the producer supplied,
+//              artifact existence, request timings
+//
+// Nothing here invents a measurement. If a run never checked accessibility there is
+// no accessibility score, and the report says the area was not examined rather than
+// printing a plausible number. A fabricated 87 is worse than a blank, because the
+// blank prompts a question and the 87 ends one.
+
+import path from 'node:path';
+
+import { createRegistry } from '../../artifacts/manager.mjs';
+import { VERDICT, SEVERITY_ORDER } from '../theme/tokens.mjs';
+import { DIMENSION_LABEL } from '../components/findings.mjs';
+import { normalize } from './normalize.mjs';
+import { versionStamp } from '../version.mjs';
+
+/**
+ * What the report is *about*, short enough to be a heading.
+ *
+ * The contract has no title field on a 1.0 result and the summary is a paragraph —
+ * using it as an `<h1>` produced a five-line heading. The host is the honest short
+ * answer, the way Lighthouse titles a report by its URL.
+ */
+export function subjectOf(result) {
+  if (result.title) return String(result.title);
+  const url = String(result.url ?? '').trim();
+  if (url) {
+    const withoutScheme = url.includes('://') ? url.slice(url.indexOf('://') + 3) : url;
+    return withoutScheme.replace(/\/+$/, '') || url;
+  }
+  return String(result.summary ?? 'QA report').split('.')[0].slice(0, 80);
+}
+
+/**
+ * The release decision.
+ *
+ * `executive.verdict` is authoritative when the run supplied one. Otherwise the
+ * classification maps onto the same surface, so a 1.0 result still renders a verdict
+ * banner instead of an empty strip. The severity counts are the last resort and the
+ * safest one: any critical means do-not-ship, whatever the prose says.
+ */
+export function verdictOf(result) {
+  const executive = result.executive ?? {};
+  let key = executive.verdict;
+
+  if (!key) {
+    const counts = result.severityCounts ?? {};
+    if (result.classification === 'blocked') key = 'blocked';
+    else if (result.classification === 'insufficient-data') key = 'insufficient-data';
+    else if ((counts.critical ?? 0) > 0) key = 'do-not-ship';
+    else if ((counts.high ?? 0) > 0) key = 'ship-with-risks';
+    else key = result.classification === 'pass' ? 'pass' : 'issues-found';
+  }
+
+  const spec = VERDICT[key] ?? VERDICT['insufficient-data'];
+  return {
+    key,
+    label: spec.label,
+    tone: spec.tone,
+    blurb: executive.headline ?? spec.blurb,
+    headline: executive.headline ?? null,
+    health: executive.health ?? null,
+    risks: executive.risks ?? [],
+    recommendedAction: executive.recommendedAction ?? null,
+    estimatedFixHours: executive.estimatedFixHours ?? null,
+    confidence: Number.isFinite(executive.confidence) ? executive.confidence : null,
+    // True when nothing in the artifact stated a verdict and this one was inferred.
+    inferred: !executive.verdict,
+  };
+}
+
+/**
+ * An overall score from severity counts.
+ *
+ * Deliberately a fixed, published function rather than a judgement: a critical costs
+ * 35 points, a high 12, a medium 4, a low 1, floored at zero. Two runs of the same
+ * application produce the same number, and a reader can check the arithmetic. It is
+ * shown as "derived" precisely so nobody mistakes it for a measurement.
+ */
+export function deriveOverall(counts) {
+  const penalty =
+    (counts.critical ?? 0) * 35 + (counts.high ?? 0) * 12 + (counts.medium ?? 0) * 4 + (counts.low ?? 0) * 1;
+  return Math.max(0, 100 - penalty);
+}
+
+/** Dimension facets for the filter bar, counted from the findings themselves. */
+function dimensionFacets(findings) {
+  const counts = new Map();
+  for (const finding of findings) {
+    const key = finding.dimension;
+    if (!key) continue;
+    counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+  return [...counts.entries()]
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([key, count]) => ({ key, label: DIMENSION_LABEL[key] ?? key, count }));
+}
+
+/**
+ * The boundary of the run, assembled from what was declared and what is derivable.
+ *
+ * An unstated boundary reads as "everything was checked". Blocked cases and unrun
+ * dimensions are boundaries the artifact already knows about, so they are added to
+ * whatever the run declared rather than left to the producer to remember.
+ */
+function notCovered(result) {
+  const items = [...(result.scope?.notCovered ?? []).map(String)];
+  const ran = new Set(result.dimensionsRun ?? []);
+
+  if (ran.size > 0) {
+    for (const [key, label] of Object.entries(DIMENSION_LABEL)) {
+      if (ran.has(key)) continue;
+      items.push(`${label} was not examined in this run.`);
+    }
+  }
+
+  const db = result.dbValidation ?? {};
+  if (db.inScope === false && !db.summary) {
+    items.push('Data was not compared against the system of record — no access was provided.');
+  }
+
+  const blocked = (result.testCases?.cases ?? []).filter((testCase) => testCase.status === 'blocked');
+  if (blocked.length > 0) {
+    items.push(`Could not be run: ${blocked.map((c) => `${c.id} (${c.title})`).join(', ')}`);
+  }
+
+  return items;
+}
+
+/**
+ * Requests worth showing, ranked by how much trouble they represent.
+ *
+ * A raw endpoint list sorted by URL buries the 4-second call among ninety static
+ * assets. Failures first, then slow, then everything flagged with any issue.
+ */
+function rankEndpoints(endpoints) {
+  const weight = (endpoint) => {
+    if (endpoint.issue === 'failed' || Number(endpoint.status) >= 500) return 0;
+    if (Number(endpoint.status) >= 400) return 1;
+    if (endpoint.issue === 'slow') return 2;
+    if (endpoint.issue) return 3;
+    return 4;
+  };
+  return [...endpoints].sort(
+    (a, b) => weight(a) - weight(b) || (Number(b.durationMs) || 0) - (Number(a.durationMs) || 0),
+  );
+}
+
+/**
+ * Build the view model.
+ *
+ * `resultPath` locates the artifact on disk so evidence paths resolve against the
+ * right directory; `outPath` is where the document will be written, which is what
+ * every href is made relative to. They are usually the same directory and must never
+ * be assumed to be.
+ */
+export function buildModel(input, options = {}) {
+  // Whatever the producer's contract, it becomes one shape here — and it is refused by
+  // name if this renderer has never been tested against it.
+  const result = normalize(input);
+
+  const resultPath = options.resultPath ? path.resolve(options.resultPath) : null;
+  const baseDir = options.baseDir
+    ? path.resolve(options.baseDir)
+    : resultPath
+      ? path.dirname(resultPath)
+      : process.cwd();
+  const outDir = options.outPath ? path.dirname(path.resolve(options.outPath)) : baseDir;
+
+  const registry = createRegistry(result, {
+    baseDir,
+    outDir,
+    hash: options.hash !== false,
+    embed: options.embed === true,
+    embedLimit: options.embedLimit,
+    // Set by the bundler, which copies evidence into its own tree and needs every link
+    // pointed at the copy rather than at wherever the file was found.
+    hrefMap: options.hrefMap ?? null,
+  });
+
+  const findings = [...(result.findings ?? [])];
+  const counts = result.severityCounts ?? { critical: 0, high: 0, medium: 0, low: 0 };
+  const testCases = result.testCases ?? null;
+  const suppliedScores = result.scores ?? {};
+
+  const scores = { ...suppliedScores };
+  const overallDerived = !Number.isFinite(suppliedScores.overall);
+  if (overallDerived) scores.overall = deriveOverall(counts);
+
+  const network = result.network ?? null;
+  const endpoints = rankEndpoints(network?.endpoints ?? []);
+
+  return {
+    // Identity
+    title: result.title ?? null,
+    subject: subjectOf(result),
+    url: result.url ?? null,
+    summary: result.summary ?? '',
+    generatedAt: result.generatedAt ?? null,
+    environment: result.environment ?? null,
+    reportVersion: result.reportVersion ?? null,
+    browserAdapter: result.browserAdapter ?? null,
+    durationMs: Number.isFinite(result.durationMs) ? result.durationMs : null,
+    contract: result.contract ?? null,
+    skill: result.skill ?? null,
+
+    // Decision
+    classification: result.classification ?? null,
+    verdict: verdictOf(result),
+    scores,
+    overallDerived,
+
+    // Findings
+    findings,
+    severityCounts: counts,
+    totalFindings: SEVERITY_ORDER.reduce((sum, key) => sum + (counts[key] ?? 0), 0),
+    dimensions: dimensionFacets(findings),
+
+    // Coverage
+    scope: result.scope ?? null,
+    dimensionsRun: result.dimensionsRun ?? [],
+    notCovered: notCovered(result),
+    testCases,
+    pages: result.pages ?? [],
+    authentication: result.authentication ?? null,
+
+    // Analysis
+    timeline: result.timeline ?? [],
+    network: network ? { ...network, endpoints } : null,
+    performance: result.performance ?? null,
+    accessibility: result.accessibility ?? null,
+    security: result.security ?? null,
+    console: result.console ?? null,
+    dbValidation: result.dbValidation ?? null,
+
+    // Release rollup only: the per-test failure list and the two written summaries.
+    failures: result.failureSummary ?? [],
+    summaries: result.summaries ?? null,
+
+    // Guidance
+    whatWorksWell: result.whatWorksWell ?? [],
+    fixOrder: result.fixOrder ?? [],
+    recommendations: result.recommendations ?? [],
+
+    // Evidence
+    registry,
+    evidence: result.evidence ?? [],
+    artifactStats: registry.stats(),
+
+    // Provenance, for the appendix. `producer` is displayed and never branched on:
+    // the renderer must behave identically whoever made the report.
+    producer: result.producer ?? null,
+    versions: versionStamp(),
+    paths: { resultPath, baseDir, outDir },
+  };
+}

package/packages/engine/lib/report/core/normalize.mjs

@@ -0,0 +1,226 @@
+// Every input shape the renderer accepts, folded into one internal shape.
+//
+// ## Why this seam exists
+//
+// The promise is that a report looks the same whoever produced it — Claude Code,
+// Cursor, Codex, a future agent nobody has written yet. That promise is only keepable
+// if *no producer ever touches presentation*, which in turn is only enforceable if
+// every producer hands over the same kind of thing: structured data, validated, and
+// then rendered by code none of them can influence.
+//
+// This module is where "the same kind of thing" is defined. It accepts:
+//
+//   qa-engineer/qa-report      the canonical, producer-neutral report (schema 2.0)
+//   qa-explore/explore-result  the exploratory contract (schema 1.x)
+//   qa-report/report-result    the release rollup (schema 1.x)
+//
+// and returns one internal shape. Adding a fourth producer means adding a normalizer
+// here — not a stylesheet, not a template, and certainly not a second renderer.
+//
+// ## Why the older contracts are not simply migrated
+//
+// Reports are archived. A result written a year ago must still render, and an
+// installed skill emitting schema 1.1 must not break when the pack updates. So the
+// older shapes are translated on read rather than deprecated on write, and the
+// canonical shape is what new producers are pointed at.
+//
+// ## What a producer may not do
+//
+// Nothing here reads a colour, a class name, a font, or a layout hint, and the
+// canonical schema has no field to carry one. An agent that wants its report to look
+// different has no mechanism to make it so — which is the point.
+
+import { schemaVersionOf, SUPPORTED_SCHEMA_VERSIONS, isSupportedSchema } from '../version.mjs';
+
+export class SchemaError extends Error {
+  name = 'SchemaError';
+}
+
+/**
+ * The canonical report, unwrapped into the internal shape.
+ *
+ * The canonical form groups fields by audience — `summary` for the decision,
+ * `coverage` for what was looked at, `issues` for the detail — because that is how a
+ * producer thinks about filling it in. The internal shape is flat because that is how
+ * fifteen sections read from it. Neither is wrong; this function is the join.
+ */
+function fromCanonical(report) {
+  const metadata = report.metadata ?? {};
+  const summary = report.summary ?? {};
+  const coverage = report.coverage ?? {};
+  const recommendations = report.recommendations ?? {};
+
+  return {
+    contract: { name: 'qa-engineer/qa-report', version: report.schemaVersion ?? '2.0' },
+    skill: metadata.producer
+      ? { name: metadata.producer.skill ?? metadata.producer.agent ?? 'unknown', version: metadata.producer.version ?? '' }
+      : undefined,
+    producer: metadata.producer ?? null,
+
+    title: metadata.title,
+    url: metadata.url,
+    environment: metadata.environment,
+    generatedAt: metadata.generatedAt,
+    durationMs: metadata.durationMs,
+    reportVersion: metadata.reportVersion,
+    browserAdapter: metadata.browserAdapter,
+    authentication: metadata.authentication,
+
+    summary: summary.text ?? '',
+    classification: summary.classification ?? null,
+    executive: {
+      verdict: summary.verdict,
+      headline: summary.headline,
+      health: summary.health,
+      risks: summary.risks,
+      recommendedAction: summary.recommendedAction,
+      estimatedFixHours: summary.estimatedFixHours,
+      confidence: summary.confidence,
+    },
+    scores: summary.scores,
+    severityCounts: summary.severityCounts ?? countSeverities(report.issues ?? []),
+
+    scope: {
+      objective: coverage.objective,
+      covered: coverage.covered,
+      notCovered: coverage.notCovered,
+    },
+    dimensionsRun: coverage.dimensionsRun,
+    pages: coverage.pages,
+    testCases: coverage.testCases,
+    timeline: coverage.timeline,
+
+    // `issues` is the canonical name; `findings` is what fifteen sections already call
+    // them. Renaming the sections to match would be churn with no reader benefit.
+    findings: report.issues ?? [],
+    artifacts: report.artifacts,
+    evidence: report.evidence ?? [],
+
+    performance: report.performance,
+    security: report.security,
+    accessibility: report.accessibility,
+    console: report.console,
+    network: report.network,
+    dbValidation: report.dataValidation,
+
+    fixOrder: recommendations.fixOrder,
+    recommendations: recommendations.actions,
+    whatWorksWell: recommendations.whatWorksWell,
+  };
+}
+
+/** Severity totals, when the producer did not count them itself. */
+function countSeverities(issues) {
+  const counts = { critical: 0, high: 0, medium: 0, low: 0 };
+  for (const issue of issues) {
+    if (counts[issue.severity] !== undefined) counts[issue.severity] += 1;
+  }
+  return counts;
+}
+
+/**
+ * Fold `qa-report/report-result` — the release rollup — into the same shape.
+ *
+ * The rollup describes the same event from further away: one summary of many runs
+ * rather than one run of a product. Its readiness verdict becomes the executive
+ * verdict and its test summary becomes the test counts. Its failure list stays a list
+ * of failures rather than being forced into `findings` with an invented severity for
+ * each row — a failing assertion is not a severity-ranked defect, and pretending
+ * otherwise would put fabricated numbers in the summary tiles.
+ */
+function fromRollup(result) {
+  const readiness = result.releaseReadiness ?? {};
+  const tests = result.testSummary ?? {};
+  const summaries = result.summaries ?? {};
+
+  return {
+    ...result,
+    executive: result.executive ?? {
+      verdict: readiness.verdict ?? result.classification,
+      headline: readiness.rationale ?? result.summary,
+      health: summaries.executive ?? null,
+    },
+    severityCounts: result.severityCounts ?? { critical: 0, high: 0, medium: 0, low: 0 },
+    testCases: result.testCases ?? (Object.keys(tests).length > 0
+      ? {
+        total: tests.total ?? 0,
+        passed: tests.passed ?? 0,
+        failed: tests.failed ?? 0,
+        blocked: tests.blocked ?? 0,
+        skipped: tests.skipped ?? 0,
+        cases: [],
+      }
+      : null),
+  };
+}
+
+/** The exploratory contract is already the internal shape; nothing to translate. */
+function fromExplore(result) {
+  return result;
+}
+
+const NORMALIZERS = Object.freeze({
+  'qa-engineer/qa-report': fromCanonical,
+  'qa-explore/explore-result': fromExplore,
+  'qa-report/report-result': fromRollup,
+});
+
+/** Contract names this renderer accepts, for an error message and for `--help`. */
+export function supportedContracts() {
+  return Object.keys(NORMALIZERS).sort();
+}
+
+/**
+ * Which producer made this report, for the provenance block.
+ *
+ * Recorded and displayed, never acted on: the renderer's behaviour must not vary by
+ * producer, or the guarantee that two agents produce identical output is only true
+ * until someone adds a special case.
+ */
+export function producerOf(result) {
+  const producer = result?.metadata?.producer ?? result?.producer;
+  if (producer) {
+    return {
+      agent: producer.agent ?? null,
+      model: producer.model ?? null,
+      skill: producer.skill ?? null,
+      version: producer.version ?? null,
+    };
+  }
+  if (result?.skill) {
+    return { agent: null, model: null, skill: result.skill.name, version: result.skill.version };
+  }
+  return null;
+}
+
+/**
+ * Normalize any accepted contract into the internal shape.
+ *
+ * Throws rather than guessing. A result whose contract is unknown, or whose schema
+ * version this renderer has never been tested against, is refused by name — rendering
+ * it half-understood would produce a document that looks authoritative and is missing
+ * whatever the unknown version added.
+ */
+export function normalize(result) {
+  const name = result?.contract?.name;
+  const normalizer = NORMALIZERS[name];
+
+  if (!normalizer) {
+    throw new SchemaError(
+      `no renderer for contract ${name === undefined ? 'None' : `'${name}'`}; supported: ` +
+        `${supportedContracts().join(', ')}`,
+    );
+  }
+
+  if (!isSupportedSchema(result)) {
+    throw new SchemaError(
+      `report declares schema version ${schemaVersionOf(result)}, which this renderer ` +
+        `does not support; supported: ${SUPPORTED_SCHEMA_VERSIONS.join(', ')}`,
+    );
+  }
+
+  const normalized = normalizer(result);
+  // Provenance survives normalization so the appendix can state who produced the
+  // report even when the contract carried it somewhere else.
+  return { ...normalized, producer: producerOf(result) };
+}