qa-engineer 0.10.0 → 0.11.0

package/packages/engine/lib/report/export/machine.mjs

@@ -0,0 +1,290 @@
+// Machine-readable exports: SARIF, JUnit, and CSV.
+//
+// These exist so a QA run can enter systems that were never going to read an HTML
+// page. SARIF puts findings in GitHub's Security tab and any code-scanning viewer;
+// JUnit puts them on a CI dashboard beside the unit tests; CSV puts them in the
+// spreadsheet a QA lead is already tracking the release from.
+//
+// ## Where the mapping is honest and where it is lossy
+//
+// SARIF and JUnit were designed for static analysis and unit tests respectively, and
+// an exploratory QA finding is neither. Rather than pretend otherwise:
+//
+//   - A finding with no `page` has no meaningful file location, so SARIF gets the
+//     target URL as its artifact URI rather than an invented source path. A reader
+//     seeing `admin.example.com/users` knows it is a runtime finding, not line 12 of a
+//     file that does not exist.
+//   - JUnit's model has no severity, so severity travels in the failure `type` and the
+//     dimension in the classname. Nothing is dropped silently; it is relocated and
+//     documented.
+//
+// Anything that genuinely does not fit — evidence images, root-cause chains — stays in
+// the JSON, which every one of these formats can point back to.
+
+import { buildModel } from '../core/model.mjs';
+import { SEVERITY } from '../theme/tokens.mjs';
+import { dimensionLabel } from '../components/findings.mjs';
+
+/** SARIF severity levels. SARIF has three; the pack has four, so medium and low share. */
+const SARIF_LEVEL = Object.freeze({ critical: 'error', high: 'error', medium: 'warning', low: 'note' });
+
+// SARIF's own 0–10 scale, so a viewer that ranks by score ranks the way the report does.
+const SARIF_RANK = Object.freeze({ critical: 10, high: 8, medium: 5, low: 2 });
+
+/** SARIF 2.1.0, one run, one rule per distinct dimension+severity pairing. */
+export function renderSarif(result, options = {}) {
+  const model = buildModel(result, { ...options, hash: false });
+
+  const rules = new Map();
+  const results = model.findings.map((finding) => {
+    const ruleId = `qa-explore/${finding.dimension ?? 'general'}`;
+    if (!rules.has(ruleId)) {
+      rules.set(ruleId, {
+        id: ruleId,
+        name: `${dimensionLabel(finding.dimension)} finding`,
+        shortDescription: { text: `${dimensionLabel(finding.dimension)} defect found by exploratory QA` },
+        fullDescription: {
+          text:
+            'Reported by qa-explore, an AI-driven exploratory QA pass against a running application. ' +
+            'Each result carries the observed behaviour, the expected behaviour, and reproduction steps.',
+        },
+        defaultConfiguration: { level: SARIF_LEVEL[finding.severity] ?? 'note' },
+        properties: { tags: ['qa', 'exploratory', finding.dimension ?? 'general'] },
+      });
+    }
+
+    const location = finding.page ?? model.url ?? 'unknown';
+    const message = [
+      finding.actual,
+      `Expected: ${finding.expected}`,
+      `Fix: ${finding.fixDirection}`,
+    ].join('\n\n');
+
+    return {
+      ruleId,
+      level: SARIF_LEVEL[finding.severity] ?? 'note',
+      rank: SARIF_RANK[finding.severity] ?? 1,
+      message: { text: message },
+      locations: [
+        {
+          physicalLocation: {
+            artifactLocation: { uri: location },
+            region: { startLine: 1 },
+          },
+          message: { text: finding.title },
+        },
+      ],
+      partialFingerprints: { findingId: String(finding.id) },
+      properties: {
+        findingId: finding.id,
+        severity: finding.severity,
+        dimension: finding.dimension,
+        status: finding.status,
+        businessImpact: finding.businessImpact ?? undefined,
+        rootCause: finding.rootCause?.summary ?? undefined,
+        regressionRisk: finding.regressionRisk?.level ?? undefined,
+        confidence: finding.confidence ?? undefined,
+        repro: finding.repro,
+      },
+    };
+  });
+
+  return {
+    $schema: 'https://json.schemastore.org/sarif-2.1.0.json',
+    version: '2.1.0',
+    runs: [
+      {
+        tool: {
+          driver: {
+            name: 'qa-engineer',
+            informationUri: 'https://github.com/abisheik88/qa-engineer',
+            rules: [...rules.values()],
+            version: model.skill?.version ?? undefined,
+          },
+        },
+        invocations: [
+          {
+            executionSuccessful: model.classification !== 'blocked',
+            endTimeUtc: model.generatedAt ?? undefined,
+          },
+        ],
+        results,
+      },
+    ],
+  };
+}
+
+/** XML-escape, for both text nodes and attribute values. */
+function x(value) {
+  return String(value ?? '')
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+    // A control character is legal in a JSON string and illegal in XML 1.0, so a
+    // console excerpt carrying one would produce a file no parser accepts.
+    // eslint-disable-next-line no-control-regex
+    .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/g, '');
+}
+
+/**
+ * JUnit XML.
+ *
+ * Two suites: the executed test cases (when the run had any) and the findings. Keeping
+ * them apart matters — a CI dashboard that mixes "the login case failed" with "the
+ * login page leaks a token into localStorage" cannot tell a regression from a review
+ * comment.
+ */
+export function renderJUnit(result, options = {}) {
+  const model = buildModel(result, { ...options, hash: false });
+  const suites = [];
+
+  const cases = model.testCases?.cases ?? [];
+  if (cases.length > 0) {
+    const bodies = cases.map((testCase) => {
+      const attributes =
+        `name="${x(testCase.title)}" classname="${x(`qa-explore.cases.${testCase.id}`)}"`;
+      if (testCase.status === 'pass') return `    <testcase ${attributes}/>`;
+      if (testCase.status === 'skipped') {
+        return `    <testcase ${attributes}><skipped/></testcase>`;
+      }
+      if (testCase.status === 'blocked') {
+        return (
+          `    <testcase ${attributes}><skipped message="Blocked: could not be executed"/></testcase>`
+        );
+      }
+      const finding = model.findings.find((entry) => entry.id === testCase.findingId);
+      const detail = finding
+        ? `${finding.actual}\n\nExpected: ${finding.expected}`
+        : (testCase.note ?? 'Case failed.');
+      return (
+        `    <testcase ${attributes}>` +
+        `<failure type="case-failed" message="${x(testCase.title)}">${x(detail)}</failure>` +
+        '</testcase>'
+      );
+    });
+    suites.push(
+      `  <testsuite name="qa-explore.cases" tests="${cases.length}" ` +
+        `failures="${cases.filter((c) => c.status === 'fail').length}" ` +
+        `skipped="${cases.filter((c) => c.status === 'skipped' || c.status === 'blocked').length}">\n` +
+        `${bodies.join('\n')}\n  </testsuite>`,
+    );
+  }
+
+  if (model.findings.length > 0) {
+    const bodies = model.findings.map((finding) => {
+      const detail = [
+        finding.actual,
+        `Expected: ${finding.expected}`,
+        `Reproduction: ${finding.repro}`,
+        `Fix: ${finding.fixDirection}`,
+      ].join('\n\n');
+      return (
+        `    <testcase name="${x(`${finding.id} ${finding.title}`)}" ` +
+        `classname="${x(`qa-explore.findings.${finding.dimension ?? 'general'}`)}">` +
+        `<failure type="${x(finding.severity)}" message="${x(finding.title)}">${x(detail)}</failure>` +
+        '</testcase>'
+      );
+    });
+    suites.push(
+      `  <testsuite name="qa-explore.findings" tests="${model.findings.length}" ` +
+        `failures="${model.findings.length}">\n${bodies.join('\n')}\n  </testsuite>`,
+    );
+  }
+
+  const totals = suites.length;
+  return (
+    '<?xml version="1.0" encoding="UTF-8"?>\n' +
+    `<testsuites name="qa-explore" tests="${cases.length + model.findings.length}" ` +
+    `failures="${cases.filter((c) => c.status === 'fail').length + model.findings.length}"` +
+    (totals === 0 ? '/>\n' : `>\n${suites.join('\n')}\n</testsuites>\n`)
+  );
+}
+
+/** RFC 4180 quoting: wrap when needed, double any embedded quote. */
+function csvCell(value) {
+  const text = String(value ?? '');
+  return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+}
+
+/**
+ * Findings as CSV.
+ *
+ * Column order is triage order, so the sheet is usable unsorted: id, severity, the two
+ * behaviours, then the context. A leading `` because Excel opens a UTF-8 CSV
+ * without one as mojibake, and a report that renders `â€"` in a stakeholder's
+ * spreadsheet has failed regardless of how correct the bytes are.
+ */
+export function renderCsv(result, options = {}) {
+  const model = buildModel(result, { ...options, hash: false });
+  const headers = [
+    'id', 'severity', 'dimension', 'status', 'title', 'page', 'actual', 'expected',
+    'businessImpact', 'repro', 'fixDirection', 'rootCause', 'regressionRisk',
+    'confidence', 'tags', 'affectedApis', 'evidenceCount',
+  ];
+
+  const rows = model.findings.map((finding) =>
+    [
+      finding.id,
+      finding.severity,
+      finding.dimension,
+      finding.status,
+      finding.title,
+      finding.page ?? '',
+      finding.actual,
+      finding.expected,
+      finding.businessImpact ?? '',
+      finding.repro,
+      finding.fixDirection,
+      finding.rootCause?.summary ?? '',
+      finding.regressionRisk?.level ?? '',
+      finding.confidence ?? '',
+      (finding.tags ?? []).join(' '),
+      (finding.affectedApis ?? []).join(' '),
+      (finding.evidence ?? []).length,
+    ]
+      .map(csvCell)
+      .join(','),
+  );
+
+  return `\ufeff${[headers.join(','), ...rows].join('\r\n')}\r\n`;
+}
+
+/**
+ * A manifest of everything a shareable bundle should contain.
+ *
+ * The pack writes no archive itself — zipping is the caller's job and every platform
+ * already has a tool for it. What the pack owns is knowing *what belongs in the
+ * archive*, which is the part that goes wrong: a bundle missing three screenshots is
+ * indistinguishable from a complete one until someone opens it.
+ */
+export function bundleManifest(result, options = {}) {
+  const model = buildModel(result, options);
+  const files = model.registry
+    .list()
+    .filter((record) => record.declaredPath && !record.external)
+    .map((record) => ({
+      path: record.declaredPath,
+      kind: record.kind,
+      bytes: record.bytes,
+      sha256: record.sha256,
+      present: record.exists,
+      reason: record.exists ? undefined : record.missingReason,
+    }));
+
+  return {
+    subject: model.subject,
+    generatedAt: model.generatedAt,
+    complete: files.every((file) => file.present),
+    counts: {
+      total: files.length,
+      present: files.filter((file) => file.present).length,
+      missing: files.filter((file) => !file.present).length,
+    },
+    totalBytes: files.reduce((sum, file) => sum + (file.bytes ?? 0), 0),
+    files,
+  };
+}
+
+export { SEVERITY };

package/packages/engine/lib/report/export/markdown.mjs

@@ -0,0 +1,323 @@
+// The Markdown rendering.
+//
+// Markdown is the format a finding gets pasted into a Jira ticket, a pull request, or
+// a Slack thread from, so it is written for *excerpting* rather than for reading end to
+// end: each finding is a self-contained block with its own heading, and no finding
+// depends on a table three sections above it to make sense.
+//
+// It is rendered from the same model as the HTML, which is the only reason the two can
+// be trusted to agree. Producing the prose separately is exactly how the first live run
+// ended up with a Markdown report that said things the JSON did not.
+
+import { footerMarkdown } from '../../analysis/branding.mjs';
+import { buildModel } from '../core/model.mjs';
+import { SEVERITY, SEVERITY_ORDER } from '../theme/tokens.mjs';
+import { dimensionLabel, STATUS_LABEL } from '../components/findings.mjs';
+import { formatDuration, formatBytes } from '../components/primitives.mjs';
+
+/** Escape the characters that would restructure a table cell or a heading. */
+function cell(value) {
+  return String(value ?? '').replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
+}
+
+function table(headers, rows) {
+  if (rows.length === 0) return '';
+  return [
+    `| ${headers.join(' | ')} |`,
+    `| ${headers.map(() => '---').join(' | ')} |`,
+    ...rows.map((row) => `| ${row.map(cell).join(' | ')} |`),
+    '',
+  ].join('\n');
+}
+
+function findingBlock(finding, model) {
+  const severity = SEVERITY[finding.severity]?.label ?? finding.severity;
+  const lines = [
+    `### ${finding.id} · ${severity} — ${finding.title}`,
+    '',
+    `**Area** ${dimensionLabel(finding.dimension)} · ` +
+      `**Status** ${STATUS_LABEL[finding.status] ?? finding.status}` +
+      (finding.page ? ` · **Page** ${finding.page}` : ''),
+    '',
+    `**Current behaviour** — ${finding.actual}`,
+    '',
+    `**Expected behaviour** — ${finding.expected}`,
+    '',
+  ];
+
+  if (finding.businessImpact) lines.push(`**Business impact** — ${finding.businessImpact}`, '');
+
+  lines.push('**Reproduction**', '');
+  if (Array.isArray(finding.steps) && finding.steps.length > 0) {
+    finding.steps.forEach((step, index) => lines.push(`${index + 1}. ${step}`));
+  } else {
+    lines.push(finding.repro);
+  }
+  lines.push('');
+
+  if (finding.rootCause?.summary) {
+    lines.push(`**Root cause** — ${finding.rootCause.summary}`, '');
+    for (const step of finding.rootCause.chain ?? []) lines.push(`- ${step}`);
+    if (finding.rootCause.chain?.length) lines.push('');
+  }
+
+  lines.push(`**Suggested fix** — ${finding.fixDirection}`, '');
+
+  if (finding.regressionRisk?.level) {
+    lines.push(
+      `**Regression risk** — ${finding.regressionRisk.level}` +
+        (finding.regressionRisk.note ? `: ${finding.regressionRisk.note}` : ''),
+      '',
+    );
+    for (const item of finding.regressionRisk.retest ?? []) lines.push(`- Re-test: ${item}`);
+    if (finding.regressionRisk.retest?.length) lines.push('');
+  }
+
+  if (finding.developerNotes) lines.push(`**Developer notes** — ${finding.developerNotes}`, '');
+
+  if (finding.metrics?.length) {
+    lines.push(
+      table(
+        ['Measurement', 'Observed', 'Budget'],
+        finding.metrics.map((metric) => [metric.label, metric.value, metric.budget ?? '—']),
+      ),
+    );
+  }
+
+  // Evidence goes through the registry so a missing file is *stated* in Markdown too,
+  // rather than becoming a dead image reference in a ticket.
+  const evidence = (finding.evidence ?? [])
+    .map((entry) => ({ entry, record: model.registry.forEvidence(entry) }))
+    .filter((pair) => pair.record);
+  if (evidence.length > 0) {
+    lines.push('**Evidence**', '');
+    for (const { entry, record } of evidence) {
+      if (!record.exists) {
+        lines.push(`- *Artifact missing* — \`${record.declaredPath}\` (${record.missingReason})`);
+        continue;
+      }
+      if (record.renderAs === 'image') {
+        lines.push(`![${record.label ?? entry.type}](${record.href})`, '');
+      } else {
+        lines.push(`- [${record.label ?? entry.type}](${record.href})`);
+      }
+      if (entry.excerpt) lines.push('', '```', entry.excerpt, '```', '');
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+/** Render the whole report as Markdown. */
+export function renderMarkdown(result, options = {}) {
+  const model = buildModel(result, options);
+  const out = [];
+
+  out.push(`# ${model.subject} — QA Report`, '');
+  out.push(
+    table(
+      ['Field', 'Value'],
+      [
+        ['URL', model.url ?? '—'],
+        ['Generated', model.generatedAt ?? '—'],
+        model.environment ? ['Environment', model.environment] : null,
+        ['Verdict', `**${model.verdict.label}** — ${model.verdict.blurb}`],
+        ['Findings', String(model.totalFindings)],
+        model.durationMs !== null ? ['Duration', formatDuration(model.durationMs)] : null,
+        model.reportVersion ? ['Report version', `v${model.reportVersion}`] : null,
+        ['Tester', 'AI-assisted QA (qa-explore)'],
+      ].filter(Boolean),
+    ),
+  );
+
+  out.push('## Summary', '', model.summary, '');
+
+  if (model.verdict.risks.length > 0) {
+    out.push('### Why', '');
+    for (const risk of model.verdict.risks) out.push(`- ${risk}`);
+    out.push('');
+  }
+  if (model.verdict.recommendedAction) {
+    out.push(`**Recommended action** — ${model.verdict.recommendedAction}`, '');
+  }
+  if (model.verdict.estimatedFixHours) {
+    const { low, high } = model.verdict.estimatedFixHours;
+    out.push(`**Estimated fix time** — ${low === high ? low : `${low}–${high}`} hours`, '');
+  }
+
+  out.push(
+    '## Findings by severity',
+    '',
+    table(
+      SEVERITY_ORDER.map((key) => SEVERITY[key].label),
+      [SEVERITY_ORDER.map((key) => String(model.severityCounts[key] ?? 0))],
+    ),
+  );
+
+  if (model.scope?.objective) out.push('## Scope', '', model.scope.objective, '');
+  if (model.scope?.covered?.length) {
+    out.push('**Exercised**', '');
+    for (const item of model.scope.covered) out.push(`- ${item}`);
+    out.push('');
+  }
+  if (model.notCovered.length > 0) {
+    out.push('**Not covered in this run**', '');
+    for (const item of model.notCovered) out.push(`- ${item}`);
+    out.push('');
+  }
+
+  if (model.pages.length > 0) {
+    out.push(
+      '## Pages',
+      '',
+      table(
+        ['Page', 'State', 'HTTP', 'Load', 'Requests'],
+        model.pages.map((page) => [
+          page.title || page.url,
+          page.status,
+          page.httpStatus ?? '—',
+          page.loadMs !== undefined ? formatDuration(page.loadMs) : '—',
+          page.requests ?? '—',
+        ]),
+      ),
+    );
+  }
+
+  out.push('## Findings', '');
+  if (model.findings.length === 0) {
+    out.push('_No findings recorded._', '');
+  } else {
+    const sorted = [...model.findings].sort(
+      (a, b) =>
+        (SEVERITY[a.severity] ?? SEVERITY.low).rank - (SEVERITY[b.severity] ?? SEVERITY.low).rank ||
+        String(a.id).localeCompare(String(b.id), 'en', { numeric: true }),
+    );
+    for (const finding of sorted) out.push(findingBlock(finding, model), '');
+  }
+
+  if (model.performance) {
+    const rows = Object.entries(model.performance)
+      .filter(([, value]) => value !== null && value !== undefined && typeof value !== 'string')
+      .map(([key, value]) => [key, key.endsWith('Bytes') ? formatBytes(value) : String(value)]);
+    if (rows.length > 0) out.push('## Performance', '', table(['Metric', 'Value'], rows));
+  }
+
+  if (model.network?.endpoints?.length) {
+    out.push(
+      '## API and network',
+      '',
+      table(
+        ['Method', 'Endpoint', 'Status', 'Time', 'Size', 'Count', 'Issue'],
+        model.network.endpoints.slice(0, 40).map((endpoint) => [
+          endpoint.method,
+          endpoint.url,
+          endpoint.status ?? '—',
+          Number.isFinite(endpoint.durationMs) ? formatDuration(endpoint.durationMs) : '—',
+          Number.isFinite(endpoint.bytes) ? formatBytes(endpoint.bytes) : '—',
+          endpoint.count ?? 1,
+          endpoint.issue ?? '',
+        ]),
+      ),
+    );
+  }
+
+  if (model.security?.checks?.length) {
+    out.push(
+      '## Security',
+      '',
+      table(
+        ['Check', 'Result', 'Detail'],
+        model.security.checks.map((check) => [check.check, check.status, check.detail ?? '']),
+      ),
+    );
+  }
+
+  if (model.accessibility?.violations?.length) {
+    out.push(
+      '## Accessibility',
+      '',
+      table(
+        ['Rule', 'Impact', 'Instances', 'Description'],
+        model.accessibility.violations.map((violation) => [
+          violation.rule, violation.impact, violation.count, violation.description ?? '',
+        ]),
+      ),
+    );
+  }
+
+  if (model.testCases?.cases?.length) {
+    out.push(
+      '## Test case coverage',
+      '',
+      table(
+        ['ID', 'Case', 'Result', 'Finding'],
+        model.testCases.cases.map((testCase) => [
+          testCase.id, testCase.title, testCase.status, testCase.findingId ?? '—',
+        ]),
+      ),
+    );
+  }
+
+  if (model.dbValidation?.comparisons?.length) {
+    out.push(
+      '## Data validation',
+      '',
+      table(
+        ['Metric', 'UI', 'Source of truth', 'Result'],
+        model.dbValidation.comparisons.map((comparison) => [
+          comparison.metric, comparison.uiValue, comparison.sourceValue,
+          comparison.match ? 'Match' : 'Mismatch',
+        ]),
+      ),
+    );
+  }
+
+  if (model.whatWorksWell.length > 0) {
+    out.push('## What works well', '');
+    for (const item of model.whatWorksWell) out.push(`- ${item}`);
+    out.push('');
+  }
+
+  if (model.fixOrder.length > 0) {
+    out.push('## Suggested fix order', '');
+    model.fixOrder.forEach((item, index) => out.push(`${index + 1}. ${item}`));
+    out.push('');
+  }
+
+  if (model.recommendations.length > 0) {
+    out.push(
+      '## Recommendations',
+      '',
+      table(
+        ['Action', 'Priority', 'Owner', 'Effort'],
+        model.recommendations.map((recommendation) => [
+          recommendation.action,
+          recommendation.priority,
+          recommendation.owner ?? '—',
+          recommendation.effort ?? '—',
+        ]),
+      ),
+    );
+  }
+
+  const stats = model.artifactStats;
+  if (stats.total > 0) {
+    out.push(
+      '## Artifacts',
+      '',
+      `${stats.present} of ${stats.total} present (${formatBytes(stats.bytes)}).` +
+        (stats.missing > 0 ? ` **${stats.missing} missing.**` : ''),
+      '',
+      table(
+        ['Kind', 'File', 'State'],
+        model.registry
+          .list()
+          .filter((record) => record.declaredPath)
+          .map((record) => [record.kind, record.declaredPath, record.exists ? 'Present' : 'Missing']),
+      ),
+    );
+  }
+
+  return `${out.join('\n').replace(/\n{3,}/g, '\n\n').trim()}\n\n${footerMarkdown()}`;
+}