qa-engineer 0.9.2 → 0.11.0

package/packages/engine/lib/analysis/evidence.mjs

@@ -0,0 +1,116 @@
+// The evidence and finding model shared by every analyzer.
+//
+// Every finding an analyzer produces carries the same structure, so downstream
+// skills (qa-debug, qa-report, qa-fix) consume one shape regardless of which
+// analyzer or framework produced it. Text fields are redacted at construction.
+//
+// Ported from qa_analysis/evidence.py. Python used dataclasses with a `to_dict()`;
+// here each factory returns the serialized shape directly, because that shape —
+// not the object — is what every consumer actually uses, and one representation
+// cannot drift from the other.
+
+import { redactText } from './redaction.mjs';
+
+export const EVIDENCE_TYPES = new Set([
+  'trace', 'har', 'junit', 'report', 'console', 'network', 'stdout',
+  'stderr', 'screenshot', 'video', 'log', 'file', 'diff',
+]);
+
+/** ISO 8601 UTC timestamp. Isolated so tests can substitute it. */
+export function utcNow() {
+  return new Date().toISOString();
+}
+
+/**
+ * One observation supporting a finding. Excerpts are redacted here, at
+ * construction, so no caller can forget to do it.
+ */
+export function evidence({ type, description, source, excerpt = '' }) {
+  if (!EVIDENCE_TYPES.has(type)) {
+    throw new Error(`unknown evidence type: ${type}`);
+  }
+  const redacted = redactText(excerpt);
+  const entry = { type, description, source };
+  if (redacted) entry.excerpt = redacted;
+  return entry;
+}
+
+/**
+ * A single diagnostic conclusion, traceable to a specific artifact.
+ *
+ * Carries everything the evidence model requires: the artifact and location it
+ * came from, when, why, the supporting evidence, a calibrated confidence, the
+ * affected tests, related artifacts, and recommended actions.
+ */
+export function finding({
+  classification,
+  reason,
+  artifact: artifactPath,
+  location,
+  confidence = null,
+  timestamp = undefined,
+  evidence: items = [],
+  affectedTests = [],
+  relatedArtifacts = [],
+  recommendations = [],
+}) {
+  const result = {
+    classification,
+    reason,
+    artifact: artifactPath,
+    location,
+    timestamp: timestamp ?? utcNow(),
+    evidence: items,
+    affectedTests,
+    relatedArtifacts,
+    recommendations,
+  };
+  if (confidence !== null && confidence !== undefined) result.confidence = confidence;
+  return result;
+}
+
+/**
+ * The envelope an analyzer emits: findings plus the artifacts it examined.
+ *
+ * A downstream skill wraps this in its own output contract; on its own it is the
+ * deterministic, machine-readable result of one analysis.
+ */
+export function analyzerOutput({
+  analyzer,
+  findings = [],
+  artifacts = [],
+  warnings = [],
+  generatedAt = undefined,
+}) {
+  return {
+    analyzer,
+    generatedAt: generatedAt ?? utcNow(),
+    findings,
+    artifacts,
+    warnings,
+  };
+}
+
+/** A discovered artifact, in the common model shared with the execution engine. */
+export function artifact({
+  type,
+  location,
+  framework = 'unknown',
+  ownership = 'qa-analysis',
+  timestamp = undefined,
+  mediaType = '',
+  testRef = '',
+  present = true,
+}) {
+  const entry = {
+    type,
+    location,
+    framework,
+    timestamp: timestamp ?? utcNow(),
+    ownership,
+    present,
+  };
+  if (mediaType) entry.mediaType = mediaType;
+  if (testRef) entry.testRef = testRef;
+  return entry;
+}

package/packages/engine/lib/analysis/har.mjs

@@ -0,0 +1,124 @@
+// HAR (HTTP Archive) parser.
+//
+// Framework-agnostic: a HAR is a standard JSON format, whoever produced it.
+// Extracts request outcomes, flags failures and slow calls, and redacts headers
+// and credentialed URLs before anything is exposed. A malformed HAR raises.
+
+import fs from 'node:fs';
+
+import { redactText, redactHeaders } from './redaction.mjs';
+import { MalformedArtifact } from './junit.mjs';
+
+/**
+ * Parse HAR JSON into a redacted network summary.
+ *
+ * `{entries, failures, slow, redacted: true}`, where each entry is
+ * `{method, url, status, durationMs, requestHeaders, responseHeaders}`. Failures
+ * are entries with status >= 400 or status 0 (no response at all).
+ */
+export function parseHarData(data, { slowMs = 1000, label = '<input>' } = {}) {
+  const rawEntries = data?.log?.entries;
+  if (!Array.isArray(rawEntries)) {
+    throw new MalformedArtifact(`not a HAR document at ${label}`);
+  }
+
+  const entries = rawEntries.map((item) => {
+    if (item === null || typeof item !== 'object' || Array.isArray(item)) {
+      throw new MalformedArtifact(`HAR log.entries contains a non-object at ${label}`);
+    }
+    const request = item.request ?? {};
+    const response = item.response ?? {};
+    return {
+      method: request.method ?? '',
+      // Redaction strips any credentials embedded in the URL.
+      url: redactText(request.url ?? ''),
+      status: toInt(response.status),
+      durationMs: toMillis(item?.time, label),
+      requestHeaders: redactHeaders(request.headers ?? []),
+      responseHeaders: redactHeaders(response.headers ?? []),
+      // Three facts a report needs that a summary of failures cannot supply: how big
+      // the response was, when it started relative to the others, and whether the
+      // server said anything about caching. All three are read straight from the HAR
+      // — none is inferred — and all three are absent from many HARs, which is why
+      // each has an explicit "unknown" rather than a plausible zero.
+      bytes: bodyBytes(response),
+      startedAt: typeof item.startedDateTime === 'string' ? item.startedDateTime : null,
+      cacheControl: headerValue(response.headers, 'cache-control'),
+      etag: headerValue(response.headers, 'etag'),
+      resourceType: typeof item._resourceType === 'string' ? item._resourceType : null,
+    };
+  });
+
+  return {
+    entries,
+    failures: entries.filter((e) => e.status === 0 || e.status >= 400),
+    slow: entries.filter((e) => e.durationMs >= slowMs),
+    redacted: true,
+  };
+}
+
+/** Read and parse a HAR file from disk. */
+export function parseHar(path, { slowMs = 1000 } = {}) {
+  let data;
+  try {
+    data = JSON.parse(fs.readFileSync(path, 'utf8'));
+  } catch (error) {
+    throw new MalformedArtifact(`could not parse HAR at ${path}: ${error.message}`);
+  }
+  return parseHarData(data, { slowMs, label: path });
+}
+
+/**
+ * Response body size in bytes, or null when the HAR does not say.
+ *
+ * `content.size` is the decoded size and `bodySize` the bytes on the wire; either is
+ * `-1` when the producer did not record it. Null rather than 0, because "we do not
+ * know how big this was" and "this response was empty" are different findings and a
+ * report that conflates them invents a fact.
+ */
+function bodyBytes(response) {
+  for (const candidate of [response?.content?.size, response?.bodySize]) {
+    const number = Number(candidate);
+    if (Number.isFinite(number) && number >= 0) return Math.trunc(number);
+  }
+  return null;
+}
+
+/** A header's value, case-insensitively, or null. */
+function headerValue(headers, name) {
+  if (!Array.isArray(headers)) return null;
+  const wanted = name.toLowerCase();
+  const found = headers.find((header) => String(header?.name ?? '').toLowerCase() === wanted);
+  return found ? String(found.value ?? '') : null;
+}
+
+/** A status that is absent or unreadable is 0 — "no response", which is a failure. */
+function toInt(value) {
+  const number = Number(value);
+  return Number.isFinite(number) ? Math.trunc(number) : 0;
+}
+
+/**
+ * HAR `time` is milliseconds already, rounded to a whole number.
+ *
+ * A value that is present but not a number raises rather than becoming 0: the
+ * same rule as the JUnit parser, and for the same reason — a fabricated duration
+ * makes a malformed document look like a clean measurement.
+ */
+function toMillis(raw, label) {
+  if (raw === undefined || raw === null || raw === '') return 0;
+  const number = Number(raw);
+  if (!Number.isFinite(number)) {
+    throw new MalformedArtifact(`entry time='${raw}' is not a number at ${label}`);
+  }
+  return roundHalfToEven(number);
+}
+
+/** Python's rounding rule, so a `.5` boundary lands on the same integer. */
+function roundHalfToEven(value) {
+  const floor = Math.floor(value);
+  const remainder = value - floor;
+  if (remainder > 0.5) return floor + 1;
+  if (remainder < 0.5) return floor;
+  return floor % 2 === 0 ? floor : floor + 1;
+}

package/packages/engine/lib/analysis/junit.mjs

@@ -0,0 +1,126 @@
+// JUnit XML parser.
+//
+// Framework-agnostic: Playwright, Selenium, Cypress, WebdriverIO, and most unit
+// runners emit JUnit XML, so this one parser normalizes them all into the pack's
+// per-test result shape. This is the concrete proof that different frameworks
+// share one contract — only where the file lives differs, not how it is read.
+//
+// Parses deterministically; a malformed document raises rather than guessing.
+
+import fs from 'node:fs';
+
+import { parseXml, find, findAll, XmlError } from './xml.mjs';
+import { redactText } from './redaction.mjs';
+
+/** Raised when an artifact cannot be parsed. Never swallowed into a guess. */
+export class MalformedArtifact extends Error {}
+
+/**
+ * Parse JUnit XML text into a normalized result.
+ *
+ * `{tests: {total, passed, failed, skipped}, executed: [...]}`, where each
+ * executed entry is `{title, file, status, durationMs, message?}`.
+ */
+export function parseJUnitText(source, label = '<input>') {
+  let root;
+  try {
+    root = parseXml(source);
+  } catch (error) {
+    if (error instanceof XmlError) {
+      throw new MalformedArtifact(`could not parse JUnit XML at ${label}: ${error.message}`);
+    }
+    throw error;
+  }
+
+  // Accept either a <testsuites> root or a single <testsuite> root.
+  let suites;
+  if (root.tag === 'testsuites') suites = findAll(root, 'testsuite');
+  else if (root.tag === 'testsuite') suites = [root];
+  else throw new MalformedArtifact(`not a JUnit document (root <${root.tag}>) at ${label}`);
+
+  const executed = [];
+  for (const suite of suites) {
+    for (const testcase of findAll(suite, 'testcase')) {
+      const failure = find(testcase, 'failure');
+      const error = find(testcase, 'error');
+      const skipped = find(testcase, 'skipped');
+
+      let status;
+      let message = '';
+      if (failure !== null || error !== null) {
+        status = 'failed';
+        const node = failure !== null ? failure : error;
+        message = redactText((node.attrs.message || node.text || '').trim());
+      } else if (skipped !== null) {
+        status = 'skipped';
+        message = redactText((skipped.attrs.message || '').trim());
+      } else {
+        status = 'passed';
+      }
+
+      const entry = {
+        title: testcase.attrs.name ?? '',
+        file: testcase.attrs.classname ?? '',
+        status,
+        durationMs: durationMs(testcase.attrs.time, label),
+      };
+      if (message) entry.message = message;
+      executed.push(entry);
+    }
+  }
+
+  return {
+    tests: {
+      total: executed.length,
+      passed: executed.filter((e) => e.status === 'passed').length,
+      failed: executed.filter((e) => e.status === 'failed').length,
+      skipped: executed.filter((e) => e.status === 'skipped').length,
+    },
+    executed,
+  };
+}
+
+/**
+ * `time` in seconds becomes whole milliseconds.
+ *
+ * Two subtleties, both pinned by the parity corpus:
+ *
+ * 1. An absent or empty attribute is zero; a value that is present but not a
+ *    finite number raises. Guessing zero there would turn a malformed document
+ *    into a plausible result, which is the one thing this parser must not do.
+ * 2. Python's `round()` rounds half to *even* and JavaScript's `Math.round`
+ *    rounds half *up*, so `time="0.0005"` gives 0 in Python and 1 here unless
+ *    the tie is broken the same way. Runners do emit such values.
+ */
+function durationMs(raw, label) {
+  if (raw === undefined || raw === null || raw === '') return 0;
+  // Reject what parseFloat would happily truncate ("1.2.3", "5s", "0x10").
+  if (!/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(String(raw).trim())) {
+    throw new MalformedArtifact(`testcase time='${raw}' is not a number at ${label}`);
+  }
+  const seconds = Number(raw);
+  if (!Number.isFinite(seconds)) {
+    throw new MalformedArtifact(`testcase time='${raw}' is not a finite number at ${label}`);
+  }
+  return roundHalfToEven(seconds * 1000);
+}
+
+/** Python's rounding rule, so a `.5` boundary lands on the same integer. */
+function roundHalfToEven(value) {
+  const floor = Math.floor(value);
+  const remainder = value - floor;
+  if (remainder > 0.5) return floor + 1;
+  if (remainder < 0.5) return floor;
+  return floor % 2 === 0 ? floor : floor + 1;
+}
+
+/** Read and parse a JUnit XML file from disk. */
+export function parseJUnit(path) {
+  let source;
+  try {
+    source = fs.readFileSync(path, 'utf8');
+  } catch (error) {
+    throw new MalformedArtifact(`could not parse JUnit XML at ${path}: ${error.message}`);
+  }
+  return parseJUnitText(source, path);
+}

package/packages/engine/lib/analysis/network.mjs

@@ -0,0 +1,237 @@
+// Turning a HAR into the report's `network` block.
+//
+// ## Why this exists
+//
+// `qa-explore` claimed an API dimension and backed it with forty-eight lines of prose
+// telling the model what to look for. The model then counted requests by eye, decided
+// which were slow, and judged which were duplicates — three things a model is bad at
+// and a parser is perfect at. The pack's founding rule is that deterministic code owns
+// facts and the model owns explanation; for this dimension the model owned both.
+//
+// So the counting moves here. What comes out is the `network` block of the report
+// contract, ready to validate: totals, per-endpoint timings and sizes, and a flag on
+// each endpoint saying what is wrong with it. The model's job becomes what it is
+// actually good at — deciding whether a duplicate request *matters* to this product,
+// and writing the finding.
+//
+// ## Facts, and inferences that are labelled as such
+//
+// Two different kinds of claim come out of a HAR, and conflating them is how a report
+// starts asserting things it cannot support:
+//
+//   fact       count, status, duration, byte size, start offset, header presence.
+//              Read directly. If the HAR does not record it, the answer is null, never
+//              a plausible zero.
+//   inference  duplicate, polling, n+1, uncached. Derived by a stated rule, over a
+//              stated threshold, and every one is reported with the evidence that
+//              triggered it so a reader can disagree.
+//
+// Nothing here decides severity. A duplicated analytics beacon and a duplicated payment
+// request are the same shape in a HAR and wildly different findings, and only something
+// that understands the product can tell them apart.
+
+import { parseHar, parseHarData } from './har.mjs';
+
+/** Defaults, all overridable, all reported back in the output so a reader sees them. */
+export const DEFAULTS = Object.freeze({
+  slowMs: 1000,
+  // 512 KB. Below this a payload is rarely the thing worth reporting; above it, on a
+  // mobile connection, it is.
+  largeBytes: 512 * 1024,
+  // Three or more identical requests at a regular cadence reads as polling rather than
+  // as a burst of duplicates. Two is a double-submit; ten evenly spaced is a timer.
+  pollingMinimum: 3,
+  // Requests to the same path shape within this window look like one screen's worth of
+  // work, which is what makes an N+1 visible rather than just "a busy session".
+  nPlusOneWindowMs: 2000,
+  nPlusOneMinimum: 4,
+});
+
+/** The path portion of a URL, without query or fragment. */
+function pathOf(url) {
+  const text = String(url ?? '');
+  const withoutScheme = text.includes('://') ? text.slice(text.indexOf('://') + 3) : text;
+  const slash = withoutScheme.indexOf('/');
+  const rest = slash === -1 ? '/' : withoutScheme.slice(slash);
+  return rest.split('?')[0].split('#')[0] || '/';
+}
+
+/**
+ * A path with its identifier segments replaced, so `/users/41` and `/users/42` collapse.
+ *
+ * This is what makes an N+1 detectable. The rule is deliberately narrow — numeric
+ * segments, UUIDs, and long hex strings — because a looser one collapses genuinely
+ * different endpoints and reports an N+1 that is not there.
+ */
+export function templatePath(url) {
+  return pathOf(url)
+    .split('/')
+    .map((segment) => {
+      if (/^\d+$/.test(segment)) return ':id';
+      if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(segment)) return ':uuid';
+      if (/^[0-9a-f]{16,}$/i.test(segment)) return ':hash';
+      return segment;
+    })
+    .join('/');
+}
+
+/** Milliseconds from the first request, or null when the HAR has no timestamps. */
+function startOffsets(entries) {
+  const times = entries.map((entry) => {
+    const parsed = entry.startedAt ? Date.parse(entry.startedAt) : Number.NaN;
+    return Number.isFinite(parsed) ? parsed : null;
+  });
+  const known = times.filter((value) => value !== null);
+  if (known.length === 0) return times.map(() => null);
+  const origin = Math.min(...known);
+  return times.map((value) => (value === null ? null : value - origin));
+}
+
+/**
+ * Is this response cacheable and uncached?
+ *
+ * Only asked of static assets, and only when the server said nothing at all. A 200 for
+ * a script with neither `Cache-Control` nor `ETag` is a real finding; the same absence
+ * on a JSON API response usually is not, because that response is often meant to be
+ * fresh every time. Guessing on the second produces noise that trains a reader to
+ * ignore the whole section.
+ */
+function looksUncached(entry) {
+  if (entry.status !== 200) return false;
+  if (entry.cacheControl || entry.etag) return false;
+  const path = pathOf(entry.url).toLowerCase();
+  return /\.(js|mjs|css|png|jpe?g|gif|webp|avif|svg|woff2?|ttf|eot|ico)$/.test(path);
+}
+
+/** Evenly spaced repeats read as a timer rather than as a burst. */
+function looksLikePolling(offsets, minimum) {
+  const known = offsets.filter((value) => value !== null).sort((a, b) => a - b);
+  if (known.length < minimum) return false;
+  const gaps = known.slice(1).map((value, index) => value - known[index]);
+  if (gaps.length < 2) return false;
+  const mean = gaps.reduce((sum, gap) => sum + gap, 0) / gaps.length;
+  if (mean < 250) return false; // a burst, not a poll
+  // Within 35% of the mean gap, every time: a scheduler, not a user.
+  return gaps.every((gap) => Math.abs(gap - mean) <= mean * 0.35);
+}
+
+/**
+ * Build the contract's `network` block from parsed HAR entries.
+ *
+ * Exported separately from the file-reading path so a caller that already has entries —
+ * a browser adapter capturing them live, say — can use the same analysis without
+ * writing a HAR to disk first.
+ */
+export function analyzeEntries(entries, options = {}) {
+  const config = { ...DEFAULTS, ...options };
+  const offsets = startOffsets(entries);
+  const enriched = entries.map((entry, index) => ({ ...entry, startedMs: offsets[index] }));
+
+  // Group by what actually identifies a call: the method and the full URL. Two GETs of
+  // the same path with different query strings are different requests, and collapsing
+  // them would report a duplicate that is not one.
+  const groups = new Map();
+  for (const entry of enriched) {
+    const key = `${entry.method} ${entry.url}`;
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key).push(entry);
+  }
+
+  // N+1: many calls to one path *shape* inside a short window.
+  const byTemplate = new Map();
+  for (const entry of enriched) {
+    const key = `${entry.method} ${templatePath(entry.url)}`;
+    if (!byTemplate.has(key)) byTemplate.set(key, []);
+    byTemplate.get(key).push(entry);
+  }
+  const nPlusOne = new Set();
+  for (const [key, group] of byTemplate) {
+    if (group.length < config.nPlusOneMinimum) continue;
+    // Distinct URLs, or it is a duplicate rather than an N+1.
+    if (new Set(group.map((entry) => entry.url)).size < config.nPlusOneMinimum) continue;
+    const known = group.map((entry) => entry.startedMs).filter((value) => value !== null);
+    if (known.length > 1 && Math.max(...known) - Math.min(...known) > config.nPlusOneWindowMs) continue;
+    nPlusOne.add(key);
+  }
+
+  const endpoints = [];
+  for (const [, group] of groups) {
+    const first = group[0];
+    const durations = group.map((entry) => entry.durationMs).filter((value) => Number.isFinite(value));
+    const sizes = group.map((entry) => entry.bytes).filter((value) => value !== null);
+    const worstStatus = group.reduce(
+      (worst, entry) => (statusRank(entry.status) > statusRank(worst) ? entry.status : worst),
+      first.status,
+    );
+    const slowest = durations.length > 0 ? Math.max(...durations) : 0;
+    const bytes = sizes.length > 0 ? Math.max(...sizes) : null;
+    const templateKey = `${first.method} ${templatePath(first.url)}`;
+
+    // One flag per endpoint, worst first: a failing request that is also slow is
+    // reported as failing, because that is what the reader must act on.
+    let issue;
+    if (worstStatus === 0 || worstStatus >= 400) issue = 'failed';
+    else if (slowest >= config.slowMs) issue = 'slow';
+    else if (looksLikePolling(group.map((entry) => entry.startedMs), config.pollingMinimum)) issue = 'polling';
+    else if (group.length > 1) issue = 'duplicate';
+    else if (nPlusOne.has(templateKey)) issue = 'n-plus-one';
+    else if (bytes !== null && bytes >= config.largeBytes) issue = 'large-payload';
+    else if (looksUncached(first)) issue = 'uncached';
+
+    endpoints.push({
+      method: first.method,
+      url: first.url,
+      ...(Number.isFinite(worstStatus) ? { status: worstStatus } : {}),
+      durationMs: slowest,
+      ...(bytes !== null ? { bytes } : {}),
+      count: group.length,
+      ...(first.startedMs !== null ? { startedMs: first.startedMs } : {}),
+      ...(issue ? { issue } : {}),
+    });
+  }
+
+  // Worst first, then slowest: the order a reader wants without sorting anything.
+  endpoints.sort(
+    (a, b) => issueRank(a.issue) - issueRank(b.issue) || (b.durationMs ?? 0) - (a.durationMs ?? 0),
+  );
+
+  const totalBytes = enriched.reduce((sum, entry) => sum + (entry.bytes ?? 0), 0);
+  const anySizes = enriched.some((entry) => entry.bytes !== null);
+
+  return {
+    totalRequests: enriched.length,
+    failedRequests: enriched.filter((entry) => entry.status === 0 || entry.status >= 400).length,
+    slowRequests: enriched.filter((entry) => entry.durationMs >= config.slowMs).length,
+    // Requests beyond the first for each identical call — the count of *wasted* calls,
+    // not the count of endpoints that were repeated.
+    duplicateRequests: [...groups.values()].reduce((sum, group) => sum + (group.length - 1), 0),
+    ...(anySizes ? { totalBytes } : {}),
+    slowThresholdMs: config.slowMs,
+    endpoints,
+  };
+}
+
+function statusRank(status) {
+  if (status === 0) return 3;
+  if (status >= 500) return 2;
+  if (status >= 400) return 1;
+  return 0;
+}
+
+const ISSUE_ORDER = ['failed', 'slow', 'n-plus-one', 'duplicate', 'polling', 'large-payload', 'uncached'];
+function issueRank(issue) {
+  const index = ISSUE_ORDER.indexOf(issue);
+  return index === -1 ? ISSUE_ORDER.length : index;
+}
+
+/** Analyze a HAR file on disk. Returns the contract's `network` block. */
+export function analyzeHar(path, options = {}) {
+  const parsed = parseHar(path, { slowMs: options.slowMs ?? DEFAULTS.slowMs });
+  return analyzeEntries(parsed.entries, options);
+}
+
+/** Analyze already-parsed HAR JSON, for a caller holding the document in memory. */
+export function analyzeHarData(data, options = {}) {
+  const parsed = parseHarData(data, { slowMs: options.slowMs ?? DEFAULTS.slowMs });
+  return analyzeEntries(parsed.entries, options);
+}