qa-engineer 0.9.2 → 0.10.0

package/packages/engine/lib/analysis/evidence.mjs

@@ -0,0 +1,116 @@
+// The evidence and finding model shared by every analyzer.
+//
+// Every finding an analyzer produces carries the same structure, so downstream
+// skills (qa-debug, qa-report, qa-fix) consume one shape regardless of which
+// analyzer or framework produced it. Text fields are redacted at construction.
+//
+// Ported from qa_analysis/evidence.py. Python used dataclasses with a `to_dict()`;
+// here each factory returns the serialized shape directly, because that shape —
+// not the object — is what every consumer actually uses, and one representation
+// cannot drift from the other.
+
+import { redactText } from './redaction.mjs';
+
+export const EVIDENCE_TYPES = new Set([
+  'trace', 'har', 'junit', 'report', 'console', 'network', 'stdout',
+  'stderr', 'screenshot', 'video', 'log', 'file', 'diff',
+]);
+
+/** ISO 8601 UTC timestamp. Isolated so tests can substitute it. */
+export function utcNow() {
+  return new Date().toISOString();
+}
+
+/**
+ * One observation supporting a finding. Excerpts are redacted here, at
+ * construction, so no caller can forget to do it.
+ */
+export function evidence({ type, description, source, excerpt = '' }) {
+  if (!EVIDENCE_TYPES.has(type)) {
+    throw new Error(`unknown evidence type: ${type}`);
+  }
+  const redacted = redactText(excerpt);
+  const entry = { type, description, source };
+  if (redacted) entry.excerpt = redacted;
+  return entry;
+}
+
+/**
+ * A single diagnostic conclusion, traceable to a specific artifact.
+ *
+ * Carries everything the evidence model requires: the artifact and location it
+ * came from, when, why, the supporting evidence, a calibrated confidence, the
+ * affected tests, related artifacts, and recommended actions.
+ */
+export function finding({
+  classification,
+  reason,
+  artifact: artifactPath,
+  location,
+  confidence = null,
+  timestamp = undefined,
+  evidence: items = [],
+  affectedTests = [],
+  relatedArtifacts = [],
+  recommendations = [],
+}) {
+  const result = {
+    classification,
+    reason,
+    artifact: artifactPath,
+    location,
+    timestamp: timestamp ?? utcNow(),
+    evidence: items,
+    affectedTests,
+    relatedArtifacts,
+    recommendations,
+  };
+  if (confidence !== null && confidence !== undefined) result.confidence = confidence;
+  return result;
+}
+
+/**
+ * The envelope an analyzer emits: findings plus the artifacts it examined.
+ *
+ * A downstream skill wraps this in its own output contract; on its own it is the
+ * deterministic, machine-readable result of one analysis.
+ */
+export function analyzerOutput({
+  analyzer,
+  findings = [],
+  artifacts = [],
+  warnings = [],
+  generatedAt = undefined,
+}) {
+  return {
+    analyzer,
+    generatedAt: generatedAt ?? utcNow(),
+    findings,
+    artifacts,
+    warnings,
+  };
+}
+
+/** A discovered artifact, in the common model shared with the execution engine. */
+export function artifact({
+  type,
+  location,
+  framework = 'unknown',
+  ownership = 'qa-analysis',
+  timestamp = undefined,
+  mediaType = '',
+  testRef = '',
+  present = true,
+}) {
+  const entry = {
+    type,
+    location,
+    framework,
+    timestamp: timestamp ?? utcNow(),
+    ownership,
+    present,
+  };
+  if (mediaType) entry.mediaType = mediaType;
+  if (testRef) entry.testRef = testRef;
+  return entry;
+}

package/packages/engine/lib/analysis/har.mjs

@@ -0,0 +1,90 @@
+// HAR (HTTP Archive) parser.
+//
+// Framework-agnostic: a HAR is a standard JSON format, whoever produced it.
+// Extracts request outcomes, flags failures and slow calls, and redacts headers
+// and credentialed URLs before anything is exposed. A malformed HAR raises.
+
+import fs from 'node:fs';
+
+import { redactText, redactHeaders } from './redaction.mjs';
+import { MalformedArtifact } from './junit.mjs';
+
+/**
+ * Parse HAR JSON into a redacted network summary.
+ *
+ * `{entries, failures, slow, redacted: true}`, where each entry is
+ * `{method, url, status, durationMs, requestHeaders, responseHeaders}`. Failures
+ * are entries with status >= 400 or status 0 (no response at all).
+ */
+export function parseHarData(data, { slowMs = 1000, label = '<input>' } = {}) {
+  const rawEntries = data?.log?.entries;
+  if (!Array.isArray(rawEntries)) {
+    throw new MalformedArtifact(`not a HAR document at ${label}`);
+  }
+
+  const entries = rawEntries.map((item) => {
+    if (item === null || typeof item !== 'object' || Array.isArray(item)) {
+      throw new MalformedArtifact(`HAR log.entries contains a non-object at ${label}`);
+    }
+    const request = item.request ?? {};
+    const response = item.response ?? {};
+    return {
+      method: request.method ?? '',
+      // Redaction strips any credentials embedded in the URL.
+      url: redactText(request.url ?? ''),
+      status: toInt(response.status),
+      durationMs: toMillis(item?.time, label),
+      requestHeaders: redactHeaders(request.headers ?? []),
+      responseHeaders: redactHeaders(response.headers ?? []),
+    };
+  });
+
+  return {
+    entries,
+    failures: entries.filter((e) => e.status === 0 || e.status >= 400),
+    slow: entries.filter((e) => e.durationMs >= slowMs),
+    redacted: true,
+  };
+}
+
+/** Read and parse a HAR file from disk. */
+export function parseHar(path, { slowMs = 1000 } = {}) {
+  let data;
+  try {
+    data = JSON.parse(fs.readFileSync(path, 'utf8'));
+  } catch (error) {
+    throw new MalformedArtifact(`could not parse HAR at ${path}: ${error.message}`);
+  }
+  return parseHarData(data, { slowMs, label: path });
+}
+
+/** A status that is absent or unreadable is 0 — "no response", which is a failure. */
+function toInt(value) {
+  const number = Number(value);
+  return Number.isFinite(number) ? Math.trunc(number) : 0;
+}
+
+/**
+ * HAR `time` is milliseconds already, rounded to a whole number.
+ *
+ * A value that is present but not a number raises rather than becoming 0: the
+ * same rule as the JUnit parser, and for the same reason — a fabricated duration
+ * makes a malformed document look like a clean measurement.
+ */
+function toMillis(raw, label) {
+  if (raw === undefined || raw === null || raw === '') return 0;
+  const number = Number(raw);
+  if (!Number.isFinite(number)) {
+    throw new MalformedArtifact(`entry time='${raw}' is not a number at ${label}`);
+  }
+  return roundHalfToEven(number);
+}
+
+/** Python's rounding rule, so a `.5` boundary lands on the same integer. */
+function roundHalfToEven(value) {
+  const floor = Math.floor(value);
+  const remainder = value - floor;
+  if (remainder > 0.5) return floor + 1;
+  if (remainder < 0.5) return floor;
+  return floor % 2 === 0 ? floor : floor + 1;
+}

package/packages/engine/lib/analysis/junit.mjs

@@ -0,0 +1,126 @@
+// JUnit XML parser.
+//
+// Framework-agnostic: Playwright, Selenium, Cypress, WebdriverIO, and most unit
+// runners emit JUnit XML, so this one parser normalizes them all into the pack's
+// per-test result shape. This is the concrete proof that different frameworks
+// share one contract — only where the file lives differs, not how it is read.
+//
+// Parses deterministically; a malformed document raises rather than guessing.
+
+import fs from 'node:fs';
+
+import { parseXml, find, findAll, XmlError } from './xml.mjs';
+import { redactText } from './redaction.mjs';
+
+/** Raised when an artifact cannot be parsed. Never swallowed into a guess. */
+export class MalformedArtifact extends Error {}
+
+/**
+ * Parse JUnit XML text into a normalized result.
+ *
+ * `{tests: {total, passed, failed, skipped}, executed: [...]}`, where each
+ * executed entry is `{title, file, status, durationMs, message?}`.
+ */
+export function parseJUnitText(source, label = '<input>') {
+  let root;
+  try {
+    root = parseXml(source);
+  } catch (error) {
+    if (error instanceof XmlError) {
+      throw new MalformedArtifact(`could not parse JUnit XML at ${label}: ${error.message}`);
+    }
+    throw error;
+  }
+
+  // Accept either a <testsuites> root or a single <testsuite> root.
+  let suites;
+  if (root.tag === 'testsuites') suites = findAll(root, 'testsuite');
+  else if (root.tag === 'testsuite') suites = [root];
+  else throw new MalformedArtifact(`not a JUnit document (root <${root.tag}>) at ${label}`);
+
+  const executed = [];
+  for (const suite of suites) {
+    for (const testcase of findAll(suite, 'testcase')) {
+      const failure = find(testcase, 'failure');
+      const error = find(testcase, 'error');
+      const skipped = find(testcase, 'skipped');
+
+      let status;
+      let message = '';
+      if (failure !== null || error !== null) {
+        status = 'failed';
+        const node = failure !== null ? failure : error;
+        message = redactText((node.attrs.message || node.text || '').trim());
+      } else if (skipped !== null) {
+        status = 'skipped';
+        message = redactText((skipped.attrs.message || '').trim());
+      } else {
+        status = 'passed';
+      }
+
+      const entry = {
+        title: testcase.attrs.name ?? '',
+        file: testcase.attrs.classname ?? '',
+        status,
+        durationMs: durationMs(testcase.attrs.time, label),
+      };
+      if (message) entry.message = message;
+      executed.push(entry);
+    }
+  }
+
+  return {
+    tests: {
+      total: executed.length,
+      passed: executed.filter((e) => e.status === 'passed').length,
+      failed: executed.filter((e) => e.status === 'failed').length,
+      skipped: executed.filter((e) => e.status === 'skipped').length,
+    },
+    executed,
+  };
+}
+
+/**
+ * `time` in seconds becomes whole milliseconds.
+ *
+ * Two subtleties, both pinned by the parity corpus:
+ *
+ * 1. An absent or empty attribute is zero; a value that is present but not a
+ *    finite number raises. Guessing zero there would turn a malformed document
+ *    into a plausible result, which is the one thing this parser must not do.
+ * 2. Python's `round()` rounds half to *even* and JavaScript's `Math.round`
+ *    rounds half *up*, so `time="0.0005"` gives 0 in Python and 1 here unless
+ *    the tie is broken the same way. Runners do emit such values.
+ */
+function durationMs(raw, label) {
+  if (raw === undefined || raw === null || raw === '') return 0;
+  // Reject what parseFloat would happily truncate ("1.2.3", "5s", "0x10").
+  if (!/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(String(raw).trim())) {
+    throw new MalformedArtifact(`testcase time='${raw}' is not a number at ${label}`);
+  }
+  const seconds = Number(raw);
+  if (!Number.isFinite(seconds)) {
+    throw new MalformedArtifact(`testcase time='${raw}' is not a finite number at ${label}`);
+  }
+  return roundHalfToEven(seconds * 1000);
+}
+
+/** Python's rounding rule, so a `.5` boundary lands on the same integer. */
+function roundHalfToEven(value) {
+  const floor = Math.floor(value);
+  const remainder = value - floor;
+  if (remainder > 0.5) return floor + 1;
+  if (remainder < 0.5) return floor;
+  return floor % 2 === 0 ? floor : floor + 1;
+}
+
+/** Read and parse a JUnit XML file from disk. */
+export function parseJUnit(path) {
+  let source;
+  try {
+    source = fs.readFileSync(path, 'utf8');
+  } catch (error) {
+    throw new MalformedArtifact(`could not parse JUnit XML at ${path}: ${error.message}`);
+  }
+  return parseJUnitText(source, path);
+}

package/packages/engine/lib/analysis/redaction.mjs

@@ -0,0 +1,127 @@
+// Credential, secret, token, and PII redaction.
+//
+// Every analyzer runs artifact text through redaction before it appears in a
+// finding, a report, or stdout. Redaction happens as evidence is captured, not
+// after — a secret must never reach a model's context or a log.
+//
+// Ported from qa_analysis/redaction.py, rule for rule and in the same order, and
+// held to identical output by scripts/check-engine-parity.mjs. Python's inline
+// `(?i)` / `(?im)` flags become JavaScript flag letters; every rule carries `g`
+// because Python's `re.sub` replaces all occurrences.
+
+export class RedactionError extends Error {}
+
+// Ordered. Order matters: match structured, high-signal secrets (JWTs, provider
+// keys) before generic fallbacks.
+const RULES = [
+  { name: 'jwt', pattern: /eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}/g },
+  { name: 'aws-access-key', pattern: /AKIA[0-9A-Z]{16}/g },
+  { name: 'github-token', pattern: /gh[pousr]_[A-Za-z0-9]{20,}/g },
+  { name: 'slack-token', pattern: /xox[baprs]-[A-Za-z0-9-]{10,}/g },
+  { name: 'openai-key', pattern: /sk-[A-Za-z0-9]{20,}/g },
+  { name: 'bearer', pattern: /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi },
+  // Sensitive header lines: keep the header name, mask the value.
+  //
+  // The value is `[^\r\n]+` rather than `.+$`, and the indent is `[ \t]*` rather
+  // than `\s*`, because `$` and `\s` do not mean the same thing in Python and
+  // JavaScript on CRLF text: Python's `.+$` swallowed the carriage return and
+  // rewrote the line ending to LF. Anchoring on a negated class removes the
+  // difference and stops the redactor corrupting a document it only meant to mask.
+  {
+    name: 'auth-header',
+    pattern: /^([ \t]*(?:authorization|proxy-authorization)[ \t]*[:=][ \t]*)[^\r\n]+/gim,
+    replacement: '$1[REDACTED:auth-header]',
+  },
+  {
+    name: 'cookie-header',
+    pattern: /^([ \t]*(?:set-cookie|cookie)[ \t]*[:=][ \t]*)[^\r\n]+/gim,
+    replacement: '$1[REDACTED:cookie-header]',
+  },
+  // Secret-like assignments: key=value / "key": "value".
+  {
+    name: 'assigned-secret',
+    pattern: /(\b(?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|client[_-]?secret)\b\s*[:=]\s*["']?)([^\s"'&]{4,})/gi,
+    replacement: '$1[REDACTED:secret]',
+  },
+  // Credentials in URLs and query strings.
+  {
+    name: 'url-credential',
+    pattern: /(:\/\/[^:/@\s]+:)([^@/\s]+)(@)/gi,
+    replacement: '$1[REDACTED:credential]$3',
+  },
+  {
+    name: 'query-secret',
+    pattern: /([?&](?:password|passwd|pwd|token|secret|api[_-]?key|access[_-]?key)=)([^&\s#]+)/gi,
+    replacement: '$1[REDACTED:secret]',
+  },
+  // PII: email addresses.
+  { name: 'email', pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g },
+];
+
+const SENSITIVE_HEADERS = new Set([
+  'authorization', 'proxy-authorization', 'cookie', 'set-cookie',
+  'x-api-key', 'x-auth-token', 'api-key', 'x-csrf-token',
+]);
+
+/** Return text with every recognized secret or PII value masked. */
+export function redactText(text) {
+  if (!text) return text;
+  let result = String(text);
+  for (const rule of RULES) {
+    // A fresh regex per call: `g` regexes carry lastIndex, and a module-level
+    // pattern reused across calls would skip matches unpredictably.
+    const pattern = new RegExp(rule.pattern.source, rule.pattern.flags);
+    result = result.replace(pattern, rule.replacement ?? `[REDACTED:${rule.name}]`);
+  }
+  return result;
+}
+
+/**
+ * Return `[{type, start, end}]` for secrets found — never the values.
+ *
+ * Used to decide whether an artifact is safe to expose without surfacing the
+ * secret itself.
+ */
+export function detectSecrets(text) {
+  const findings = [];
+  if (!text) return findings;
+  const source = String(text);
+  for (const rule of RULES) {
+    const pattern = new RegExp(rule.pattern.source, rule.pattern.flags);
+    for (const match of source.matchAll(pattern)) {
+      findings.push({ type: rule.name, start: match.index, end: match.index + match[0].length });
+    }
+  }
+  // Stable sort by position, so two secrets at the same offset keep rule order.
+  findings.sort((a, b) => a.start - b.start);
+  return findings;
+}
+
+/**
+ * Mask the values of sensitive HTTP headers.
+ *
+ * Accepts an object, or an array of `{name, value}` entries (the HAR shape), and
+ * returns the same shape with sensitive values replaced.
+ */
+export function redactHeaders(headers) {
+  const mask = (name, value) =>
+    SENSITIVE_HEADERS.has(String(name).trim().toLowerCase())
+      ? '[REDACTED:header]'
+      : redactText(value);
+
+  if (Array.isArray(headers)) {
+    return headers.map((entry) =>
+      entry && typeof entry === 'object' && 'name' in entry
+        ? { ...entry, value: mask(String(entry.name), String(entry.value ?? '')) }
+        : entry,
+    );
+  }
+  if (headers && typeof headers === 'object') {
+    return Object.fromEntries(
+      Object.entries(headers).map(([key, value]) => [key, mask(key, String(value))]),
+    );
+  }
+  return headers;
+}
+
+export const RULE_NAMES = RULES.map((rule) => rule.name);