qa-engineer 0.9.2 → 0.11.0

package/packages/engine/lib/diagnostics/timeline.mjs

@@ -0,0 +1,101 @@
+// Deterministic timeline reconstruction.
+//
+// Reconstructs the ordered sequence of a run — start, browser launch, navigation,
+// requests and responses, console errors, assertions, failure, cleanup — from the
+// execution result and the analysis findings. It records only stages for which there
+// is evidence; it never invents an event to fill the shape. Reusable by qa-debug and
+// qa-report.
+
+// Canonical phase order, used to sort events that share (or lack) a timestamp.
+export const PHASE_ORDER = [
+  'execution-start', 'browser-launch', 'navigation', 'request', 'response',
+  'console-error', 'assertion', 'failure', 'cleanup', 'execution-finish',
+];
+const PHASE_INDEX = new Map(PHASE_ORDER.map((phase, index) => [phase, index]));
+
+// Evidence type -> the timeline phase it contributes.
+const EVIDENCE_PHASE = {
+  network: 'response',
+  console: 'console-error',
+  trace: 'navigation',
+  junit: 'assertion',
+  report: 'assertion',
+};
+
+/**
+ * Build an ordered timeline: `[{order, phase, detail, source, timestamp?}]`.
+ *
+ * Deterministic: same inputs, same timeline. Only evidenced stages appear.
+ */
+export function buildTimeline(executionResult, findings = null) {
+  const events = [];
+  const execution = (executionResult ?? {}).execution ?? {};
+
+  const started = execution.startedAt;
+  if (started || (executionResult !== null && executionResult !== undefined && truthy(executionResult))) {
+    events.push(event('execution-start', 'Run started', 'execution-result', started ?? null));
+  }
+
+  // Contribute a stage per evidence entry we actually have.
+  for (const finding of findings ?? []) {
+    for (const item of finding.evidence ?? []) {
+      const phase = EVIDENCE_PHASE[item.type];
+      if (phase) {
+        events.push(event(phase, item.description ?? '', item.source ?? 'analysis', null));
+      }
+    }
+  }
+
+  // A failure stage per failed test recorded in the result.
+  for (const test of (executionResult ?? {}).executed ?? []) {
+    if (test.status === 'failed') {
+      events.push(event('failure', `Test failed: ${test.title ?? ''}`,
+        test.file ?? 'execution-result', null));
+    }
+  }
+
+  const finished = execution.finishedAt;
+  if (finished) {
+    events.push(event('cleanup', 'Run cleaned up', 'execution-result', null));
+    events.push(event('execution-finish', 'Run finished', 'execution-result', finished));
+  }
+
+  // Deterministic order: by canonical phase first (it encodes the logical
+  // sequence), then by timestamp within a phase, then by insertion order. Phase
+  // order is primary because per-event wall-clock times are often unavailable.
+  const decorated = events.map((entry, seq) => ({ entry, seq }));
+  decorated.sort((a, b) => {
+    const phaseDelta = (PHASE_INDEX.get(a.entry.phase) ?? 99) - (PHASE_INDEX.get(b.entry.phase) ?? 99);
+    if (phaseDelta !== 0) return phaseDelta;
+    const left = a.entry.timestamp || '';
+    const right = b.entry.timestamp || '';
+    if (left !== right) return left < right ? -1 : 1;
+    return a.seq - b.seq;
+  });
+
+  // Key order matters: these events are serialized and compared byte-for-byte
+  // against the Python implementation, which builds phase/detail/source/timestamp
+  // and then appends `order`. Assigning `order` before `timestamp` would produce
+  // the same data in a different JSON shape.
+  return decorated.map(({ entry }, order) => {
+    const result = { phase: entry.phase, detail: entry.detail, source: entry.source };
+    if (entry.timestamp !== null && entry.timestamp !== undefined) result.timestamp = entry.timestamp;
+    result.order = order;
+    return result;
+  });
+}
+
+function event(phase, detail, source, timestamp) {
+  return { phase, detail, source, timestamp };
+}
+
+/**
+ * Python's `if started or execution_result:` treats an empty dict as falsey, so a
+ * bare `{}` produces no start event while `{"tests": {...}}` does. JavaScript
+ * considers every object truthy, so the emptiness has to be asked about directly.
+ */
+function truthy(value) {
+  if (Array.isArray(value)) return value.length > 0;
+  if (value && typeof value === 'object') return Object.keys(value).length > 0;
+  return Boolean(value);
+}

package/packages/engine/lib/frameworks/junit-frameworks.mjs

@@ -0,0 +1,48 @@
+// The JUnit-XML frameworks: Selenium, Cypress, WebdriverIO.
+//
+// All three emit JUnit XML, so their normalization *is* the framework-agnostic JUnit
+// parser — each adapter only points at the right artifact and tags provenance. That
+// thinness is the proof of the pack's multi-framework claim: the second, third and
+// fourth frameworks cost a few lines each, because the contract and the parsing are
+// shared and only the artifact location differs (ADR-0013).
+//
+// The three lived in separate Python modules that were identical but for a name and
+// a glob list. Here they are one table, because three copies of the same four lines
+// is three places for them to drift.
+
+import { parseJUnit } from '../analysis/junit.mjs';
+import { classify } from '../analysis/taxonomy.mjs';
+
+/** framework -> where that framework's runners conventionally write JUnit XML. */
+export const RESULT_GLOBS = {
+  selenium: ['**/target/surefire-reports/*.xml', '**/test-results/*.xml', '**/junit*.xml'],
+  cypress: ['**/results/*.xml', '**/cypress/results/*.xml', '**/junit*.xml'],
+  webdriverio: ['**/junit*.xml', '**/results/*.xml', '**/test-results/*.xml'],
+};
+
+export const FRAMEWORKS = Object.keys(RESULT_GLOBS);
+
+/**
+ * Normalize a run into the shared result shape.
+ *
+ * Identical output for every framework — that is the point. The `framework`
+ * argument selects nothing in the parsing; it exists so a caller can be explicit
+ * about provenance and so an unknown name is refused rather than silently treated
+ * as one of the three.
+ */
+export function normalize(framework, junitPath) {
+  if (!Object.prototype.hasOwnProperty.call(RESULT_GLOBS, framework)) {
+    throw new Error(`unknown JUnit framework: ${framework} (known: ${FRAMEWORKS.join(', ')})`);
+  }
+  return parseJUnit(junitPath);
+}
+
+/**
+ * Classify a failure using the shared taxonomy.
+ *
+ * There is no per-framework classifier, because failure classes are
+ * framework-agnostic: a missing element is a locator failure whoever reported it.
+ */
+export function classifyFailure(message, httpStatus = null) {
+  return classify(message, httpStatus);
+}

package/packages/engine/lib/frameworks/playwright.mjs

@@ -0,0 +1,158 @@
+// Playwright-specific analyzers: trace and JSON report.
+//
+// This is the Playwright *adapter* for the analysis platform. It depends on the
+// framework-agnostic analysis core for the evidence model, redaction, and the
+// failure taxonomy; it adds only what is Playwright-specific — the shape of a
+// trace.zip and of Playwright's JSON reporter. Everything it emits is in the shared
+// normalized shape, so nothing downstream knows it was Playwright.
+//
+// Framework knowledge stays inside the adapter: the core never grows a
+// `--framework` flag (ADR-0013).
+
+import fs from 'node:fs';
+
+import { MalformedArtifact } from '../analysis/junit.mjs';
+import { redactText } from '../analysis/redaction.mjs';
+import * as taxonomy from '../analysis/taxonomy.mjs';
+import { openZip, isZip, ZipError } from '../analysis/zip.mjs';
+
+export const FRAMEWORK = 'playwright';
+
+/**
+ * Parse Playwright's JSON reporter output into the normalized result shape.
+ *
+ * Mirrors the JUnit parser's output (`{tests, executed}`) so a Playwright run
+ * normalizes identically whether it emitted JSON or JUnit.
+ */
+export function parseReport(path) {
+  let data;
+  try {
+    data = JSON.parse(fs.readFileSync(path, 'utf8'));
+  } catch (error) {
+    throw new MalformedArtifact(`could not parse Playwright report at ${path}: ${error.message}`);
+  }
+
+  const executed = [];
+  for (const suite of data.suites ?? []) walkSuite(suite, executed);
+
+  return {
+    tests: {
+      total: executed.length,
+      passed: executed.filter((e) => e.status === 'passed').length,
+      failed: executed.filter((e) => e.status === 'failed').length,
+      skipped: executed.filter((e) => e.status === 'skipped').length,
+    },
+    executed,
+  };
+}
+
+function walkSuite(suite, executed) {
+  for (const spec of suite.specs ?? []) {
+    for (const test of spec.tests ?? []) {
+      const results = test.results ?? [];
+      const status = statusOf(results);
+      let message = '';
+      if (status === 'failed' && results.length > 0) {
+        const errors = results[results.length - 1].errors ?? [];
+        if (errors.length > 0) {
+          message = redactText((errors[0].message ?? '').trim());
+        }
+      }
+      const entry = {
+        title: spec.title ?? '',
+        file: suite.file ?? '',
+        status,
+        durationMs: results.length > 0 ? (results[results.length - 1].duration ?? 0) : 0,
+        retries: Math.max(0, results.length - 1),
+      };
+      if (message) entry.message = message;
+      executed.push(entry);
+    }
+  }
+  for (const child of suite.suites ?? []) walkSuite(child, executed);
+}
+
+function statusOf(results) {
+  if (results.length === 0) return 'skipped';
+  const final = results[results.length - 1].status;
+  if (final === 'passed' || final === 'expected') return results.length > 1 ? 'flaky' : 'passed';
+  if (final === 'skipped') return 'skipped';
+  return 'failed';
+}
+
+/**
+ * Extract a deterministic summary from a Playwright trace.zip.
+ *
+ * A trace is a zip of newline-delimited JSON event files. This lists the actions,
+ * surfaces the last error, and counts console and network events — enough for a
+ * diagnostic skill to reason over, without decoding the full binary.
+ */
+export function analyzeTrace(path) {
+  let archive;
+  try {
+    const buffer = fs.readFileSync(path);
+    if (!isZip(buffer)) throw new MalformedArtifact(`not a valid trace zip: ${path}`);
+    archive = openZip(path);
+  } catch (error) {
+    if (error instanceof MalformedArtifact) throw error;
+    throw new MalformedArtifact(`not a valid trace zip: ${path}`);
+  }
+
+  const actions = [];
+  const errors = [];
+  let console = 0;
+  let network = 0;
+
+  try {
+    const named = archive.entries.filter(
+      (entry) => entry.name.endsWith('.trace') || entry.name.endsWith('.jsonl') || entry.name.includes('trace'),
+    );
+    for (const entry of named.length > 0 ? named : archive.entries) {
+      const text = archive.read(entry).toString('utf8');
+      for (const line of text.split('\n')) {
+        const raw = line.trim();
+        if (!raw || !raw.startsWith('{')) continue;
+        let event;
+        try {
+          event = JSON.parse(raw);
+        } catch {
+          continue;
+        }
+        const kind = event.type ?? event.kind;
+        if (kind === 'action' || kind === 'before') {
+          actions.push(redactText(String(event.apiName ?? event.method ?? '')));
+        } else if (kind === 'console') {
+          console += 1;
+        } else if (kind === 'resource' || kind === 'network' || kind === 'http') {
+          network += 1;
+        }
+        if (event.error || kind === 'error') {
+          const message = event.error && typeof event.error === 'object'
+            ? event.error.message
+            : event.message;
+          if (message) errors.push(redactText(String(message)));
+        }
+      }
+    }
+  } catch (error) {
+    if (error instanceof ZipError) {
+      throw new MalformedArtifact(`could not read trace ${path}: ${error.message}`);
+    }
+    throw error;
+  }
+
+  const lastError = errors.length > 0 ? errors[errors.length - 1] : '';
+  const classified = lastError
+    ? taxonomy.classify(lastError)
+    : { classification: taxonomy.UNKNOWN, confidence: 0.2, reason: 'No error found in trace.' };
+
+  return {
+    actions: actions.filter(Boolean),
+    consoleEvents: console,
+    networkEvents: network,
+    errors,
+    classification: classified.classification,
+    confidence: classified.confidence,
+    reason: classified.reason,
+  };
+}