plum-e2e 2.8.5 → 2.9.0

package/backend/_scaffold/utils/browser.ts

@@ -15,16 +15,71 @@
  * along with Plum. If not, see https://www.gnu.org/licenses/.
  */
 
+// Wires up Plum's session recording — removing or reordering code here can silently break report replay.
+
 import { chromium, firefox, webkit, Browser, BrowserContext, Page } from 'playwright';
 import * as fs from 'fs';
 import * as path from 'path';
+import * as zlib from 'zlib';
+
+// Must match the mime type Plum's server expects — do not change.
+const RRWEB_MIME_TYPE = 'application/x-plum-rrweb+json';
+// Always attached, even for a scenario with no recorded events, so the
+// worker that ran it is still recoverable for grouping.
+const WORKER_META_MIME_TYPE = 'application/x-plum-worker+json';
+// @rrweb/record's package.json only exports its main entry ("."), so a deep
+// require.resolve() of the UMD bundle is blocked by Node's exports map — resolve
+// the (exported) main entry instead and locate the sibling file on disk.
+const RECORD_BUNDLE_PATH = path.join(
+	path.dirname(require.resolve('@rrweb/record')),
+	'record.umd.min.cjs'
+);
+
+interface TabRecording {
+	tabId: string;
+	tabIndex: number;
+	events: unknown[];
+	openedAt: number;
+	closedAt: number | null;
+	liveFlushedCount: number;
+}
 
 let _browser: Browser;
 let _context: BrowserContext;
 let _page: Page;
-let _ssCounter = 0;
+let _liveRRwebCounter = 0;
+let _liveRRwebTimer: ReturnType<typeof setInterval> | null = null;
+let _tabs: Map<Page, TabRecording> = new Map();
+let _tabCounter = 0;
+let _workerId = 1;
 
 export const page = (): Page => _page;
+export const context = (): BrowserContext => _context;
+
+function tabIdForIndex(index: number): string {
+	return index === 0 ? 'main' : `tab-${index + 1}`;
+}
+
+// A static page (nothing left to interact with) can go a long time between
+// rrweb events, or emit none at all after its initial load — its own event
+// timestamps are a poor proxy for how long it stayed relevant. Real
+// open/close times let the replay UI line multiple tabs up on one timeline
+// without guessing from event gaps.
+function attachRecorder(pg: Page): void {
+	const tabIndex = _tabCounter++;
+	const recording: TabRecording = {
+		tabId: tabIdForIndex(tabIndex),
+		tabIndex,
+		events: [],
+		openedAt: Date.now(),
+		closedAt: null,
+		liveFlushedCount: 0
+	};
+	_tabs.set(pg, recording);
+	pg.on('close', () => {
+		recording.closedAt = Date.now();
+	});
+}
 
 export async function setup(): Promise<void> {
 	const isHeadless = process.env.IS_HEADLESS?.toLowerCase() !== 'false';
@@ -33,50 +88,149 @@ export async function setup(): Promise<void> {
 		browserName === 'firefox' ? firefox : browserName === 'webkit' ? webkit : chromium;
 	_browser = await browserType.launch({ headless: isHeadless });
 	_context = await _browser.newContext();
+
+	_tabs = new Map();
+	_tabCounter = 0;
+	// Cucumber forks one OS process per --parallel worker and injects this env
+	// var into each — 0-indexed, so display/report as 1-based like the rest of
+	// the worker-count UI.
+	const parsedWorkerId = parseInt(process.env.CUCUMBER_WORKER_ID ?? '', 10);
+	_workerId = Number.isFinite(parsedWorkerId) ? parsedWorkerId + 1 : 1;
+
+	// Context-level exposeBinding/addInitScript apply to every page in the
+	// context automatically — current and future (popups, target=_blank tabs) —
+	// so recording setup never races a new tab's first navigation.
+	await _context.exposeBinding('__plumEmitRRwebEvent', (source, eventJson: string) => {
+		const recording = source.page && _tabs.get(source.page);
+		if (!recording) return;
+		try {
+			recording.events.push(JSON.parse(eventJson));
+		} catch {
+			// malformed event — drop it, recording is best-effort
+		}
+	});
+	await _context.addInitScript({ path: RECORD_BUNDLE_PATH });
+	await _context.addInitScript(() => {
+		// addInitScript runs in every frame, including hidden ad/tracking iframes.
+		// Recordings are tracked per-Page, so an unguarded sub-frame session would
+		// corrupt the tab's event stream with bogus 0x0 "about:blank" entries.
+		// @ts-ignore
+		if (window.self !== window.top) return;
+		// @ts-ignore
+		if (window.rrwebRecord) {
+			// @ts-ignore
+			window.rrwebRecord.record({
+				emit: (event: unknown) => {
+					// @ts-ignore — exposed by BrowserContext.exposeBinding above
+					window.__plumEmitRRwebEvent(JSON.stringify(event));
+				}
+			});
+		}
+	});
+
+	_context.on('page', attachRecorder);
 	_page = await _context.newPage();
+
+	// Only when someone's actually watching live — a scheduled/background run
+	// with no viewer shouldn't pay for this.
+	if (process.env.PLUM_SS_DIR) {
+		_liveRRwebTimer = setInterval(flushLiveRRwebEvents, 500);
+	}
 }
 
-export async function screenshotStep(
-	attach: (data: Buffer, mime: string) => Promise<void>
-): Promise<void> {
+// Sends only what's newly arrived since the last tick, per tab, so the live
+// viewer gets a steady trickle instead of the full buffer growing unbounded.
+function flushLiveRRwebEvents(): void {
+	const ssDir = process.env.PLUM_SS_DIR;
+	if (!ssDir) return;
+	for (const recording of _tabs.values()) {
+		const newEvents = recording.events.slice(recording.liveFlushedCount);
+		if (newEvents.length === 0) continue;
+		recording.liveFlushedCount = recording.events.length;
+		try {
+			const seq = `${String(Date.now()).padStart(16, '0')}-${String(++_liveRRwebCounter).padStart(4, '0')}`;
+			fs.writeFileSync(
+				path.join(ssDir, `${seq}.rrweb.json`),
+				JSON.stringify({
+					workerId: _workerId,
+					tabId: recording.tabId,
+					tabIndex: recording.tabIndex,
+					events: newEvents
+				})
+			);
+		} catch {
+			// best-effort — live streaming shouldn't affect the recording itself
+		}
+	}
+}
+
+/**
+ * Injects a labeled rrweb custom event at the current recording timestamp so
+ * the replay UI can show which step was running at any point in the timeline.
+ */
+export async function markStepStart(stepName: string): Promise<void> {
 	if (!_page) return;
 	try {
-		const screenshot = await _page.screenshot({ type: 'png' });
-		await attach(screenshot, 'image/png');
+		await _page.evaluate((name) => {
+			// @ts-ignore — rrwebRecord is injected by the record.umd.min.cjs bundle
+			if (window.rrwebRecord?.record?.addCustomEvent) {
+				// @ts-ignore
+				window.rrwebRecord.record.addCustomEvent('step', { name });
+			}
+		}, stepName);
 	} catch {
-		// page may be navigating or already closed
+		// best-effort — a missing marker just means the replay UI won't show a
+		// step label at that point, it doesn't affect the recording itself
 	}
 }
 
-export async function streamLiveScreenshot(stepName: string): Promise<void> {
-	const ssDir = process.env.PLUM_SS_DIR;
-	if (!ssDir || !_page) return;
+/**
+ * Flushes every tab's buffered rrweb events (one per opened tab/popup) as a
+ * gzip-compressed Cucumber attachment, tagged with the mime type Plum's
+ * server looks for.
+ */
+export async function flushRecordings(
+	attach: (data: Buffer, mime: string) => Promise<void>
+): Promise<void> {
+	if (_liveRRwebTimer) {
+		clearInterval(_liveRRwebTimer);
+		_liveRRwebTimer = null;
+	}
+	// One last live flush so the stream doesn't miss whatever happened between
+	// the final tick and scenario end.
+	flushLiveRRwebEvents();
+
 	try {
-		const seq = `${String(Date.now()).padStart(16, '0')}-${String(++_ssCounter).padStart(4, '0')}`;
-		const screenshot = await _page.screenshot({ type: 'jpeg', quality: 70 });
-		fs.writeFileSync(
-			path.join(ssDir, `${seq}.ss.json`),
-			JSON.stringify({ stepName, data: screenshot.toString('base64') })
+		await attach(
+			Buffer.from(JSON.stringify({ workerId: _workerId }), 'utf8'),
+			WORKER_META_MIME_TYPE
 		);
 	} catch {
-		// ignore — live streaming is best-effort
+		// best-effort — a missing worker marker just falls back to workerId 1
 	}
-}
 
-export async function teardown(
-	attach: (data: Buffer, mime: string) => Promise<void>,
-	failed: boolean
-): Promise<void> {
-	if (failed && _page) {
-		const screenshotDir = 'reports/screenshots';
-		if (!fs.existsSync(screenshotDir)) {
-			fs.mkdirSync(screenshotDir, { recursive: true });
+	const flushedAt = Date.now();
+	for (const recording of _tabs.values()) {
+		if (recording.events.length === 0) continue;
+		try {
+			const payload = JSON.stringify({
+				workerId: _workerId,
+				tabId: recording.tabId,
+				tabIndex: recording.tabIndex,
+				events: recording.events,
+				openedAt: recording.openedAt,
+				// A tab still open when the scenario ends (typically the main tab)
+				// stayed relevant through to the flush, not just its last DOM event.
+				closedAt: recording.closedAt ?? flushedAt
+			});
+			const gz = zlib.gzipSync(Buffer.from(payload, 'utf8'));
+			await attach(gz, RRWEB_MIME_TYPE);
+		} catch {
+			// a failed recording flush shouldn't fail the scenario
 		}
-		const screenshotPath = path.join(screenshotDir, `screenshot_${Date.now()}.png`);
-		await _page.screenshot({ path: screenshotPath });
-		const screenshotData = fs.readFileSync(screenshotPath);
-		await attach(screenshotData, 'image/png');
-		fs.unlinkSync(screenshotPath);
 	}
+}
+
+export async function teardown(): Promise<void> {
 	await _browser?.close();
 }

package/backend/_scaffold/utils/hooks.ts

@@ -15,24 +15,53 @@
  * along with Plum. If not, see https://www.gnu.org/licenses/.
  */
 
-import { Before, After, AfterStep, ITestCaseHookParameter } from '@cucumber/cucumber';
-import { setup, teardown, screenshotStep, streamLiveScreenshot } from './browser';
+// Wires up Plum's session recording — removing or reordering code here can silently break report replay.
+
+import { Before, After, BeforeStep, ITestCaseHookParameter } from '@cucumber/cucumber';
+import { setup, teardown, flushRecordings, markStepStart } from './browser';
 import dotenv from 'dotenv';
 
 dotenv.config();
 
+/**
+ * Pickle steps carry no keyword (Cucumber normalizes Given/When/Then/And/But
+ * away during Gherkin → Pickle compilation) — recover it by walking the
+ * gherkinDocument for the AST node the pickle step was compiled from.
+ */
+function resolveStepKeyword(gherkinDocument: any, pickleStep: any): string {
+	const astNodeId = pickleStep?.astNodeIds?.[0];
+	if (!astNodeId) return '';
+	const steps: any[] = [];
+	for (const child of gherkinDocument?.feature?.children ?? []) {
+		if (child.background) steps.push(...child.background.steps);
+		if (child.scenario) steps.push(...child.scenario.steps);
+		for (const ruleChild of child.rule?.children ?? []) {
+			if (ruleChild.background) steps.push(...ruleChild.background.steps);
+			if (ruleChild.scenario) steps.push(...ruleChild.scenario.steps);
+		}
+	}
+	return steps.find((s) => s.id === astNodeId)?.keyword?.trim() ?? '';
+}
+
 Before(async ({ pickle }: ITestCaseHookParameter) => {
 	const tags = pickle.tags.map((t) => t.name).join(' ');
 	console.log(`\n▶ ${pickle.name}${tags ? `  ${tags}` : ''}`);
 	await setup();
 });
 
-AfterStep(async function ({ pickleStep, result }: { pickleStep: any; result: any }) {
-	if (result?.status === 'SKIPPED') return;
-	await screenshotStep(this.attach.bind(this));
-	await streamLiveScreenshot(pickleStep?.text ?? '');
+BeforeStep(async function ({
+	pickleStep,
+	gherkinDocument
+}: {
+	pickleStep: any;
+	gherkinDocument: any;
+}) {
+	const keyword = resolveStepKeyword(gherkinDocument, pickleStep);
+	const text = pickleStep?.text ?? '';
+	await markStepStart(keyword ? `${keyword} ${text}` : text);
 });
 
-After(async function (scenario: ITestCaseHookParameter) {
-	await teardown(this.attach.bind(this), scenario.result?.status === 'FAILED');
+After(async function () {
+	await flushRecordings(this.attach.bind(this));
+	await teardown();
 });

package/backend/app.js

@@ -3,10 +3,8 @@
  * Licensed under the MIT License. See LICENSE file in the project root for details.
  */
 
-const path = require('path');
 const express = require('express');
 const cors = require('cors');
-const { SCREENSHOTS_DIR } = require('./lib/reportFilename');
 const { isNodeMode } = require('./constants/env');
 const app = express();
 
@@ -16,9 +14,6 @@ app.use(cors({ origin: '*' }));
 // before a real test suite does.
 app.use(express.json({ limit: '500mb' }));
 
-// Serve screenshot files written during report processing
-app.use('/screenshots', express.static(SCREENSHOTS_DIR));
-
 // Routes
 const nodeRoutes = require('./routes/node.routes');
 app.use('/api', nodeRoutes);

package/backend/config/scripts/generate-report.js

@@ -36,7 +36,7 @@ if (!process.env.DATABASE_URL) {
 		const reportService = require('../../services/reportService');
 		const triggerType = normaliseTrigger(process.env.TRIGGER);
 		const rawTag = process.env.TAG || '@all-tests';
-		const nodeCount = Math.max(
+		const workerCount = Math.max(
 			1,
 			parseInt(process.env.REPORT_RUNNERS || process.env.PARALLEL || '1', 10) || 1
 		);
@@ -45,7 +45,7 @@ if (!process.env.DATABASE_URL) {
 			rawCucumberJson: raw,
 			tags: rawTag,
 			triggerType,
-			nodeCount,
+			workerCount,
 			browser: process.env.BROWSER || DEFAULT_BROWSER,
 			runnerName: process.env.RUNNER_NAME || null,
 			runnerId: process.env.RUNNER_ID || null,

package/backend/constants/socketEvents.js

@@ -15,23 +15,26 @@ const SOCKET_EVENTS = Object.freeze({
 	CANCEL_TEST: 'cancel-test',
 	LOG: 'log',
 	DONE: 'done',
-	STEP_SCREENSHOT: 'step-screenshot',
 
 	// Multi-lane distributed run (single interactive run, several runners)
 	RUNNER_LANES_INIT: 'runner-lanes-init',
 	RUNNER_LANE_LOG: 'runner-lane-log',
 	RUNNER_LANE_STATUS: 'runner-lane-status',
-	RUNNER_LANE_SCREENSHOT: 'runner-lane-screenshot',
 
 	// Background runs (cron / REST / MCP triggered, no single owning socket)
 	BG_RUN_START: 'bg-run-start',
 	BG_RUN_LOG: 'bg-run-log',
 	BG_RUN_DONE: 'bg-run-done',
-	BG_RUN_SCREENSHOT: 'bg-run-screenshot',
 	BG_RUN_LANES_INIT: 'bg-run-lanes-init',
 	BG_RUN_LANE_LOG: 'bg-run-lane-log',
 	BG_RUN_LANE_STATUS: 'bg-run-lane-status',
-	BG_RUN_LANE_SCREENSHOT: 'bg-run-lane-screenshot',
+
+	// Live rrweb streaming — one shape for every run type, always
+	// carrying a lane id (BUILT_IN_RUNNER_ID for the plain single-run case) and
+	// a workerId, so a single built-in run with --parallel workers is finally
+	// attributable per worker instead of one flat interleaved stream.
+	RUNNER_LANE_RRWEB_BATCH: 'runner-lane-rrweb-batch',
+	BG_RUN_LANE_RRWEB_BATCH: 'bg-run-lane-rrweb-batch',
 
 	// Global notifications (any client, not tied to a specific run)
 	REPORT_READY: 'report-ready'

package/backend/lib/reportFilename.js

@@ -7,7 +7,6 @@ const path = require('path');
 const fs = require('fs');
 
 const REPORTS_DIR = path.resolve(process.cwd(), 'reports');
-const SCREENSHOTS_DIR = path.join(REPORTS_DIR, 'screenshots');
 
 /**
  * Reads the transient cucumber_report.json written by the most recent local test run.
@@ -22,4 +21,4 @@ function readCucumberReportFile() {
 	}
 }
 
-module.exports = { REPORTS_DIR, SCREENSHOTS_DIR, readCucumberReportFile };
+module.exports = { REPORTS_DIR, readCucumberReportFile };

package/backend/lib/{screenshotPoller.js → rrwebPoller.js}

@@ -6,20 +6,23 @@
 const fs = require('fs');
 const path = require('path');
 
-function startSsPoller(ssDir, onScreenshot) {
+// The test process runs several levels below whatever calls this, with no
+// direct pipe back, so rrweb batches live-stream by small files dropped and
+// picked up here.
+function startRRwebPoller(ssDir, onRRwebBatch) {
 	const seenFiles = new Set();
 	return setInterval(() => {
 		try {
 			const files = fs
 				.readdirSync(ssDir)
-				.filter((f) => f.endsWith('.ss.json'))
+				.filter((f) => f.endsWith('.rrweb.json'))
 				.sort();
 			for (const f of files) {
 				if (seenFiles.has(f)) continue;
 				seenFiles.add(f);
 				const filePath = path.join(ssDir, f);
 				const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
-				onScreenshot(data);
+				onRRwebBatch?.(data);
 				try {
 					fs.unlinkSync(filePath);
 				} catch {}
@@ -28,4 +31,4 @@ function startSsPoller(ssDir, onScreenshot) {
 	}, 400);
 }
 
-module.exports = { startSsPoller };
+module.exports = { startRRwebPoller };

package/backend/lib/serverBootstrap.js

@@ -43,10 +43,12 @@ function wireRealtimeServices(io, isNodeMode) {
 	if (isNodeMode) return { cronService: null, backupCronService: null };
 
 	const socketHandler = require('../websockets/socketHandler.js');
+	const nodeSocketHandler = require('../websockets/nodeSocketHandler.js');
 	const cronService = require('../services/cronService');
 	const backupCronService = require('../services/backupCronService');
 
 	socketHandler(io);
+	nodeSocketHandler(io);
 	cronService.setSocketIO(io);
 	require('../routes/trigger.routes').setSocketIO(io);
 
@@ -125,6 +127,7 @@ function handleNodeModeStartup(port) {
 
 async function handleFullModeStartup(io, testsDir) {
 	syncAutomatedFlags();
+	cleanupLegacyScreenshots();
 
 	const chokidar = await loadChokidar();
 	if (!chokidar) return;
@@ -133,6 +136,17 @@ async function handleFullModeStartup(io, testsDir) {
 	watchReports(chokidar, io);
 }
 
+// Report rows no longer reference screenshot files (replaced by rrweb
+// recordings), so any leftover files on disk are dead weight. Safe to run
+// every startup: a second pass on an already-gone directory is a no-op.
+function cleanupLegacyScreenshots() {
+	const screenshotsDir = path.join(process.cwd(), 'reports', 'screenshots');
+	if (!fs.existsSync(screenshotsDir)) return;
+	fs.rm(screenshotsDir, { recursive: true, force: true }, (err) => {
+		if (!err) console.log('🧹 Removed legacy screenshots directory');
+	});
+}
+
 function syncAutomatedFlags() {
 	// Sync automated flags from feature files on every startup
 	require('../services/reportService')

package/backend/logs/runner-cmtbz5b1l0000mr0110b27w5k.log

@@ -0,0 +1,8 @@
+/*
+This file is part of Plum.
+Licensed under the MIT License. See LICENSE file in the project root for details.
+*/
+📂 Loading tests from: /Users/silverlunah/Projects/plum/backend/tests
+Backend running on port 3099 (node/runner mode)
+(node:10858) [DEP0190] DeprecationWarning: Passing args to a child process with shell option true can lead to security vulnerabilities, as the arguments are not escaped, only concatenated.
+(Use `node --trace-deprecation ...` to show where the warning was created)

package/backend/mcp/server.js

@@ -12,7 +12,6 @@
  * runs in-process inside the Plum backend, not as a separate subprocess.
  */
 
-const fs = require('fs');
 const path = require('path');
 
 // Use absolute paths to bypass the SDK's wildcard export mapping
@@ -29,7 +28,6 @@ const { McpServer } = require(path.join(sdkCjs, 'server', 'mcp.js'));
 const { z } = require('zod');
 const { TRIGGER_TYPE } = require('../constants/triggers');
 const { JOB_STATUS } = require('../constants/jobStatus');
-const { SCREENSHOTS_DIR } = require('../lib/reportFilename');
 const testSuiteService = require('../services/testSuiteService');
 const testCaseService = require('../services/testCaseService');
 const triggerService = require('../services/triggerService');
@@ -83,13 +81,6 @@ function summariseReport(report) {
 	};
 }
 
-// ---------------------------------------------------------------------------
-// Screenshots
-// ---------------------------------------------------------------------------
-
-const SCREENSHOT_FILENAME_RE = /^[\w.-]+\.(png|jpg|jpeg)$/i;
-const SCREENSHOT_MIME_TYPES = { png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg' };
-
 // ---------------------------------------------------------------------------
 // Server setup
 // ---------------------------------------------------------------------------
@@ -450,11 +441,9 @@ function createMcpServer({ userId }) {
 		'get_report_scenario_detail',
 		[
 			'Get full per-scenario, per-step detail for a Plum test report — the data needed to diagnose',
-			"and self-heal a failing test: every step's status, duration, full error message, and",
-			'screenshot URL (if one was captured for that step).',
+			"and self-heal a failing test: every step's status, duration, and full error message.",
 			'',
-			'Use get_report_screenshot to fetch the actual screenshot image for a filename returned here,',
-			'and get_report_logs for the raw test-run stdout/stderr.'
+			'Use get_report_logs for the raw test-run stdout/stderr.'
 		].join('\n'),
 		{
 			reportId: z.number().int().describe('Numeric report ID'),
@@ -482,8 +471,7 @@ function createMcpServer({ userId }) {
 							name: st.name,
 							status: st.status,
 							duration: st.duration,
-							error: st.error ?? null,
-							screenshot: st.screenshot ?? null
+							error: st.error ?? null
 						}))
 					}))
 			);
@@ -494,38 +482,6 @@ function createMcpServer({ userId }) {
 		}
 	);
 
-	server.tool(
-		'get_report_screenshot',
-		'Fetch a screenshot captured during a test step and return it as an image, so it can be viewed ' +
-			"directly. Get the filename from get_report_scenario_detail's step.screenshot field.",
-		{
-			filename: z.string().describe('Screenshot filename, e.g. "3f9c1e2a-....png"')
-		},
-		async ({ filename }) => {
-			if (!SCREENSHOT_FILENAME_RE.test(filename)) {
-				throw new Error(`Invalid screenshot filename: ${filename}`);
-			}
-			const filePath = path.join(SCREENSHOTS_DIR, filename);
-			let buffer;
-			try {
-				buffer = await fs.promises.readFile(filePath);
-			} catch {
-				throw new Error(`Screenshot not found: ${filename}`);
-			}
-			const ext = filename.split('.').pop().toLowerCase();
-
-			return {
-				content: [
-					{
-						type: 'image',
-						data: buffer.toString('base64'),
-						mimeType: SCREENSHOT_MIME_TYPES[ext]
-					}
-				]
-			};
-		}
-	);
-
 	server.tool(
 		'get_report_logs',
 		'Get the raw stdout/stderr log output captured during a Plum test run, tagged per runner. ' +