plum-e2e 2.5.14 → 2.6.1

package/backend/services/cronService.js

@@ -17,13 +17,19 @@
 
 const cron = require('node-cron');
 const { spawn } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
 const prisma = require('./prisma');
 const runnerService = require('./runnerService');
 const reportService = require('./reportService');
+const settingsService = require('./settingsService');
 const notificationService = require('./notificationService');
+const { startSsPoller } = require('../lib/screenshotPoller');
 const { BUILT_IN_RUNNER_ID, TRIGGER_REMOTE } = require('../constants/triggers');
 const { getTestIdsForTag, chunkTests, buildTagExpression } = require('../lib/testChunker');
 const { readCucumberReportFile } = require('../lib/reportFilename');
+const { runWithRetries } = require('../lib/retryRunner');
 
 const scheduledJobs = {};
 let _io = null;
@@ -60,35 +66,88 @@ async function resolveLaneInfos(runnerIds) {
 // Run paths
 // ---------------------------------------------------------------------------
 
+/**
+ * Spawns one `npm run test` attempt for a single-built-in-runner cron task.
+ * When `suppressSave` is set, PLUM_MODE=node forces generate-report.js to skip
+ * its own DB save — used by the retry path, which persists exactly one merged
+ * report itself once every attempt is done.
+ */
+function runSingleBuiltInAttempt({ taskName, currentTag, workers, browser, suppressSave, onLog }) {
+	return new Promise((resolve) => {
+		const ssDir = path.join(os.tmpdir(), `plum-cron-ss-${Date.now()}`);
+		fs.mkdirSync(ssDir, { recursive: true });
+
+		const env = {
+			...process.env,
+			TAG: currentTag,
+			TRIGGER: taskName,
+			BROWSER: browser,
+			PLUM_SS_DIR: ssDir
+		};
+		if (workers > 1) env.PARALLEL = String(workers);
+		if (suppressSave) env.PLUM_MODE = 'node';
+
+		const task = spawn('npm', ['run', 'test'], { env, shell: true });
+
+		const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
+			if (_io) _io.emit('bg-run-screenshot', { runId: taskName, stepName, data });
+		});
+
+		task.stdout.on('data', (d) => {
+			process.stdout.write(d);
+			onLog(d.toString());
+		});
+		task.stderr.on('data', (d) => {
+			process.stderr.write(d);
+			onLog(`[ERROR] ${d.toString()}`);
+		});
+		task.on('close', (code) => {
+			clearInterval(ssPoller);
+			fs.rm(ssDir, { recursive: true, force: true }, () => {});
+			resolve({ code, raw: suppressSave ? readCucumberReportFile() : null });
+		});
+	});
+}
+
 /**
  * Single built-in runner — spawns tests locally.
  * TRIGGER is set to taskName so generate-report.js can persist it correctly.
  */
-function runSingleBuiltIn({ taskName, tags, workers, browser, notifyDiscord, notifySlack }) {
-	const env = {
-		...process.env,
-		TAG: tags,
-		TRIGGER: taskName,
-		BROWSER: browser
+async function runSingleBuiltIn({ taskName, tags, workers, browser, notifyDiscord, notifySlack }) {
+	const startedAt = Date.now();
+	const { maxRetries } = await settingsService.getProject();
+
+	const onLog = (text) => {
+		if (_io) _io.emit('bg-run-log', { runId: taskName, log: text });
 	};
-	if (workers > 1) env.PARALLEL = String(workers);
 
-	const task = spawn('npm', ['run', 'test'], { env, shell: true });
-	task.stdout.on('data', (d) => process.stdout.write(d));
-	task.stderr.on('data', (d) => process.stderr.write(d));
-	task.on('close', (code) => {
+	if (maxRetries === 0) {
+		const { code } = await runSingleBuiltInAttempt({
+			taskName,
+			currentTag: tags,
+			workers,
+			browser,
+			suppressSave: false,
+			onLog
+		});
 		console.log(`Task "${taskName}" finished with code ${code}`);
-		if (_io) _io.emit('cron-done', { taskName, code });
-
-		if (notifyDiscord || notifySlack) {
-			prisma.report
-				.findFirst({
-					where: { triggerType: taskName },
-					orderBy: { createdAt: 'desc' },
-					select: { id: true, status: true, content: true }
-				})
-				.then((report) => {
-					if (!report) return;
+
+		prisma.report
+			.findFirst({
+				where: { triggerType: taskName },
+				orderBy: { createdAt: 'desc' },
+				select: { id: true, status: true, content: true }
+			})
+			.then((report) => {
+				if (_io) _io.emit('bg-run-done', { runId: taskName, code, reportId: report?.id ?? null });
+
+				if (report) {
+					prisma.report
+						.update({ where: { id: report.id }, data: { duration: Date.now() - startedAt } })
+						.catch(() => {});
+				}
+
+				if (report && (notifyDiscord || notifySlack)) {
 					return notificationService.send({
 						jobName: taskName,
 						status: report.status,
@@ -99,10 +158,63 @@ function runSingleBuiltIn({ taskName, tags, workers, browser, notifyDiscord, not
 						notifyDiscord,
 						notifySlack
 					});
-				})
-				.catch((e) => console.error(`[cron] Notification failed: ${e.message}`));
-		}
+				}
+			})
+			.catch((e) => {
+				console.error(`[cron] Notification failed: ${e.message}`);
+				if (_io) _io.emit('bg-run-done', { runId: taskName, code, reportId: null });
+			});
+		return;
+	}
+
+	const { code, rawJson, attempts } = await runWithRetries({
+		maxRetries,
+		spawnAttempt: async (tagOverride) => {
+			const { code, raw } = await runSingleBuiltInAttempt({
+				taskName,
+				currentTag: tagOverride ?? tags,
+				workers,
+				browser,
+				suppressSave: true,
+				onLog
+			});
+			return { code, rawJson: raw ? JSON.parse(raw) : [] };
+		},
+		onLog
 	});
+
+	console.log(`Task "${taskName}" finished with code ${code}`);
+
+	let report = null;
+	try {
+		report = await reportService.saveReport({
+			rawCucumberJson: rawJson,
+			tags,
+			triggerType: taskName,
+			browser,
+			duration: Date.now() - startedAt,
+			attempts
+		});
+	} catch (e) {
+		console.error(`[cron] Failed to save report: ${e.message}`);
+	}
+
+	if (_io) _io.emit('bg-run-done', { runId: taskName, code, reportId: report?.id ?? null });
+
+	if (report && (notifyDiscord || notifySlack)) {
+		notificationService
+			.send({
+				jobName: taskName,
+				status: report.status,
+				content: report.content,
+				browser,
+				tags,
+				reportId: report.id,
+				notifyDiscord,
+				notifySlack
+			})
+			.catch((e) => console.error(`[cron] Notification failed: ${e.message}`));
+	}
 }
 
 /**
@@ -118,6 +230,8 @@ async function runDistributed({
 	notifyDiscord,
 	notifySlack
 }) {
+	const dispatchStartedAt = Date.now();
+	const { maxRetries } = await settingsService.getProject();
 	const allIds = getTestIdsForTag(tags);
 	const chunks = chunkTests(allIds, runnerIds.length);
 
@@ -127,24 +241,39 @@ async function runDistributed({
 
 	if (activeRunnerIds.length === 0) {
 		console.log(`Task "${taskName}" — no tests found, skipping.`);
-		if (_io) _io.emit('cron-done', { taskName, code: 0 });
+		if (_io) _io.emit('bg-run-done', { runId: taskName, code: 0, reportId: null });
 		return;
 	}
 
 	const laneInfos = await resolveLaneInfos(activeRunnerIds);
 
+	if (_io) {
+		_io.emit('bg-run-lanes-init', {
+			runId: taskName,
+			lanes: laneInfos.map((l, i) => ({ id: l.id, name: l.name, testCount: chunks[i].length }))
+		});
+	}
+
 	const collectedReports = new Array(activeRunnerIds.length).fill(null);
+	const laneAttempts = new Array(activeRunnerIds.length).fill(null);
 	let doneCount = 0;
 	let overallCode = 0;
 
-	function onLaneDone(idx, code, reportContent) {
+	function onLaneDone(idx, laneId, code, reportContent, attempts = null) {
 		if (code !== 0) overallCode = code;
 		collectedReports[idx] = reportContent;
+		laneAttempts[idx] = attempts;
 		doneCount++;
+		if (_io) {
+			_io.emit('bg-run-lane-status', {
+				runId: taskName,
+				laneId,
+				status: code === 0 ? 'done' : 'error'
+			});
+		}
 
 		if (doneCount === activeRunnerIds.length) {
 			console.log(`Task "${taskName}" — all runners done (exit ${overallCode})`);
-			if (_io) _io.emit('cron-done', { taskName, code: overallCode });
 
 			reportService
 				.saveCombinedReport({
@@ -153,9 +282,18 @@ async function runDistributed({
 					overallCode,
 					tag: tags,
 					triggerType: taskName,
-					browser
+					browser,
+					duration: Date.now() - dispatchStartedAt,
+					attemptsByLane: laneAttempts
 				})
 				.then((saved) => {
+					if (_io) {
+						_io.emit('bg-run-done', {
+							runId: taskName,
+							code: overallCode,
+							reportId: saved?.id ?? null
+						});
+					}
 					if (saved && (notifyDiscord || notifySlack)) {
 						return notificationService.send({
 							jobName: taskName,
@@ -169,36 +307,96 @@ async function runDistributed({
 						});
 					}
 				})
-				.catch((e) => console.error(`[cron] Failed to save combined report: ${e.message}`));
+				.catch((e) => {
+					console.error(`[cron] Failed to save combined report: ${e.message}`);
+					if (_io) _io.emit('bg-run-done', { runId: taskName, code: overallCode, reportId: null });
+				});
 		}
 	}
 
 	for (let i = 0; i < activeRunnerIds.length; i++) {
 		const lane = laneInfos[i];
+		const laneId = lane.id;
 		const chunkTag = buildTagExpression(chunks[i]);
 
 		if (lane.id === BUILT_IN_RUNNER_ID) {
-			const env = {
-				...process.env,
-				TAG: chunkTag,
-				TRIGGER: TRIGGER_REMOTE, // node-mode: file naming only, not persisted to DB
-				BROWSER: browser,
-				PLUM_MODE: 'node'
+			const idx = i;
+			const onLog = (text) => {
+				if (_io) _io.emit('bg-run-lane-log', { runId: taskName, laneId, log: text });
 			};
-			if (workers > 1) env.PARALLEL = String(workers);
 
-			const task = spawn('npm', ['run', 'test'], { env, shell: true });
-			task.stdout.on('data', (d) => process.stdout.write(d));
-			task.stderr.on('data', (d) => process.stderr.write(d));
-			const idx = i;
-			task.on('close', (code) => onLaneDone(idx, code, readCucumberReportFile()));
+			const spawnBuiltInLaneAttempt = (currentTag) =>
+				new Promise((resolve) => {
+					const ssDir = path.join(os.tmpdir(), `plum-cron-ss-${Date.now()}-${idx}`);
+					fs.mkdirSync(ssDir, { recursive: true });
+
+					const env = {
+						...process.env,
+						TAG: currentTag,
+						TRIGGER: TRIGGER_REMOTE, // node-mode: file naming only, not persisted to DB
+						BROWSER: browser,
+						PLUM_MODE: 'node',
+						PLUM_SS_DIR: ssDir
+					};
+					if (workers > 1) env.PARALLEL = String(workers);
+
+					const task = spawn('npm', ['run', 'test'], { env, shell: true });
+
+					const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
+						if (_io)
+							_io.emit('bg-run-lane-screenshot', { runId: taskName, laneId, stepName, data });
+					});
+
+					task.stdout.on('data', (d) => {
+						process.stdout.write(d);
+						onLog(d.toString());
+					});
+					task.stderr.on('data', (d) => {
+						process.stderr.write(d);
+						onLog(`[ERROR] ${d.toString()}`);
+					});
+					task.on('close', (code) => {
+						clearInterval(ssPoller);
+						fs.rm(ssDir, { recursive: true, force: true }, () => {});
+						const raw = readCucumberReportFile() ?? '[]';
+						resolve({ code, rawJson: JSON.parse(raw) });
+					});
+				});
+
+			runWithRetries({
+				maxRetries,
+				spawnAttempt: (t) => spawnBuiltInLaneAttempt(t ?? chunkTag),
+				onLog
+			}).then(({ code, rawJson, attempts }) =>
+				onLaneDone(idx, laneId, code, JSON.stringify(rawJson), attempts)
+			);
 		} else {
 			const idx = i;
-			runnerService.dispatchAndPoll(
-				lane.id,
-				{ tags: chunkTag, browser, workers },
-				(log) => process.stdout.write(log),
-				(code, content) => onLaneDone(idx, code, content)
+			const onLog = (log) => {
+				process.stdout.write(log);
+				if (_io) _io.emit('bg-run-lane-log', { runId: taskName, laneId, log });
+			};
+
+			const spawnRemoteLaneAttempt = (currentTag) =>
+				new Promise((resolve) => {
+					runnerService.dispatchAndPoll(
+						lane.id,
+						{ tags: currentTag, browser, workers },
+						onLog,
+						(code, content) => resolve({ code, rawJson: content ? JSON.parse(content) : [] }),
+						({ stepName, data }) => {
+							if (_io)
+								_io.emit('bg-run-lane-screenshot', { runId: taskName, laneId, stepName, data });
+						}
+					);
+				});
+
+			runWithRetries({
+				maxRetries,
+				spawnAttempt: (t) => spawnRemoteLaneAttempt(t ?? chunkTag),
+				onLog
+			}).then(({ code, rawJson, attempts }) =>
+				onLaneDone(idx, laneId, code, JSON.stringify(rawJson), attempts)
 			);
 		}
 	}
@@ -220,7 +418,14 @@ async function runCronJob(job) {
 	} = job;
 	const runnerIds = parseRunnerIds(runnerIdsStr);
 
-	if (_io) _io.emit('cron-start', { taskName });
+	if (_io) {
+		_io.emit('bg-run-start', {
+			runId: taskName,
+			kind: 'cron',
+			label: taskName,
+			meta: { tag: tags, browser, workers }
+		});
+	}
 	console.log(`Running scheduled task: "${taskName}" → runners: ${runnerIds.join(', ')}`);
 
 	const isSingleBuiltIn = runnerIds.length === 1 && runnerIds[0] === BUILT_IN_RUNNER_ID;

package/backend/services/reportService.js

@@ -139,6 +139,64 @@ function featureMergeKey(feature) {
 	return uri || feature.id || feature.name;
 }
 
+// Mirrors frontend's isTestCaseTag (frontend/src/lib/utils/format.js) — keep in
+// sync so retry-attempt counts line up with how the report page groups scenarios.
+function isTestCaseTag(tag) {
+	return /^@test[\w-]*/i.test(tag) || /^@tc-?\d+/i.test(tag);
+}
+
+function scenarioIdTag(scenario) {
+	return (scenario.tags ?? []).map((t) => t.name).find(isTestCaseTag) ?? null;
+}
+
+function scenarioFailed(scenario) {
+	return (scenario.steps ?? []).some((s) => s.result?.status === 'failed');
+}
+
+/**
+ * Test-ID tags (see scenarioIdTag) of every failed scenario in a raw Cucumber
+ * JSON payload, deduped. Used to scope the next retry attempt to just the
+ * scenarios that need re-running.
+ */
+function getFailedIdTags(rawJson) {
+	const tags = new Set();
+	for (const feature of rawJson) {
+		for (const scenario of feature.elements ?? []) {
+			if (scenarioFailed(scenario)) {
+				const idTag = scenarioIdTag(scenario);
+				if (idTag) tags.add(idTag);
+			}
+		}
+	}
+	return [...tags];
+}
+
+/**
+ * Folds one retry attempt's raw Cucumber JSON into the accumulated result,
+ * replacing each retried scenario's prior-round entry with this round's
+ * outcome (matched by Cucumber's own stable scenario id). Mutates
+ * `attemptsMap` in place, recording the highest round number each scenario
+ * appeared in — that number *is* its total attempt count, since a scenario
+ * only reappears in a later round if it failed every round before it.
+ */
+function mergeRawAttempt(accumulated, attemptRawJson, round, attemptsMap) {
+	for (const feature of attemptRawJson) {
+		const key = featureMergeKey(feature);
+		let accFeature = accumulated.find((f) => featureMergeKey(f) === key);
+		if (!accFeature) {
+			accFeature = { ...feature, elements: [] };
+			accumulated.push(accFeature);
+		}
+		for (const scenario of feature.elements ?? []) {
+			accFeature.elements = accFeature.elements.filter((e) => e.id !== scenario.id);
+			accFeature.elements.push(scenario);
+			const idTag = scenarioIdTag(scenario);
+			if (idTag) attemptsMap[idTag] = round;
+		}
+	}
+	return accumulated;
+}
+
 function deleteScreenshotFiles(content) {
 	for (const file of collectScreenshotFiles(content)) {
 		const p = path.join(SCREENSHOTS_DIR, file);
@@ -156,7 +214,7 @@ function deleteScreenshotFiles(content) {
  *
  * Returns { features, status } where status is 'PASS' | 'FAIL'.
  */
-function processCucumberJson(raw) {
+function processCucumberJson(raw, attempts = {}) {
 	fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
 
 	const features = raw.map((feature) => {
@@ -215,6 +273,7 @@ function processCucumberJson(raw) {
 				tags: (scenario.tags ?? []).map((t) => t.name),
 				status: worstStatus,
 				duration: steps.reduce((s, st) => s + st.duration, 0),
+				attempts: attempts[scenarioIdTag(scenario)] ?? 1,
 				steps
 			};
 		});
@@ -291,6 +350,7 @@ const getReportDetail = async (id) => {
 			createdAt: true,
 			content: true,
 			logs: true,
+			duration: true,
 			testRun: { select: { id: true, title: true } }
 		}
 	});
@@ -327,10 +387,12 @@ const saveReport = async ({
 	runnerId,
 	testRunId,
 	forceFail = false,
-	logs = null
+	logs = null,
+	duration = null,
+	attempts = {}
 }) => {
 	const normTrigger = normaliseTrigger(triggerType);
-	const { features, status: derivedStatus } = processCucumberJson(rawCucumberJson);
+	const { features, status: derivedStatus } = processCucumberJson(rawCucumberJson, attempts);
 	const status = forceFail ? 'FAIL' : derivedStatus;
 	const cronJobId = await resolveCronJobId(normTrigger);
 
@@ -346,7 +408,8 @@ const saveReport = async ({
 			cronJobId,
 			testRunId: testRunId ?? null,
 			content: { features },
-			logs: logs || null
+			logs: logs || null,
+			duration
 		}
 	});
 	syncAutomatedTags(report.id, features, testRunId ?? null);
@@ -364,6 +427,7 @@ const saveReport = async ({
  *   tag: string,
  *   triggerType: string,
  *   browser: string,
+ *   duration: number,
  * }} opts
  */
 const saveCombinedReport = async ({
@@ -374,7 +438,9 @@ const saveCombinedReport = async ({
 	triggerType,
 	browser,
 	testRunId,
-	laneLogs = null
+	laneLogs = null,
+	duration = null,
+	attemptsByLane = null
 }) => {
 	const featureMap = new Map();
 	for (const content of reports) {
@@ -408,6 +474,10 @@ const saveCombinedReport = async ({
 		if (parts.length > 0) combinedLogs = parts.join('\n\n');
 	}
 
+	// Chunked lanes run disjoint sets of tests, so their attempt maps never
+	// collide — a plain merge is safe.
+	const attempts = attemptsByLane ? Object.assign({}, ...attemptsByLane.filter(Boolean)) : {};
+
 	return saveReport({
 		rawCucumberJson: combined,
 		tags: tag,
@@ -418,7 +488,9 @@ const saveCombinedReport = async ({
 		runnerId: null,
 		testRunId: testRunId ?? null,
 		forceFail: reports.some((r) => r === null),
-		logs: combinedLogs
+		logs: combinedLogs,
+		duration,
+		attempts
 	});
 };
 
@@ -469,5 +541,7 @@ module.exports = {
 	saveCombinedReport,
 	syncAutomatedFromFeatures,
 	deleteReport,
-	deleteReports
+	deleteReports,
+	getFailedIdTags,
+	mergeRawAttempt
 };

package/backend/services/runnerService.js

@@ -85,6 +85,29 @@ async function ping(id) {
 	return probe({ url: runner.url, token: runner.token });
 }
 
+// ---------------------------------------------------------------------------
+// Remote control
+// ---------------------------------------------------------------------------
+
+async function callControlEndpoint(id, endpoint, timeoutMs) {
+	const runner = await getById(id);
+	if (!runner) return { ok: false, error: 'Runner not found' };
+	try {
+		const res = await fetch(`${runner.url}/api/${endpoint}`, {
+			method: 'POST',
+			headers: { Authorization: `Bearer ${runner.token}` },
+			signal: AbortSignal.timeout(timeoutMs)
+		});
+		if (!res.ok) return { ok: false, error: `HTTP ${res.status}` };
+		return { ok: true };
+	} catch (e) {
+		return { ok: false, error: e.message };
+	}
+}
+
+const stop = (id) => callControlEndpoint(id, 'shutdown', 5000);
+const restart = (id) => callControlEndpoint(id, 'restart', 5000);
+
 // ---------------------------------------------------------------------------
 // Remote execution
 // ---------------------------------------------------------------------------
@@ -235,5 +258,7 @@ module.exports = {
 	getById,
 	probe,
 	ping,
+	stop,
+	restart,
 	dispatchAndPoll
 };

package/backend/services/settingsService.js

@@ -26,11 +26,12 @@ const getProject = async () => {
 	return project;
 };
 
-const updateProject = async ({ name, logoUrl, timezone }) => {
+const updateProject = async ({ name, logoUrl, timezone, maxRetries }) => {
 	const data = {
 		name: name ?? '',
 		logoUrl: logoUrl ?? '',
-		...(timezone !== undefined && { timezone })
+		...(timezone !== undefined && { timezone }),
+		...(maxRetries !== undefined && { maxRetries: Number(maxRetries) || 0 })
 	};
 	const project = await prisma.project.upsert({
 		where: { id: 1 },