plum-e2e 2.6.0 → 2.6.1

package/backend/services/cronService.js

@@ -23,11 +23,13 @@ const path = require('path');
 const prisma = require('./prisma');
 const runnerService = require('./runnerService');
 const reportService = require('./reportService');
+const settingsService = require('./settingsService');
 const notificationService = require('./notificationService');
 const { startSsPoller } = require('../lib/screenshotPoller');
 const { BUILT_IN_RUNNER_ID, TRIGGER_REMOTE } = require('../constants/triggers');
 const { getTestIdsForTag, chunkTests, buildTagExpression } = require('../lib/testChunker');
 const { readCucumberReportFile } = require('../lib/reportFilename');
+const { runWithRetries } = require('../lib/retryRunner');
 
 const scheduledJobs = {};
 let _io = null;
@@ -64,41 +66,70 @@ async function resolveLaneInfos(runnerIds) {
 // Run paths
 // ---------------------------------------------------------------------------
 
+/**
+ * Spawns one `npm run test` attempt for a single-built-in-runner cron task.
+ * When `suppressSave` is set, PLUM_MODE=node forces generate-report.js to skip
+ * its own DB save — used by the retry path, which persists exactly one merged
+ * report itself once every attempt is done.
+ */
+function runSingleBuiltInAttempt({ taskName, currentTag, workers, browser, suppressSave, onLog }) {
+	return new Promise((resolve) => {
+		const ssDir = path.join(os.tmpdir(), `plum-cron-ss-${Date.now()}`);
+		fs.mkdirSync(ssDir, { recursive: true });
+
+		const env = {
+			...process.env,
+			TAG: currentTag,
+			TRIGGER: taskName,
+			BROWSER: browser,
+			PLUM_SS_DIR: ssDir
+		};
+		if (workers > 1) env.PARALLEL = String(workers);
+		if (suppressSave) env.PLUM_MODE = 'node';
+
+		const task = spawn('npm', ['run', 'test'], { env, shell: true });
+
+		const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
+			if (_io) _io.emit('bg-run-screenshot', { runId: taskName, stepName, data });
+		});
+
+		task.stdout.on('data', (d) => {
+			process.stdout.write(d);
+			onLog(d.toString());
+		});
+		task.stderr.on('data', (d) => {
+			process.stderr.write(d);
+			onLog(`[ERROR] ${d.toString()}`);
+		});
+		task.on('close', (code) => {
+			clearInterval(ssPoller);
+			fs.rm(ssDir, { recursive: true, force: true }, () => {});
+			resolve({ code, raw: suppressSave ? readCucumberReportFile() : null });
+		});
+	});
+}
+
 /**
  * Single built-in runner — spawns tests locally.
  * TRIGGER is set to taskName so generate-report.js can persist it correctly.
  */
-function runSingleBuiltIn({ taskName, tags, workers, browser, notifyDiscord, notifySlack }) {
-	const ssDir = path.join(os.tmpdir(), `plum-cron-ss-${Date.now()}`);
-	fs.mkdirSync(ssDir, { recursive: true });
+async function runSingleBuiltIn({ taskName, tags, workers, browser, notifyDiscord, notifySlack }) {
 	const startedAt = Date.now();
+	const { maxRetries } = await settingsService.getProject();
 
-	const env = {
-		...process.env,
-		TAG: tags,
-		TRIGGER: taskName,
-		BROWSER: browser,
-		PLUM_SS_DIR: ssDir
+	const onLog = (text) => {
+		if (_io) _io.emit('bg-run-log', { runId: taskName, log: text });
 	};
-	if (workers > 1) env.PARALLEL = String(workers);
-
-	const task = spawn('npm', ['run', 'test'], { env, shell: true });
-
-	const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
-		if (_io) _io.emit('bg-run-screenshot', { runId: taskName, stepName, data });
-	});
 
-	task.stdout.on('data', (d) => {
-		process.stdout.write(d);
-		if (_io) _io.emit('bg-run-log', { runId: taskName, log: d.toString() });
-	});
-	task.stderr.on('data', (d) => {
-		process.stderr.write(d);
-		if (_io) _io.emit('bg-run-log', { runId: taskName, log: `[ERROR] ${d.toString()}` });
-	});
-	task.on('close', (code) => {
-		clearInterval(ssPoller);
-		fs.rm(ssDir, { recursive: true, force: true }, () => {});
+	if (maxRetries === 0) {
+		const { code } = await runSingleBuiltInAttempt({
+			taskName,
+			currentTag: tags,
+			workers,
+			browser,
+			suppressSave: false,
+			onLog
+		});
 		console.log(`Task "${taskName}" finished with code ${code}`);
 
 		prisma.report
@@ -133,7 +164,57 @@ function runSingleBuiltIn({ taskName, tags, workers, browser, notifyDiscord, not
 				console.error(`[cron] Notification failed: ${e.message}`);
 				if (_io) _io.emit('bg-run-done', { runId: taskName, code, reportId: null });
 			});
+		return;
+	}
+
+	const { code, rawJson, attempts } = await runWithRetries({
+		maxRetries,
+		spawnAttempt: async (tagOverride) => {
+			const { code, raw } = await runSingleBuiltInAttempt({
+				taskName,
+				currentTag: tagOverride ?? tags,
+				workers,
+				browser,
+				suppressSave: true,
+				onLog
+			});
+			return { code, rawJson: raw ? JSON.parse(raw) : [] };
+		},
+		onLog
 	});
+
+	console.log(`Task "${taskName}" finished with code ${code}`);
+
+	let report = null;
+	try {
+		report = await reportService.saveReport({
+			rawCucumberJson: rawJson,
+			tags,
+			triggerType: taskName,
+			browser,
+			duration: Date.now() - startedAt,
+			attempts
+		});
+	} catch (e) {
+		console.error(`[cron] Failed to save report: ${e.message}`);
+	}
+
+	if (_io) _io.emit('bg-run-done', { runId: taskName, code, reportId: report?.id ?? null });
+
+	if (report && (notifyDiscord || notifySlack)) {
+		notificationService
+			.send({
+				jobName: taskName,
+				status: report.status,
+				content: report.content,
+				browser,
+				tags,
+				reportId: report.id,
+				notifyDiscord,
+				notifySlack
+			})
+			.catch((e) => console.error(`[cron] Notification failed: ${e.message}`));
+	}
 }
 
 /**
@@ -150,6 +231,7 @@ async function runDistributed({
 	notifySlack
 }) {
 	const dispatchStartedAt = Date.now();
+	const { maxRetries } = await settingsService.getProject();
 	const allIds = getTestIdsForTag(tags);
 	const chunks = chunkTests(allIds, runnerIds.length);
 
@@ -173,12 +255,14 @@ async function runDistributed({
 	}
 
 	const collectedReports = new Array(activeRunnerIds.length).fill(null);
+	const laneAttempts = new Array(activeRunnerIds.length).fill(null);
 	let doneCount = 0;
 	let overallCode = 0;
 
-	function onLaneDone(idx, laneId, code, reportContent) {
+	function onLaneDone(idx, laneId, code, reportContent, attempts = null) {
 		if (code !== 0) overallCode = code;
 		collectedReports[idx] = reportContent;
+		laneAttempts[idx] = attempts;
 		doneCount++;
 		if (_io) {
 			_io.emit('bg-run-lane-status', {
@@ -199,7 +283,8 @@ async function runDistributed({
 					tag: tags,
 					triggerType: taskName,
 					browser,
-					duration: Date.now() - dispatchStartedAt
+					duration: Date.now() - dispatchStartedAt,
+					attemptsByLane: laneAttempts
 				})
 				.then((saved) => {
 					if (_io) {
@@ -235,53 +320,83 @@ async function runDistributed({
 		const chunkTag = buildTagExpression(chunks[i]);
 
 		if (lane.id === BUILT_IN_RUNNER_ID) {
-			const ssDir = path.join(os.tmpdir(), `plum-cron-ss-${Date.now()}-${i}`);
-			fs.mkdirSync(ssDir, { recursive: true });
-
-			const env = {
-				...process.env,
-				TAG: chunkTag,
-				TRIGGER: TRIGGER_REMOTE, // node-mode: file naming only, not persisted to DB
-				BROWSER: browser,
-				PLUM_MODE: 'node',
-				PLUM_SS_DIR: ssDir
+			const idx = i;
+			const onLog = (text) => {
+				if (_io) _io.emit('bg-run-lane-log', { runId: taskName, laneId, log: text });
 			};
-			if (workers > 1) env.PARALLEL = String(workers);
 
-			const task = spawn('npm', ['run', 'test'], { env, shell: true });
+			const spawnBuiltInLaneAttempt = (currentTag) =>
+				new Promise((resolve) => {
+					const ssDir = path.join(os.tmpdir(), `plum-cron-ss-${Date.now()}-${idx}`);
+					fs.mkdirSync(ssDir, { recursive: true });
+
+					const env = {
+						...process.env,
+						TAG: currentTag,
+						TRIGGER: TRIGGER_REMOTE, // node-mode: file naming only, not persisted to DB
+						BROWSER: browser,
+						PLUM_MODE: 'node',
+						PLUM_SS_DIR: ssDir
+					};
+					if (workers > 1) env.PARALLEL = String(workers);
+
+					const task = spawn('npm', ['run', 'test'], { env, shell: true });
+
+					const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
+						if (_io)
+							_io.emit('bg-run-lane-screenshot', { runId: taskName, laneId, stepName, data });
+					});
 
-			const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
-				if (_io) _io.emit('bg-run-lane-screenshot', { runId: taskName, laneId, stepName, data });
-			});
+					task.stdout.on('data', (d) => {
+						process.stdout.write(d);
+						onLog(d.toString());
+					});
+					task.stderr.on('data', (d) => {
+						process.stderr.write(d);
+						onLog(`[ERROR] ${d.toString()}`);
+					});
+					task.on('close', (code) => {
+						clearInterval(ssPoller);
+						fs.rm(ssDir, { recursive: true, force: true }, () => {});
+						const raw = readCucumberReportFile() ?? '[]';
+						resolve({ code, rawJson: JSON.parse(raw) });
+					});
+				});
 
-			task.stdout.on('data', (d) => {
-				process.stdout.write(d);
-				if (_io) _io.emit('bg-run-lane-log', { runId: taskName, laneId, log: d.toString() });
-			});
-			task.stderr.on('data', (d) => {
-				const text = `[ERROR] ${d.toString()}`;
-				process.stderr.write(d);
-				if (_io) _io.emit('bg-run-lane-log', { runId: taskName, laneId, log: text });
-			});
-			const idx = i;
-			task.on('close', (code) => {
-				clearInterval(ssPoller);
-				fs.rm(ssDir, { recursive: true, force: true }, () => {});
-				onLaneDone(idx, laneId, code, readCucumberReportFile());
-			});
+			runWithRetries({
+				maxRetries,
+				spawnAttempt: (t) => spawnBuiltInLaneAttempt(t ?? chunkTag),
+				onLog
+			}).then(({ code, rawJson, attempts }) =>
+				onLaneDone(idx, laneId, code, JSON.stringify(rawJson), attempts)
+			);
 		} else {
 			const idx = i;
-			runnerService.dispatchAndPoll(
-				lane.id,
-				{ tags: chunkTag, browser, workers },
-				(log) => {
-					process.stdout.write(log);
-					if (_io) _io.emit('bg-run-lane-log', { runId: taskName, laneId, log });
-				},
-				(code, content) => onLaneDone(idx, laneId, code, content),
-				({ stepName, data }) => {
-					if (_io) _io.emit('bg-run-lane-screenshot', { runId: taskName, laneId, stepName, data });
-				}
+			const onLog = (log) => {
+				process.stdout.write(log);
+				if (_io) _io.emit('bg-run-lane-log', { runId: taskName, laneId, log });
+			};
+
+			const spawnRemoteLaneAttempt = (currentTag) =>
+				new Promise((resolve) => {
+					runnerService.dispatchAndPoll(
+						lane.id,
+						{ tags: currentTag, browser, workers },
+						onLog,
+						(code, content) => resolve({ code, rawJson: content ? JSON.parse(content) : [] }),
+						({ stepName, data }) => {
+							if (_io)
+								_io.emit('bg-run-lane-screenshot', { runId: taskName, laneId, stepName, data });
+						}
+					);
+				});
+
+			runWithRetries({
+				maxRetries,
+				spawnAttempt: (t) => spawnRemoteLaneAttempt(t ?? chunkTag),
+				onLog
+			}).then(({ code, rawJson, attempts }) =>
+				onLaneDone(idx, laneId, code, JSON.stringify(rawJson), attempts)
 			);
 		}
 	}

package/backend/services/reportService.js

@@ -139,6 +139,64 @@ function featureMergeKey(feature) {
 	return uri || feature.id || feature.name;
 }
 
+// Mirrors frontend's isTestCaseTag (frontend/src/lib/utils/format.js) — keep in
+// sync so retry-attempt counts line up with how the report page groups scenarios.
+function isTestCaseTag(tag) {
+	return /^@test[\w-]*/i.test(tag) || /^@tc-?\d+/i.test(tag);
+}
+
+function scenarioIdTag(scenario) {
+	return (scenario.tags ?? []).map((t) => t.name).find(isTestCaseTag) ?? null;
+}
+
+function scenarioFailed(scenario) {
+	return (scenario.steps ?? []).some((s) => s.result?.status === 'failed');
+}
+
+/**
+ * Test-ID tags (see scenarioIdTag) of every failed scenario in a raw Cucumber
+ * JSON payload, deduped. Used to scope the next retry attempt to just the
+ * scenarios that need re-running.
+ */
+function getFailedIdTags(rawJson) {
+	const tags = new Set();
+	for (const feature of rawJson) {
+		for (const scenario of feature.elements ?? []) {
+			if (scenarioFailed(scenario)) {
+				const idTag = scenarioIdTag(scenario);
+				if (idTag) tags.add(idTag);
+			}
+		}
+	}
+	return [...tags];
+}
+
+/**
+ * Folds one retry attempt's raw Cucumber JSON into the accumulated result,
+ * replacing each retried scenario's prior-round entry with this round's
+ * outcome (matched by Cucumber's own stable scenario id). Mutates
+ * `attemptsMap` in place, recording the highest round number each scenario
+ * appeared in — that number *is* its total attempt count, since a scenario
+ * only reappears in a later round if it failed every round before it.
+ */
+function mergeRawAttempt(accumulated, attemptRawJson, round, attemptsMap) {
+	for (const feature of attemptRawJson) {
+		const key = featureMergeKey(feature);
+		let accFeature = accumulated.find((f) => featureMergeKey(f) === key);
+		if (!accFeature) {
+			accFeature = { ...feature, elements: [] };
+			accumulated.push(accFeature);
+		}
+		for (const scenario of feature.elements ?? []) {
+			accFeature.elements = accFeature.elements.filter((e) => e.id !== scenario.id);
+			accFeature.elements.push(scenario);
+			const idTag = scenarioIdTag(scenario);
+			if (idTag) attemptsMap[idTag] = round;
+		}
+	}
+	return accumulated;
+}
+
 function deleteScreenshotFiles(content) {
 	for (const file of collectScreenshotFiles(content)) {
 		const p = path.join(SCREENSHOTS_DIR, file);
@@ -156,7 +214,7 @@ function deleteScreenshotFiles(content) {
  *
  * Returns { features, status } where status is 'PASS' | 'FAIL'.
  */
-function processCucumberJson(raw) {
+function processCucumberJson(raw, attempts = {}) {
 	fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
 
 	const features = raw.map((feature) => {
@@ -215,6 +273,7 @@ function processCucumberJson(raw) {
 				tags: (scenario.tags ?? []).map((t) => t.name),
 				status: worstStatus,
 				duration: steps.reduce((s, st) => s + st.duration, 0),
+				attempts: attempts[scenarioIdTag(scenario)] ?? 1,
 				steps
 			};
 		});
@@ -329,10 +388,11 @@ const saveReport = async ({
 	testRunId,
 	forceFail = false,
 	logs = null,
-	duration = null
+	duration = null,
+	attempts = {}
 }) => {
 	const normTrigger = normaliseTrigger(triggerType);
-	const { features, status: derivedStatus } = processCucumberJson(rawCucumberJson);
+	const { features, status: derivedStatus } = processCucumberJson(rawCucumberJson, attempts);
 	const status = forceFail ? 'FAIL' : derivedStatus;
 	const cronJobId = await resolveCronJobId(normTrigger);
 
@@ -379,7 +439,8 @@ const saveCombinedReport = async ({
 	browser,
 	testRunId,
 	laneLogs = null,
-	duration = null
+	duration = null,
+	attemptsByLane = null
 }) => {
 	const featureMap = new Map();
 	for (const content of reports) {
@@ -413,6 +474,10 @@ const saveCombinedReport = async ({
 		if (parts.length > 0) combinedLogs = parts.join('\n\n');
 	}
 
+	// Chunked lanes run disjoint sets of tests, so their attempt maps never
+	// collide — a plain merge is safe.
+	const attempts = attemptsByLane ? Object.assign({}, ...attemptsByLane.filter(Boolean)) : {};
+
 	return saveReport({
 		rawCucumberJson: combined,
 		tags: tag,
@@ -424,7 +489,8 @@ const saveCombinedReport = async ({
 		testRunId: testRunId ?? null,
 		forceFail: reports.some((r) => r === null),
 		logs: combinedLogs,
-		duration
+		duration,
+		attempts
 	});
 };
 
@@ -475,5 +541,7 @@ module.exports = {
 	saveCombinedReport,
 	syncAutomatedFromFeatures,
 	deleteReport,
-	deleteReports
+	deleteReports,
+	getFailedIdTags,
+	mergeRawAttempt
 };

package/backend/services/runnerService.js

@@ -85,6 +85,29 @@ async function ping(id) {
 	return probe({ url: runner.url, token: runner.token });
 }
 
+// ---------------------------------------------------------------------------
+// Remote control
+// ---------------------------------------------------------------------------
+
+async function callControlEndpoint(id, endpoint, timeoutMs) {
+	const runner = await getById(id);
+	if (!runner) return { ok: false, error: 'Runner not found' };
+	try {
+		const res = await fetch(`${runner.url}/api/${endpoint}`, {
+			method: 'POST',
+			headers: { Authorization: `Bearer ${runner.token}` },
+			signal: AbortSignal.timeout(timeoutMs)
+		});
+		if (!res.ok) return { ok: false, error: `HTTP ${res.status}` };
+		return { ok: true };
+	} catch (e) {
+		return { ok: false, error: e.message };
+	}
+}
+
+const stop = (id) => callControlEndpoint(id, 'shutdown', 5000);
+const restart = (id) => callControlEndpoint(id, 'restart', 5000);
+
 // ---------------------------------------------------------------------------
 // Remote execution
 // ---------------------------------------------------------------------------
@@ -235,5 +258,7 @@ module.exports = {
 	getById,
 	probe,
 	ping,
+	stop,
+	restart,
 	dispatchAndPoll
 };

package/backend/services/settingsService.js

@@ -26,11 +26,12 @@ const getProject = async () => {
 	return project;
 };
 
-const updateProject = async ({ name, logoUrl, timezone }) => {
+const updateProject = async ({ name, logoUrl, timezone, maxRetries }) => {
 	const data = {
 		name: name ?? '',
 		logoUrl: logoUrl ?? '',
-		...(timezone !== undefined && { timezone })
+		...(timezone !== undefined && { timezone }),
+		...(maxRetries !== undefined && { maxRetries: Number(maxRetries) || 0 })
 	};
 	const project = await prisma.project.upsert({
 		where: { id: 1 },