plum-e2e 2.6.0 → 2.6.2

package/backend/app.js

@@ -22,7 +22,10 @@ const { SCREENSHOTS_DIR } = require('./lib/reportFilename');
 const app = express();
 
 app.use(cors({ origin: '*' }));
-app.use(express.json());
+// Dispatching a run to a node ships the whole tests/ tree (base64-encoded,
+// fixtures included) as one JSON body — Express's 100kb default 413s well
+// before a real test suite does.
+app.use(express.json({ limit: '500mb' }));
 
 // Serve screenshot files written during report processing
 app.use('/screenshots', express.static(SCREENSHOTS_DIR));

package/backend/lib/retryRunner.js

@@ -0,0 +1,61 @@
+/*
+ * This file is part of Plum.
+ *
+ * Plum is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Plum is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Plum. If not, see https://www.gnu.org/licenses/.
+ */
+
+const reportService = require('../services/reportService');
+
+/**
+ * Runs a suite attempt via the caller-supplied `spawnAttempt`, re-running only
+ * the scenarios that failed (up to `maxRetries` extra rounds) until either
+ * everything passes or retries are exhausted. Agnostic to how an attempt is
+ * actually executed — the same loop drives both a local `npm run test` spawn
+ * and a remote `dispatchAndPoll` call, since both reduce to
+ * `(tagOverride, round) => Promise<{ code, rawJson }>`.
+ *
+ * @param {{
+ *   maxRetries: number,
+ *   spawnAttempt: (tagOverride: string|null, round: number) => Promise<{ code: number, rawJson: object[] }>,
+ *   onLog: (text: string) => void
+ * }} opts
+ * @returns {Promise<{ code: number, rawJson: object[], attempts: Record<string, number> }>}
+ */
+async function runWithRetries({ maxRetries, spawnAttempt, onLog }) {
+	const accumulated = [];
+	const attempts = {};
+	let round = 1;
+	let tagOverride = null;
+	let code = 0;
+
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		const result = await spawnAttempt(tagOverride, round);
+		code = result.code;
+		reportService.mergeRawAttempt(accumulated, result.rawJson, round, attempts);
+
+		const failedIds = reportService.getFailedIdTags(result.rawJson);
+		if (failedIds.length === 0 || round > maxRetries) break;
+
+		onLog(
+			`\n[RETRY] ${failedIds.length} test(s) failed — retrying (attempt ${round + 1}/${maxRetries + 1})\n`
+		);
+		tagOverride = failedIds.join(' or ');
+		round++;
+	}
+
+	return { code, rawJson: accumulated, attempts };
+}
+
+module.exports = { runWithRetries };

package/backend/prisma/migrations/20260714070000_add_project_max_retries/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Project" ADD COLUMN "maxRetries" INTEGER NOT NULL DEFAULT 0;

package/backend/prisma/schema.prisma

@@ -102,6 +102,7 @@ model Project {
   backupLastRunAt    DateTime?
   backupLastStatus   String    @default("")
   mcpKey             String    @default("")
+  maxRetries         Int       @default(0)
 }
 
 model User {

package/backend/routes/node.routes.js

@@ -45,6 +45,26 @@ router.post('/shutdown', authGuard, (req, res) => {
 	setTimeout(() => process.exit(0), 200);
 });
 
+// Self-restart for node-mode processes: spawns a replacement using this
+// process's own env (RUNNER_ID/PORT/NODE_TOKEN), then exits. The replacement
+// binds the same port, retrying past the brief EADDRINUSE window left by this
+// process shutting down (see the listen retry in server.js).
+router.post('/restart', authGuard, (req, res) => {
+	if (process.env.PLUM_MODE !== 'node') {
+		return res.status(403).json({ error: 'Not a node runner' });
+	}
+	const id = process.env.RUNNER_ID;
+	if (!id) {
+		return res.status(400).json({ error: 'Runner has no RUNNER_ID — cannot self-restart' });
+	}
+	res.json({ ok: true });
+	setTimeout(() => {
+		const { startNode } = require('../lib/runnerProcess');
+		startNode({ id, port: process.env.PORT || '3001', token: process.env.NODE_TOKEN });
+		process.exit(0);
+	}, 200);
+});
+
 // Start a remote test job
 router.post('/execute', authGuard, (req, res) => {
 	const { tags, browser = 'chromium', workers = 1, tests = null, env: userEnv = {} } = req.body;

package/backend/routes/runners.routes.js

@@ -64,16 +64,7 @@ router.put('/:id', async (req, res) => {
 
 router.delete('/:id', async (req, res) => {
 	try {
-		const runner = await runnerService.getById(req.params.id);
-		if (runner) {
-			try {
-				await fetch(`${runner.url}/api/shutdown`, {
-					method: 'POST',
-					headers: { Authorization: `Bearer ${runner.token}` },
-					signal: AbortSignal.timeout(3000)
-				});
-			} catch {}
-		}
+		await runnerService.stop(req.params.id);
 		await runnerService.remove(req.params.id);
 		res.json({ message: 'Runner deleted' });
 	} catch (e) {
@@ -90,4 +81,22 @@ router.post('/:id/ping', async (req, res) => {
 	}
 });
 
+router.post('/:id/stop', async (req, res) => {
+	try {
+		const result = await runnerService.stop(req.params.id);
+		res.json(result);
+	} catch (e) {
+		res.status(500).json({ ok: false, error: e.message });
+	}
+});
+
+router.post('/:id/restart', async (req, res) => {
+	try {
+		const result = await runnerService.restart(req.params.id);
+		res.json(result);
+	} catch (e) {
+		res.status(500).json({ ok: false, error: e.message });
+	}
+});
+
 module.exports = router;

package/backend/routes/settings.routes.js

@@ -33,8 +33,8 @@ router.get('/project', async (req, res, next) => {
 
 router.post('/project', async (req, res, next) => {
 	try {
-		const { name, logoUrl, timezone } = req.body;
-		const project = await settingsService.updateProject({ name, logoUrl, timezone });
+		const { name, logoUrl, timezone, maxRetries } = req.body;
+		const project = await settingsService.updateProject({ name, logoUrl, timezone, maxRetries });
 		res.json(project);
 	} catch (e) {
 		next(e);

package/backend/routes/trigger.routes.js

@@ -26,6 +26,10 @@ const { jwtAuth } = require('../middleware/jwtAuth');
 const prisma = require('../services/prisma');
 const { startSsPoller } = require('../lib/screenshotPoller');
 const { TRIGGER_TYPE } = require('../constants/triggers');
+const settingsService = require('../services/settingsService');
+const reportService = require('../services/reportService');
+const { readCucumberReportFile } = require('../lib/reportFilename');
+const { runWithRetries } = require('../lib/retryRunner');
 
 const BACKEND_DIR = path.resolve(__dirname, '..');
 const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour
@@ -51,28 +55,12 @@ router.post('/', jwtAuth, async (req, res, next) => {
 
 		const { tag = '', browser = 'chromium', workers = 1, baseUrl, testRunId, source } = req.body;
 		const trigger = source === 'mcp' ? TRIGGER_TYPE.MCP : TRIGGER_TYPE.EXTERNAL;
+		const { maxRetries } = await settingsService.getProject();
 
 		const jobId = randomUUID();
 		const startedAt = Date.now();
 		jobs.set(jobId, { status: 'running', exitCode: null, reportId: null, startedAt });
 
-		const ssDir = path.join(os.tmpdir(), `plum-trigger-ss-${jobId}`);
-		fs.mkdirSync(ssDir, { recursive: true });
-
-		const env = {
-			...process.env,
-			TAG: tag,
-			TRIGGER: trigger,
-			BROWSER: browser,
-			REPORT_RUNNERS: String(workers),
-			PLUM_SS_DIR: ssDir
-		};
-		if (Number(workers) > 1) env.PARALLEL = String(workers);
-		if (testRunId) env.TEST_RUN_ID = testRunId;
-		if (baseUrl) env.BASE_URL = baseUrl;
-
-		const proc = spawn('npm', ['run', 'test'], { env, shell: true, cwd: BACKEND_DIR });
-
 		if (_io) {
 			_io.emit('bg-run-start', {
 				runId: jobId,
@@ -82,48 +70,108 @@ router.post('/', jwtAuth, async (req, res, next) => {
 			});
 		}
 
-		const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
-			if (_io) _io.emit('bg-run-screenshot', { runId: jobId, stepName, data });
-		});
-
-		proc.stdout.on('data', (d) => {
-			if (_io) _io.emit('bg-run-log', { runId: jobId, log: d.toString() });
-		});
-		proc.stderr.on('data', (d) => {
-			if (_io) _io.emit('bg-run-log', { runId: jobId, log: `[ERROR] ${d.toString()}` });
-		});
-
-		proc.on('close', async (code) => {
-			clearInterval(ssPoller);
-			fs.rm(ssDir, { recursive: true, force: true }, () => {});
-
-			let reportId = null;
-			try {
-				// Find the latest report created after this job started
-				const report = await prisma.report.findFirst({
-					where: { createdAt: { gte: new Date(startedAt) } },
-					orderBy: { createdAt: 'desc' },
-					select: { id: true, status: true }
+		const onLog = (text) => {
+			if (_io) _io.emit('bg-run-log', { runId: jobId, log: text });
+		};
+
+		function runAttempt(currentTag, suppressSave) {
+			return new Promise((resolve) => {
+				const ssDir = path.join(os.tmpdir(), `plum-trigger-ss-${jobId}-${Date.now()}`);
+				fs.mkdirSync(ssDir, { recursive: true });
+
+				const env = {
+					...process.env,
+					TAG: currentTag,
+					TRIGGER: trigger,
+					BROWSER: browser,
+					REPORT_RUNNERS: String(workers),
+					PLUM_SS_DIR: ssDir
+				};
+				if (Number(workers) > 1) env.PARALLEL = String(workers);
+				if (testRunId) env.TEST_RUN_ID = testRunId;
+				if (baseUrl) env.BASE_URL = baseUrl;
+				if (suppressSave) env.PLUM_MODE = 'node';
+
+				const proc = spawn('npm', ['run', 'test'], { env, shell: true, cwd: BACKEND_DIR });
+
+				const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
+					if (_io) _io.emit('bg-run-screenshot', { runId: jobId, stepName, data });
 				});
-				reportId = report?.id ?? null;
-				if (reportId) {
-					await prisma.report.update({
-						where: { id: reportId },
-						data: { duration: Date.now() - startedAt }
+
+				proc.stdout.on('data', (d) => onLog(d.toString()));
+				proc.stderr.on('data', (d) => onLog(`[ERROR] ${d.toString()}`));
+
+				proc.on('close', (code) => {
+					clearInterval(ssPoller);
+					fs.rm(ssDir, { recursive: true, force: true }, () => {});
+					resolve({ code, raw: suppressSave ? readCucumberReportFile() : null });
+				});
+			});
+		}
+
+		if (maxRetries === 0) {
+			runAttempt(tag, false).then(async ({ code }) => {
+				let reportId = null;
+				try {
+					// Find the latest report created after this job started
+					const report = await prisma.report.findFirst({
+						where: { createdAt: { gte: new Date(startedAt) } },
+						orderBy: { createdAt: 'desc' },
+						select: { id: true, status: true }
+					});
+					reportId = report?.id ?? null;
+					if (reportId) {
+						await prisma.report.update({
+							where: { id: reportId },
+							data: { duration: Date.now() - startedAt }
+						});
+					}
+					jobs.set(jobId, {
+						status: code === 130 ? 'cancelled' : 'done',
+						exitCode: code,
+						reportId,
+						startedAt
 					});
+				} catch {
+					jobs.set(jobId, { status: 'done', exitCode: code, reportId: null, startedAt });
+				}
+
+				if (_io) _io.emit('bg-run-done', { runId: jobId, code, reportId });
+			});
+		} else {
+			runWithRetries({
+				maxRetries,
+				spawnAttempt: async (tagOverride) => {
+					const { code, raw } = await runAttempt(tagOverride ?? tag, true);
+					return { code, rawJson: raw ? JSON.parse(raw) : [] };
+				},
+				onLog
+			}).then(async ({ code, rawJson, attempts }) => {
+				let reportId = null;
+				try {
+					const report = await reportService.saveReport({
+						rawCucumberJson: rawJson,
+						tags: tag,
+						triggerType: trigger,
+						browser,
+						testRunId: testRunId ?? null,
+						duration: Date.now() - startedAt,
+						attempts
+					});
+					reportId = report.id;
+					jobs.set(jobId, {
+						status: code === 130 ? 'cancelled' : 'done',
+						exitCode: code,
+						reportId,
+						startedAt
+					});
+				} catch {
+					jobs.set(jobId, { status: 'done', exitCode: code, reportId: null, startedAt });
 				}
-				jobs.set(jobId, {
-					status: code === 130 ? 'cancelled' : 'done',
-					exitCode: code,
-					reportId,
-					startedAt
-				});
-			} catch {
-				jobs.set(jobId, { status: 'done', exitCode: code, reportId: null, startedAt });
-			}
 
-			if (_io) _io.emit('bg-run-done', { runId: jobId, code, reportId });
-		});
+				if (_io) _io.emit('bg-run-done', { runId: jobId, code, reportId });
+			});
+		}
 
 		res.status(202).json({ jobId, status: 'running' });
 	} catch (e) {

package/backend/scripts/manage-runners.mjs

@@ -88,6 +88,17 @@ async function deleteRunner(id) {
 	}
 }
 
+/**
+ * Stops/restarts a runner over the network via the primary's control routes,
+ * which hit the runner's own /api/shutdown|restart endpoints — works for any
+ * reachable runner, not just ones whose process this manager owns by PID.
+ */
+async function controlRunner(id, action) {
+	const res = await fetch(`${API_URL}/runners/${id}/${action}`, { method: 'POST' });
+	const body = await res.json().catch(() => ({}));
+	if (!res.ok || body.ok === false) throw new Error(body.error || `HTTP ${res.status}`);
+}
+
 /**
  * Resolves the display + control state for every runner: reachability (ping),
  * whether we own a live process for it, and whether we can control it at all.
@@ -156,12 +167,8 @@ function prepareNodeEnv() {
 async function runAction(r) {
 	const options = [];
 
-	if (!r.local) {
-		clack.log.info(
-			pc.dim(`"${r.name}" runs on another machine — it can be pinged but not controlled here.`)
-		);
-		options.push({ value: 'ping', label: 'Ping' });
-	} else if (r.managed) {
+	if (r.managed) {
+		// Local, and this manager owns its process — control it directly by PID.
 		options.push(
 			{ value: 'stop', label: pc.red('Stop') },
 			{ value: 'restart', label: pc.yellow('Restart') },
@@ -169,14 +176,18 @@ async function runAction(r) {
 			{ value: 'ping', label: 'Ping' }
 		);
 	} else if (r.online) {
-		clack.log.info(
-			pc.dim(
-				`"${r.name}" is up but was not started by this manager — stop it from its own terminal.`
-			)
+		// Remote, or local but started outside this manager (no PID to own) —
+		// either way the runner's own /api/shutdown|restart endpoints are
+		// reachable over the network via the primary's control routes.
+		options.push(
+			{ value: 'stop', label: pc.red('Stop') },
+			{ value: 'restart', label: pc.yellow('Restart') },
+			{ value: 'ping', label: 'Ping' }
 		);
-		options.push({ value: 'ping', label: 'Ping' });
-	} else {
+	} else if (r.local) {
 		options.push({ value: 'start', label: pc.green('Start') }, { value: 'ping', label: 'Ping' });
+	} else {
+		options.push({ value: 'ping', label: 'Ping' });
 	}
 
 	options.push(
@@ -194,15 +205,39 @@ async function runAction(r) {
 		const entry = startNode({ id: r.id, port, token: r.token });
 		clack.log.success(pc.green(`Started "${r.name}" on port ${port} (pid ${entry.pid})`));
 	} else if (action === 'stop') {
-		const ok = stopNode(r.id);
-		clack.log.success(ok ? pc.green(`Stopped "${r.name}"`) : pc.dim(`"${r.name}" was not running`));
+		if (r.managed) {
+			const ok = stopNode(r.id);
+			clack.log.success(
+				ok ? pc.green(`Stopped "${r.name}"`) : pc.dim(`"${r.name}" was not running`)
+			);
+		} else {
+			const s = clack.spinner();
+			s.start(`Stopping "${r.name}"...`);
+			try {
+				await controlRunner(r.id, 'stop');
+				s.stop(pc.green(`Stopped "${r.name}"`));
+			} catch (e) {
+				s.stop(pc.red(`Could not stop "${r.name}": ${e.message}`));
+			}
+		}
 	} else if (action === 'restart') {
-		const s = clack.spinner();
-		s.start(`Restarting "${r.name}"...`);
-		stopNode(r.id);
-		await new Promise((resolve) => setTimeout(resolve, 600));
-		const entry = startNode({ id: r.id, port, token: r.token });
-		s.stop(pc.green(`Restarted "${r.name}" (pid ${entry.pid})`));
+		if (r.managed) {
+			const s = clack.spinner();
+			s.start(`Restarting "${r.name}"...`);
+			stopNode(r.id);
+			await new Promise((resolve) => setTimeout(resolve, 600));
+			const entry = startNode({ id: r.id, port, token: r.token });
+			s.stop(pc.green(`Restarted "${r.name}" (pid ${entry.pid})`));
+		} else {
+			const s = clack.spinner();
+			s.start(`Restarting "${r.name}"...`);
+			try {
+				await controlRunner(r.id, 'restart');
+				s.stop(pc.green(`Restarted "${r.name}"`));
+			} catch (e) {
+				s.stop(pc.red(`Could not restart "${r.name}": ${e.message}`));
+			}
+		}
 	} else if (action === 'log') {
 		const entry = runnerProcess.loadRegistry()[r.id];
 		clack.note(entry?.logFile ?? '(no log file)', 'Log file');

package/backend/server.js

@@ -40,6 +40,20 @@ if (!fs.existsSync(testsDir)) {
 const isNodeMode = process.env.PLUM_MODE === 'node';
 const port = parseInt(process.env.PORT || '3001', 10);
 
+// A self-restart (POST /api/restart) spawns the replacement before this
+// process has released the port, so the first bind attempt can briefly hit
+// EADDRINUSE — retry instead of dying immediately.
+let listenRetriesLeft = 20;
+server.on('error', (err) => {
+	if (err.code === 'EADDRINUSE' && listenRetriesLeft > 0) {
+		listenRetriesLeft -= 1;
+		setTimeout(() => server.listen(port), 250);
+	} else {
+		console.error(`❌ Failed to bind port ${port}:`, err.message);
+		process.exit(1);
+	}
+});
+
 let cronService = null;
 let backupCronService = null;
 if (!isNodeMode) {
@@ -77,8 +91,13 @@ async function start() {
 				const cleanup = () => {
 					try {
 						const reg = loadRegistry();
-						delete reg[runnerId];
-						saveRegistry(reg);
+						// A self-restart already wrote the replacement's pid under this
+						// id before this process exits — only clear the entry if it's
+						// still ours, so we don't erase the new process's registration.
+						if (reg[runnerId]?.pid === process.pid) {
+							delete reg[runnerId];
+							saveRegistry(reg);
+						}
 					} catch {}
 				};
 				process.once('SIGTERM', cleanup);