plum-e2e 2.5.14 → 2.6.1

package/README.md

@@ -71,7 +71,7 @@ Full documentation is available at:
 | [Running Tests Locally](https://outline.silverlunah.com/s/12bf21d1-02ba-49e9-b0df-908976407afd/doc/running-tests-locally-GGhFcqaAQ8)       | `plum run-test` flags, parallel runs, debugging tips          |
 | [Setting Up the Server](https://outline.silverlunah.com/s/12bf21d1-02ba-49e9-b0df-908976407afd/doc/setting-up-the-server-vj0Ab1kJVs)       | Production server setup, reverse proxy (Nginx/Caddy), Docker  |
 | [Setting Up Nodes](https://outline.silverlunah.com/s/12bf21d1-02ba-49e9-b0df-908976407afd/doc/setting-up-nodes-dtmekJGJia)                 | Runner nodes, systemd service, managing nodes                 |
-| [Integrations](https://outline.silverlunah.com/s/12bf21d1-02ba-49e9-b0df-908976407afd/doc/integrations-qfiqfmdP0j)                         | Discord & Slack webhook notifications                         |
+| [Integrations](https://outline.silverlunah.com/s/12bf21d1-02ba-49e9-b0df-908976407afd/doc/integrations-qfiqfmdP0j)                         | Discord & Slack webhook notifications, CI/external triggers   |
 | [Backup](https://outline.silverlunah.com/s/12bf21d1-02ba-49e9-b0df-908976407afd/doc/backup-RNNObJfct9)                                     | Backup strategy                                               |
 
 ---

package/backend/app.js

@@ -22,7 +22,10 @@ const { SCREENSHOTS_DIR } = require('./lib/reportFilename');
 const app = express();
 
 app.use(cors({ origin: '*' }));
-app.use(express.json());
+// Dispatching a run to a node ships the whole tests/ tree (base64-encoded,
+// fixtures included) as one JSON body — Express's 100kb default 413s well
+// before a real test suite does.
+app.use(express.json({ limit: '500mb' }));
 
 // Serve screenshot files written during report processing
 app.use('/screenshots', express.static(SCREENSHOTS_DIR));

package/backend/constants/triggers.js

@@ -24,7 +24,8 @@
 const TRIGGER_TYPE = Object.freeze({
 	MANUAL: 'manual-trigger',
 	CLI: 'command-line-trigger',
-	MCP: 'mcp-trigger'
+	MCP: 'mcp-trigger',
+	EXTERNAL: 'external-trigger'
 });
 
 /**
@@ -45,6 +46,7 @@ const NON_SCHEDULED_TRIGGERS = new Set([
 	TRIGGER_TYPE.MANUAL,
 	TRIGGER_TYPE.CLI,
 	TRIGGER_TYPE.MCP,
+	TRIGGER_TYPE.EXTERNAL,
 	TRIGGER_REMOTE,
 	'undefined'
 ]);

package/backend/lib/retryRunner.js

@@ -0,0 +1,61 @@
+/*
+ * This file is part of Plum.
+ *
+ * Plum is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Plum is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Plum. If not, see https://www.gnu.org/licenses/.
+ */
+
+const reportService = require('../services/reportService');
+
+/**
+ * Runs a suite attempt via the caller-supplied `spawnAttempt`, re-running only
+ * the scenarios that failed (up to `maxRetries` extra rounds) until either
+ * everything passes or retries are exhausted. Agnostic to how an attempt is
+ * actually executed — the same loop drives both a local `npm run test` spawn
+ * and a remote `dispatchAndPoll` call, since both reduce to
+ * `(tagOverride, round) => Promise<{ code, rawJson }>`.
+ *
+ * @param {{
+ *   maxRetries: number,
+ *   spawnAttempt: (tagOverride: string|null, round: number) => Promise<{ code: number, rawJson: object[] }>,
+ *   onLog: (text: string) => void
+ * }} opts
+ * @returns {Promise<{ code: number, rawJson: object[], attempts: Record<string, number> }>}
+ */
+async function runWithRetries({ maxRetries, spawnAttempt, onLog }) {
+	const accumulated = [];
+	const attempts = {};
+	let round = 1;
+	let tagOverride = null;
+	let code = 0;
+
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		const result = await spawnAttempt(tagOverride, round);
+		code = result.code;
+		reportService.mergeRawAttempt(accumulated, result.rawJson, round, attempts);
+
+		const failedIds = reportService.getFailedIdTags(result.rawJson);
+		if (failedIds.length === 0 || round > maxRetries) break;
+
+		onLog(
+			`\n[RETRY] ${failedIds.length} test(s) failed — retrying (attempt ${round + 1}/${maxRetries + 1})\n`
+		);
+		tagOverride = failedIds.join(' or ');
+		round++;
+	}
+
+	return { code, rawJson: accumulated, attempts };
+}
+
+module.exports = { runWithRetries };

package/backend/lib/screenshotPoller.js

@@ -0,0 +1,43 @@
+/*
+ * This file is part of Plum.
+ *
+ * Plum is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Plum is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Plum. If not, see https://www.gnu.org/licenses/.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+function startSsPoller(ssDir, onScreenshot) {
+	const seenFiles = new Set();
+	return setInterval(() => {
+		try {
+			const files = fs
+				.readdirSync(ssDir)
+				.filter((f) => f.endsWith('.ss.json'))
+				.sort();
+			for (const f of files) {
+				if (seenFiles.has(f)) continue;
+				seenFiles.add(f);
+				const filePath = path.join(ssDir, f);
+				const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+				onScreenshot(data);
+				try {
+					fs.unlinkSync(filePath);
+				} catch {}
+			}
+		} catch {}
+	}, 400);
+}
+
+module.exports = { startSsPoller };

package/backend/mcp/server.js

@@ -340,7 +340,14 @@ server.tool(
 			)
 	},
 	async ({ tag, browser, workers, baseUrl, testRunId }) => {
-		const { jobId } = await post('/trigger', { tag, browser, workers, baseUrl, testRunId });
+		const { jobId } = await post('/trigger', {
+			tag,
+			browser,
+			workers,
+			baseUrl,
+			testRunId,
+			source: 'mcp'
+		});
 
 		const job = await pollJob(jobId);
 

package/backend/prisma/migrations/20260713090000_add_report_duration/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Report" ADD COLUMN "duration" INTEGER;

package/backend/prisma/migrations/20260714070000_add_project_max_retries/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Project" ADD COLUMN "maxRetries" INTEGER NOT NULL DEFAULT 0;

package/backend/prisma/schema.prisma

@@ -68,6 +68,7 @@ model Report {
   testRun     TestRun?          @relation(fields: [testRunId], references: [id], onDelete: SetNull)
   content     Json
   logs        String?
+  duration    Int?
   createdAt   DateTime          @default(now())
   testHistory TestCaseHistory[]
 
@@ -101,6 +102,7 @@ model Project {
   backupLastRunAt    DateTime?
   backupLastStatus   String    @default("")
   mcpKey             String    @default("")
+  maxRetries         Int       @default(0)
 }
 
 model User {

package/backend/routes/node.routes.js

@@ -45,6 +45,26 @@ router.post('/shutdown', authGuard, (req, res) => {
 	setTimeout(() => process.exit(0), 200);
 });
 
+// Self-restart for node-mode processes: spawns a replacement using this
+// process's own env (RUNNER_ID/PORT/NODE_TOKEN), then exits. The replacement
+// binds the same port, retrying past the brief EADDRINUSE window left by this
+// process shutting down (see the listen retry in server.js).
+router.post('/restart', authGuard, (req, res) => {
+	if (process.env.PLUM_MODE !== 'node') {
+		return res.status(403).json({ error: 'Not a node runner' });
+	}
+	const id = process.env.RUNNER_ID;
+	if (!id) {
+		return res.status(400).json({ error: 'Runner has no RUNNER_ID — cannot self-restart' });
+	}
+	res.json({ ok: true });
+	setTimeout(() => {
+		const { startNode } = require('../lib/runnerProcess');
+		startNode({ id, port: process.env.PORT || '3001', token: process.env.NODE_TOKEN });
+		process.exit(0);
+	}, 200);
+});
+
 // Start a remote test job
 router.post('/execute', authGuard, (req, res) => {
 	const { tags, browser = 'chromium', workers = 1, tests = null, env: userEnv = {} } = req.body;

package/backend/routes/runners.routes.js

@@ -64,16 +64,7 @@ router.put('/:id', async (req, res) => {
 
 router.delete('/:id', async (req, res) => {
 	try {
-		const runner = await runnerService.getById(req.params.id);
-		if (runner) {
-			try {
-				await fetch(`${runner.url}/api/shutdown`, {
-					method: 'POST',
-					headers: { Authorization: `Bearer ${runner.token}` },
-					signal: AbortSignal.timeout(3000)
-				});
-			} catch {}
-		}
+		await runnerService.stop(req.params.id);
 		await runnerService.remove(req.params.id);
 		res.json({ message: 'Runner deleted' });
 	} catch (e) {
@@ -90,4 +81,22 @@ router.post('/:id/ping', async (req, res) => {
 	}
 });
 
+router.post('/:id/stop', async (req, res) => {
+	try {
+		const result = await runnerService.stop(req.params.id);
+		res.json(result);
+	} catch (e) {
+		res.status(500).json({ ok: false, error: e.message });
+	}
+});
+
+router.post('/:id/restart', async (req, res) => {
+	try {
+		const result = await runnerService.restart(req.params.id);
+		res.json(result);
+	} catch (e) {
+		res.status(500).json({ ok: false, error: e.message });
+	}
+});
+
 module.exports = router;

package/backend/routes/settings.routes.js

@@ -33,8 +33,8 @@ router.get('/project', async (req, res, next) => {
 
 router.post('/project', async (req, res, next) => {
 	try {
-		const { name, logoUrl, timezone } = req.body;
-		const project = await settingsService.updateProject({ name, logoUrl, timezone });
+		const { name, logoUrl, timezone, maxRetries } = req.body;
+		const project = await settingsService.updateProject({ name, logoUrl, timezone, maxRetries });
 		res.json(project);
 	} catch (e) {
 		next(e);

package/backend/routes/trigger.routes.js

@@ -16,13 +16,20 @@
  */
 
 const path = require('path');
+const fs = require('fs');
+const os = require('os');
 const { spawn } = require('child_process');
 const { randomUUID } = require('crypto');
 const express = require('express');
 const router = express.Router();
 const { jwtAuth } = require('../middleware/jwtAuth');
 const prisma = require('../services/prisma');
+const { startSsPoller } = require('../lib/screenshotPoller');
 const { TRIGGER_TYPE } = require('../constants/triggers');
+const settingsService = require('../services/settingsService');
+const reportService = require('../services/reportService');
+const { readCucumberReportFile } = require('../lib/reportFilename');
+const { runWithRetries } = require('../lib/retryRunner');
 
 const BACKEND_DIR = path.resolve(__dirname, '..');
 const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour
@@ -30,6 +37,11 @@ const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour
 // In-memory job store: jobId → { status, exitCode, reportId, startedAt }
 const jobs = new Map();
 
+let _io = null;
+function setSocketIO(io) {
+	_io = io;
+}
+
 function pruneOldJobs() {
 	const cutoff = Date.now() - JOB_TTL_MS;
 	for (const [id, job] of jobs) {
@@ -41,43 +53,125 @@ router.post('/', jwtAuth, async (req, res, next) => {
 	try {
 		pruneOldJobs();
 
-		const { tag = '', browser = 'chromium', workers = 1, baseUrl, testRunId } = req.body;
+		const { tag = '', browser = 'chromium', workers = 1, baseUrl, testRunId, source } = req.body;
+		const trigger = source === 'mcp' ? TRIGGER_TYPE.MCP : TRIGGER_TYPE.EXTERNAL;
+		const { maxRetries } = await settingsService.getProject();
 
 		const jobId = randomUUID();
 		const startedAt = Date.now();
 		jobs.set(jobId, { status: 'running', exitCode: null, reportId: null, startedAt });
 
-		const env = {
-			...process.env,
-			TAG: tag,
-			TRIGGER: TRIGGER_TYPE.MCP,
-			BROWSER: browser,
-			REPORT_RUNNERS: String(workers)
+		if (_io) {
+			_io.emit('bg-run-start', {
+				runId: jobId,
+				kind: trigger,
+				label: trigger === TRIGGER_TYPE.MCP ? 'MCP run' : 'External run',
+				meta: { tag, browser, workers }
+			});
+		}
+
+		const onLog = (text) => {
+			if (_io) _io.emit('bg-run-log', { runId: jobId, log: text });
 		};
-		if (Number(workers) > 1) env.PARALLEL = String(workers);
-		if (testRunId) env.TEST_RUN_ID = testRunId;
-		if (baseUrl) env.BASE_URL = baseUrl;
-
-		const proc = spawn('npm', ['run', 'test'], { env, shell: true, cwd: BACKEND_DIR });
-
-		proc.on('close', async (code) => {
-			try {
-				// Find the latest report created after this job started
-				const report = await prisma.report.findFirst({
-					where: { createdAt: { gte: new Date(startedAt) } },
-					orderBy: { createdAt: 'desc' },
-					select: { id: true, status: true }
+
+		function runAttempt(currentTag, suppressSave) {
+			return new Promise((resolve) => {
+				const ssDir = path.join(os.tmpdir(), `plum-trigger-ss-${jobId}-${Date.now()}`);
+				fs.mkdirSync(ssDir, { recursive: true });
+
+				const env = {
+					...process.env,
+					TAG: currentTag,
+					TRIGGER: trigger,
+					BROWSER: browser,
+					REPORT_RUNNERS: String(workers),
+					PLUM_SS_DIR: ssDir
+				};
+				if (Number(workers) > 1) env.PARALLEL = String(workers);
+				if (testRunId) env.TEST_RUN_ID = testRunId;
+				if (baseUrl) env.BASE_URL = baseUrl;
+				if (suppressSave) env.PLUM_MODE = 'node';
+
+				const proc = spawn('npm', ['run', 'test'], { env, shell: true, cwd: BACKEND_DIR });
+
+				const ssPoller = startSsPoller(ssDir, ({ stepName, data }) => {
+					if (_io) _io.emit('bg-run-screenshot', { runId: jobId, stepName, data });
 				});
-				jobs.set(jobId, {
-					status: code === 130 ? 'cancelled' : 'done',
-					exitCode: code,
-					reportId: report?.id ?? null,
-					startedAt
+
+				proc.stdout.on('data', (d) => onLog(d.toString()));
+				proc.stderr.on('data', (d) => onLog(`[ERROR] ${d.toString()}`));
+
+				proc.on('close', (code) => {
+					clearInterval(ssPoller);
+					fs.rm(ssDir, { recursive: true, force: true }, () => {});
+					resolve({ code, raw: suppressSave ? readCucumberReportFile() : null });
 				});
-			} catch {
-				jobs.set(jobId, { status: 'done', exitCode: code, reportId: null, startedAt });
-			}
-		});
+			});
+		}
+
+		if (maxRetries === 0) {
+			runAttempt(tag, false).then(async ({ code }) => {
+				let reportId = null;
+				try {
+					// Find the latest report created after this job started
+					const report = await prisma.report.findFirst({
+						where: { createdAt: { gte: new Date(startedAt) } },
+						orderBy: { createdAt: 'desc' },
+						select: { id: true, status: true }
+					});
+					reportId = report?.id ?? null;
+					if (reportId) {
+						await prisma.report.update({
+							where: { id: reportId },
+							data: { duration: Date.now() - startedAt }
+						});
+					}
+					jobs.set(jobId, {
+						status: code === 130 ? 'cancelled' : 'done',
+						exitCode: code,
+						reportId,
+						startedAt
+					});
+				} catch {
+					jobs.set(jobId, { status: 'done', exitCode: code, reportId: null, startedAt });
+				}
+
+				if (_io) _io.emit('bg-run-done', { runId: jobId, code, reportId });
+			});
+		} else {
+			runWithRetries({
+				maxRetries,
+				spawnAttempt: async (tagOverride) => {
+					const { code, raw } = await runAttempt(tagOverride ?? tag, true);
+					return { code, rawJson: raw ? JSON.parse(raw) : [] };
+				},
+				onLog
+			}).then(async ({ code, rawJson, attempts }) => {
+				let reportId = null;
+				try {
+					const report = await reportService.saveReport({
+						rawCucumberJson: rawJson,
+						tags: tag,
+						triggerType: trigger,
+						browser,
+						testRunId: testRunId ?? null,
+						duration: Date.now() - startedAt,
+						attempts
+					});
+					reportId = report.id;
+					jobs.set(jobId, {
+						status: code === 130 ? 'cancelled' : 'done',
+						exitCode: code,
+						reportId,
+						startedAt
+					});
+				} catch {
+					jobs.set(jobId, { status: 'done', exitCode: code, reportId: null, startedAt });
+				}
+
+				if (_io) _io.emit('bg-run-done', { runId: jobId, code, reportId });
+			});
+		}
 
 		res.status(202).json({ jobId, status: 'running' });
 	} catch (e) {
@@ -92,3 +186,4 @@ router.get('/:jobId', jwtAuth, (req, res) => {
 });
 
 module.exports = router;
+router.setSocketIO = setSocketIO;

package/backend/server.js

@@ -40,6 +40,20 @@ if (!fs.existsSync(testsDir)) {
 const isNodeMode = process.env.PLUM_MODE === 'node';
 const port = parseInt(process.env.PORT || '3001', 10);
 
+// A self-restart (POST /api/restart) spawns the replacement before this
+// process has released the port, so the first bind attempt can briefly hit
+// EADDRINUSE — retry instead of dying immediately.
+let listenRetriesLeft = 20;
+server.on('error', (err) => {
+	if (err.code === 'EADDRINUSE' && listenRetriesLeft > 0) {
+		listenRetriesLeft -= 1;
+		setTimeout(() => server.listen(port), 250);
+	} else {
+		console.error(`❌ Failed to bind port ${port}:`, err.message);
+		process.exit(1);
+	}
+});
+
 let cronService = null;
 let backupCronService = null;
 if (!isNodeMode) {
@@ -48,6 +62,7 @@ if (!isNodeMode) {
 	backupCronService = require('./services/backupCronService');
 	socketHandler(io);
 	cronService.setSocketIO(io);
+	require('./routes/trigger.routes').setSocketIO(io);
 }
 
 async function start() {
@@ -76,8 +91,13 @@ async function start() {
 				const cleanup = () => {
 					try {
 						const reg = loadRegistry();
-						delete reg[runnerId];
-						saveRegistry(reg);
+						// A self-restart already wrote the replacement's pid under this
+						// id before this process exits — only clear the entry if it's
+						// still ours, so we don't erase the new process's registration.
+						if (reg[runnerId]?.pid === process.pid) {
+							delete reg[runnerId];
+							saveRegistry(reg);
+						}
 					} catch {}
 				};
 				process.once('SIGTERM', cleanup);