plum-e2e 2.9.17 → 2.12.0

package/backend/services/runExecutorService.js

@@ -0,0 +1,423 @@
+/*
+ * This file is part of Plum.
+ * Licensed under the MIT License. See LICENSE file in the project root for details.
+ */
+
+const { spawn } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const runnerService = require('./runnerService');
+const reportService = require('./reportService');
+const settingsService = require('./settingsService');
+const notificationService = require('./notificationService');
+const { startRRwebPoller } = require('../lib/rrwebPoller');
+const { runWithRetries } = require('../lib/retryRunner');
+const { BUILT_IN_RUNNER_ID, TRIGGER_REMOTE } = require('../constants/triggers');
+const { PLUM_MODE_NODE } = require('../constants/env');
+const { SOCKET_EVENTS } = require('../constants/socketEvents');
+const { JOB_STATUS, REPORT_STATUS } = require('../constants/jobStatus');
+const { getTestIdsForTag, chunkTests, buildTagExpression } = require('../lib/testChunker');
+const { getTestSuites } = require('./testService');
+const { readCucumberReportFile } = require('../lib/reportFilename');
+
+const BACKEND_DIR = path.resolve(__dirname, '..');
+const CANCEL_CODE = 130;
+
+// runId → live handles, so cancel() can stop every process and remote job a run owns.
+const inflight = new Map();
+
+function handles(runId) {
+	let rec = inflight.get(runId);
+	if (!rec) {
+		rec = { procs: new Set(), remoteJobs: [], cancelled: false };
+		inflight.set(runId, rec);
+	}
+	return rec;
+}
+
+function isCancelled(runId) {
+	return inflight.get(runId)?.cancelled === true;
+}
+
+/** Best-effort stop of everything a run owns — local child processes and remote node jobs. */
+async function cancel(runId) {
+	const rec = inflight.get(runId);
+	if (!rec) return false;
+	rec.cancelled = true;
+	for (const proc of rec.procs) {
+		try {
+			proc.kill('SIGTERM');
+		} catch {}
+	}
+	await Promise.allSettled(
+		rec.remoteJobs.map(({ runnerId, jobId }) => runnerService.cancelRemoteJob(runnerId, jobId))
+	);
+	return true;
+}
+
+// Stand-in Cucumber JSON for a lane that never produced a real report (process
+// crashed, node unreachable) so its scenarios still show as failed by name
+// instead of silently vanishing from the combined report.
+function makeSyntheticFailReport(laneName, testIds, reason) {
+	const nameMap = {};
+	try {
+		const { suites } = getTestSuites();
+		for (const suite of suites) {
+			for (const test of suite.tests) {
+				for (const id of Array.isArray(test.id) ? test.id : [test.id]) {
+					nameMap[id] = test.testCase;
+				}
+			}
+		}
+	} catch {}
+
+	return JSON.stringify([
+		{
+			id: 'runner-error',
+			uri: 'runner-error',
+			name: `Runner: ${laneName}`,
+			keyword: 'Feature',
+			elements: testIds.map((id) => ({
+				id: id.replace(/^@/, '').toLowerCase(),
+				name: nameMap[id] || id.replace(/^@/, ''),
+				keyword: 'Scenario',
+				type: 'scenario',
+				tags: [{ name: id.startsWith('@') ? id : `@${id}` }],
+				steps: [
+					{
+						keyword: 'Given ',
+						name: 'the scenario was assigned to this runner',
+						result: {
+							status: 'failed',
+							error_message: `Runner "${laneName}" did not complete: ${reason}`,
+							duration: 0
+						}
+					}
+				]
+			}))
+		}
+	]);
+}
+
+// ---------------------------------------------------------------------------
+// Built-in (local) attempt
+// ---------------------------------------------------------------------------
+
+// Always spawned with PLUM_MODE=node so generate-report.js skips its own DB
+// write — this service persists exactly one report per run itself. The queue
+// serialises every run that targets the built-in runner, so the shared
+// reports/cucumber_report.json is never contended.
+function spawnBuiltInAttempt({
+	runId,
+	laneId,
+	tag,
+	workers,
+	browser,
+	testRunId,
+	baseUrl,
+	onLog,
+	io
+}) {
+	return new Promise((resolve) => {
+		const ssDir = path.join(os.tmpdir(), `plum-ss-${runId}-${laneId}-${Date.now()}`);
+		fs.mkdirSync(ssDir, { recursive: true });
+
+		const env = {
+			...process.env,
+			TAG: tag,
+			TRIGGER: TRIGGER_REMOTE,
+			BROWSER: browser,
+			REPORT_RUNNERS: String(workers),
+			PLUM_MODE: PLUM_MODE_NODE,
+			PLUM_SS_DIR: ssDir
+		};
+		if (Number(workers) > 1) env.PARALLEL = String(workers);
+		if (testRunId) env.TEST_RUN_ID = testRunId;
+		if (baseUrl) env.BASE_URL = baseUrl;
+
+		const proc = spawn('npm', ['run', 'test'], { env, shell: true, cwd: BACKEND_DIR });
+		handles(runId).procs.add(proc);
+
+		const ssPoller = startRRwebPoller(ssDir, (batch) => {
+			io && io.emit(SOCKET_EVENTS.BG_RUN_LANE_RRWEB_BATCH, { runId, id: laneId, ...batch });
+		});
+
+		proc.stdout.on('data', (d) => onLog(d.toString()));
+		proc.stderr.on('data', (d) => onLog(`[ERROR] ${d.toString()}`));
+
+		proc.on('close', (code) => {
+			ssPoller.stop();
+			fs.rm(ssDir, { recursive: true, force: true }, () => {});
+			handles(runId).procs.delete(proc);
+			resolve({ code, raw: readCucumberReportFile() });
+		});
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Single built-in runner
+// ---------------------------------------------------------------------------
+
+async function runBuiltIn(run, io, emit) {
+	const startedAt = Date.now();
+	const { maxRetries } = await settingsService.getProject();
+	const laneId = BUILT_IN_RUNNER_ID;
+
+	let logBuffer = '';
+	const onLog = (text) => {
+		logBuffer += text;
+		emit(SOCKET_EVENTS.BG_RUN_LANE_LOG, { laneId, log: text });
+	};
+
+	emit(SOCKET_EVENTS.BG_RUN_LANES_INIT, {
+		lanes: [{ id: laneId, name: 'Built-in', testCount: getTestIdsForTag(run.tag).length }]
+	});
+
+	const { code, rawJson, attempts } = await runWithRetries({
+		maxRetries,
+		spawnAttempt: async (tagOverride) => {
+			const { code, raw } = await spawnBuiltInAttempt({
+				runId: run.id,
+				laneId,
+				tag: tagOverride ?? run.tag,
+				workers: run.workers,
+				browser: run.browser,
+				testRunId: run.testRunId,
+				baseUrl: run.baseUrl,
+				onLog,
+				io
+			});
+			return { code, rawJson: raw ? JSON.parse(raw) : [] };
+		},
+		onLog
+	});
+
+	const cancelled = isCancelled(run.id) || code === CANCEL_CODE;
+	emit(SOCKET_EVENTS.BG_RUN_LANE_STATUS, {
+		laneId,
+		status: cancelled ? JOB_STATUS.CANCELLED : code === 0 ? JOB_STATUS.DONE : JOB_STATUS.ERROR
+	});
+
+	if (cancelled) return { code: CANCEL_CODE, reportId: null };
+
+	const report = await reportService.saveReport({
+		rawCucumberJson: rawJson,
+		tags: run.tag,
+		triggerType: run.triggerType,
+		workerCount: run.workers,
+		browser: run.browser,
+		testRunId: run.testRunId ?? null,
+		logs: logBuffer || null,
+		duration: Date.now() - startedAt,
+		attempts
+	});
+
+	io && io.emit(SOCKET_EVENTS.REPORT_READY);
+	await maybeNotify(run, report);
+	return { code: report.status === REPORT_STATUS.PASS ? 0 : 1, reportId: report.id };
+}
+
+// ---------------------------------------------------------------------------
+// Distributed (multi-runner) path
+// ---------------------------------------------------------------------------
+
+async function runDistributed(run, io, emit, laneInfos, chunks) {
+	const startedAt = Date.now();
+	const { maxRetries } = await settingsService.getProject();
+
+	emit(SOCKET_EVENTS.BG_RUN_LANES_INIT, {
+		lanes: laneInfos.map((l, i) => ({ id: l.id, name: l.name, testCount: chunks[i].length }))
+	});
+
+	const total = laneInfos.length;
+	const collectedReports = new Array(total).fill(null);
+	const laneAttempts = new Array(total).fill(null);
+	const laneLogs = {};
+	for (const l of laneInfos) laneLogs[l.id] = '';
+
+	const laneResults = await Promise.all(
+		laneInfos.map((lane, i) => runLane(run, io, emit, lane, chunks[i], maxRetries, laneLogs))
+	);
+
+	let overallCode = 0;
+	laneResults.forEach((res, i) => {
+		if (res.code !== 0) overallCode = res.code;
+		collectedReports[i] = res.content;
+		laneAttempts[i] = res.attempts;
+		emit(SOCKET_EVENTS.BG_RUN_LANE_STATUS, {
+			laneId: laneInfos[i].id,
+			status: res.code === 0 ? JOB_STATUS.DONE : JOB_STATUS.ERROR
+		});
+	});
+
+	if (isCancelled(run.id)) return { code: CANCEL_CODE, reportId: null };
+
+	const saved = await reportService.saveCombinedReport({
+		reports: collectedReports,
+		runners: laneInfos,
+		workers: run.workers,
+		overallCode,
+		tag: run.tag,
+		triggerType: run.triggerType,
+		browser: run.browser,
+		testRunId: run.testRunId ?? null,
+		laneLogs,
+		duration: Date.now() - startedAt,
+		attemptsByLane: laneAttempts
+	});
+
+	io && io.emit(SOCKET_EVENTS.REPORT_READY);
+	await maybeNotify(run, saved);
+	return { code: saved.status === REPORT_STATUS.PASS ? 0 : 1, reportId: saved.id };
+}
+
+function runLane(run, io, emit, lane, chunkIds, maxRetries, laneLogs) {
+	const laneId = lane.id;
+	const chunkTag = buildTagExpression(chunkIds);
+	const onLog = (log) => {
+		laneLogs[laneId] += log;
+		emit(SOCKET_EVENTS.BG_RUN_LANE_LOG, { laneId, log });
+	};
+
+	const attempt =
+		lane.id === BUILT_IN_RUNNER_ID
+			? (currentTag) =>
+					spawnBuiltInAttempt({
+						runId: run.id,
+						laneId,
+						tag: currentTag,
+						workers: run.workers,
+						browser: run.browser,
+						testRunId: run.testRunId,
+						baseUrl: run.baseUrl,
+						onLog,
+						io
+					}).then(({ code, raw }) => ({
+						code,
+						rawJson: JSON.parse(
+							raw ?? makeSyntheticFailReport(lane.name, chunkIds, 'process exited with error')
+						)
+					}))
+			: (currentTag) =>
+					new Promise((resolve) => {
+						runnerService.dispatchAndPoll(
+							laneId,
+							{
+								tags: currentTag,
+								browser: run.browser,
+								workers: run.workers,
+								baseUrl: run.baseUrl
+							},
+							onLog,
+							(code, content) =>
+								resolve({
+									code,
+									rawJson: JSON.parse(
+										content ??
+											makeSyntheticFailReport(
+												lane.name,
+												chunkIds,
+												'could not fetch report from runner'
+											)
+									)
+								}),
+							(batch) =>
+								io &&
+								io.emit(SOCKET_EVENTS.BG_RUN_LANE_RRWEB_BATCH, {
+									runId: run.id,
+									id: laneId,
+									...batch
+								}),
+							(jobId) => handles(run.id).remoteJobs.push({ runnerId: laneId, jobId })
+						);
+					});
+
+	return runWithRetries({
+		maxRetries,
+		spawnAttempt: (t) => attempt(t ?? chunkTag),
+		onLog
+	}).then(({ code, rawJson, attempts }) => ({
+		code,
+		content: JSON.stringify(rawJson),
+		attempts
+	}));
+}
+
+// ---------------------------------------------------------------------------
+// Shared
+// ---------------------------------------------------------------------------
+
+async function maybeNotify(run, report) {
+	if (!report || (!run.notifyDiscord && !run.notifySlack)) return;
+	try {
+		await notificationService.send({
+			jobName: run.label,
+			status: report.status,
+			content: report.content,
+			browser: run.browser,
+			tags: run.tag,
+			reportId: report.id,
+			notifyDiscord: run.notifyDiscord,
+			notifySlack: run.notifySlack
+		});
+	} catch (e) {
+		console.error(`[run-executor] Notification failed: ${e.message}`);
+	}
+}
+
+// Runs one queued job (a plain RunQueue row) to completion and resolves with
+// { code, reportId, note? }. Takes only plain fields — no closure — so the queue
+// can re-dispatch a persisted row after a server restart.
+async function execute(run, io) {
+	handles(run.id); // register before any await so an early cancel is seen
+	const emit = (event, extra) => io && io.emit(event, { runId: run.id, ...extra });
+
+	// Drop runner ids that no longer exist — a stale selection or a runner
+	// deleted while this job sat in the queue must not wedge it.
+	const validated = [];
+	for (const id of run.runnerIds) {
+		if (id === BUILT_IN_RUNNER_ID || (await runnerService.getById(id))) validated.push(id);
+	}
+	if (validated.length === 0) {
+		return { code: 0, reportId: null, note: 'Target runner no longer exists — run skipped.' };
+	}
+
+	emit(SOCKET_EVENTS.BG_RUN_START, {
+		kind: run.kind,
+		label: run.label,
+		meta: {
+			tag: run.tag,
+			workers: run.workers,
+			browser: run.browser,
+			startedBy: run.startedBy ?? null
+		}
+	});
+
+	try {
+		const isSingleBuiltIn = validated.length === 1 && validated[0] === BUILT_IN_RUNNER_ID;
+		if (isSingleBuiltIn) return await runBuiltIn(run, io, emit);
+
+		const allIds = getTestIdsForTag(run.tag);
+		const chunks = chunkTests(allIds, validated.length);
+		// Surplus runners beyond the non-empty chunk count would each re-run the
+		// full tag expression and duplicate scenarios — drop them.
+		const activeIds = validated.slice(0, chunks.length);
+		if (activeIds.length === 0) {
+			emit(SOCKET_EVENTS.BG_RUN_LOG, { log: 'No tests found matching the selected tag.\n' });
+			return { code: 0, reportId: null, note: 'No tests matched — run skipped.' };
+		}
+		const laneInfos = await Promise.all(
+			activeIds.map(async (id) => {
+				if (id === BUILT_IN_RUNNER_ID) return { id, name: 'Built-in', dbId: null };
+				const r = await runnerService.getById(id);
+				return { id, name: r?.name ?? id, dbId: r?.id ?? null };
+			})
+		);
+		return await runDistributed(run, io, emit, laneInfos, chunks);
+	} finally {
+		inflight.delete(run.id);
+	}
+}
+
+module.exports = { execute, cancel, isCancelled };

package/backend/services/runQueueService.js

@@ -0,0 +1,276 @@
+/*
+ * This file is part of Plum.
+ * Licensed under the MIT License. See LICENSE file in the project root for details.
+ */
+
+const { randomUUID } = require('crypto');
+const prisma = require('./prisma');
+const runExecutorService = require('./runExecutorService');
+const { BUILT_IN_RUNNER_ID } = require('../constants/triggers');
+const { DEFAULT_BROWSER } = require('../constants/defaults');
+const { SOCKET_EVENTS } = require('../constants/socketEvents');
+const { JOB_STATUS } = require('../constants/jobStatus');
+
+const QUEUED = 'queued';
+const RUNNING = 'running';
+const DONE = 'done';
+const CANCEL_CODE = 130;
+const PRUNE_MS = 60 * 60 * 1000;
+
+let _io = null;
+function setSocketIO(io) {
+	_io = io;
+}
+
+function parseRunnerIds(str) {
+	const ids = (str || '')
+		.split(',')
+		.map((s) => s.trim())
+		.filter(Boolean);
+	return ids.length > 0 ? ids : [BUILT_IN_RUNNER_ID];
+}
+
+function normaliseRunnerIds(arr) {
+	return Array.isArray(arr) && arr.length > 0 ? arr : [BUILT_IN_RUNNER_ID];
+}
+
+function rowToJob(row) {
+	return {
+		id: row.id,
+		kind: row.kind,
+		triggerType: row.triggerType,
+		label: row.label,
+		tag: row.tag,
+		workers: row.workers,
+		browser: row.browser,
+		runnerIds: parseRunnerIds(row.runnerIds),
+		testRunId: row.testRunId,
+		baseUrl: row.baseUrl,
+		notifyDiscord: row.notifyDiscord,
+		notifySlack: row.notifySlack,
+		startedBy: row.startedBy
+	};
+}
+
+function meta(job) {
+	return {
+		tag: job.tag ?? '',
+		workers: Number(job.workers) > 1 ? Number(job.workers) : 1,
+		browser: job.browser ?? DEFAULT_BROWSER,
+		startedBy: job.startedBy ?? null
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Pump — decides which queued rows may start
+// ---------------------------------------------------------------------------
+
+let pumping = false;
+let pumpAgain = false;
+
+async function pump() {
+	if (pumping) {
+		pumpAgain = true;
+		return;
+	}
+	pumping = true;
+	try {
+		do {
+			pumpAgain = false;
+			await pumpOnce();
+		} while (pumpAgain);
+	} catch (e) {
+		console.error('[run-queue] pump failed:', e.message);
+	} finally {
+		pumping = false;
+	}
+}
+
+async function pumpOnce() {
+	const rows = await prisma.runQueue.findMany({
+		where: { status: { in: [QUEUED, RUNNING] } },
+		orderBy: { queuedAt: 'asc' }
+	});
+
+	// A runner is busy if a running row holds it. A queued row that can't start
+	// yet still reserves its runners so a later job on the same runner waits
+	// behind it (FIFO per runner) instead of jumping the line.
+	const busy = new Set();
+	for (const r of rows) {
+		if (r.status === RUNNING) parseRunnerIds(r.runnerIds).forEach((id) => busy.add(id));
+	}
+
+	for (const r of rows) {
+		if (r.status !== QUEUED) continue;
+		const ids = parseRunnerIds(r.runnerIds);
+		const blocked = ids.some((id) => busy.has(id));
+		ids.forEach((id) => busy.add(id));
+		if (blocked) continue;
+
+		await prisma.runQueue.update({
+			where: { id: r.id },
+			data: { status: RUNNING, startedAt: new Date() }
+		});
+		dispatch(r);
+	}
+}
+
+function dispatch(row) {
+	runExecutorService
+		.execute(rowToJob(row), _io)
+		.then((res) => finalise(row.id, res))
+		.catch((err) => {
+			console.error(`[run-queue] run ${row.id} threw:`, err);
+			finalise(row.id, { code: 1, reportId: null, note: err.message });
+		});
+}
+
+async function finalise(id, { code, reportId, note }) {
+	await prisma.runQueue.update({
+		where: { id },
+		data: {
+			status: DONE,
+			exitCode: code,
+			reportId: reportId ?? null,
+			note: note ?? null,
+			finishedAt: new Date()
+		}
+	});
+	if (_io) _io.emit(SOCKET_EVENTS.BG_RUN_DONE, { runId: id, code, reportId: reportId ?? null });
+	prune().catch(() => {});
+	pump();
+}
+
+async function prune() {
+	await prisma.runQueue.deleteMany({
+		where: { status: DONE, finishedAt: { lt: new Date(Date.now() - PRUNE_MS) } }
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+// Adds a run request to the queue and returns its id. It starts as soon as
+// every runner it targets is free; disjoint runner sets run in parallel.
+async function enqueue(job) {
+	const id = job.id || randomUUID();
+	const runnerIds = normaliseRunnerIds(job.runnerIds);
+
+	await prisma.runQueue.create({
+		data: {
+			id,
+			kind: job.kind,
+			triggerType: job.triggerType,
+			label: job.label ?? '',
+			status: QUEUED,
+			tag: job.tag ?? '',
+			workers: Number(job.workers) > 1 ? Number(job.workers) : 1,
+			browser: job.browser ?? DEFAULT_BROWSER,
+			runnerIds: runnerIds.join(','),
+			testRunId: job.testRunId ?? null,
+			baseUrl: job.baseUrl ?? null,
+			runTitle: job.runTitle ?? null,
+			startedBy: job.startedBy ?? null,
+			notifyDiscord: job.notifyDiscord === true,
+			notifySlack: job.notifySlack === true
+		}
+	});
+
+	if (_io) {
+		_io.emit(SOCKET_EVENTS.BG_RUN_QUEUED, {
+			runId: id,
+			kind: job.kind,
+			label: job.label ?? '',
+			meta: meta(job),
+			runnerIds
+		});
+	}
+
+	pump();
+	return id;
+}
+
+// Stops a run in any state: a queued row is dropped, a running one has its local
+// processes and remote node jobs killed. Emits bg-run-done either way.
+async function cancel(id) {
+	const row = await prisma.runQueue.findUnique({ where: { id } });
+	if (!row || row.status === DONE) return false;
+
+	if (row.status === QUEUED) {
+		await prisma.runQueue.update({
+			where: { id },
+			data: {
+				status: DONE,
+				exitCode: CANCEL_CODE,
+				note: 'Cancelled before it started',
+				finishedAt: new Date()
+			}
+		});
+		if (_io) _io.emit(SOCKET_EVENTS.BG_RUN_DONE, { runId: id, code: CANCEL_CODE, reportId: null });
+		pump();
+		return true;
+	}
+
+	// Running — the executor's own promise then resolves with code 130 and
+	// finalise() emits bg-run-done.
+	await runExecutorService.cancel(id);
+	return true;
+}
+
+// Back-compat shape for the HTTP trigger poll (GET /trigger/:jobId).
+async function getJob(id) {
+	const row = await prisma.runQueue.findUnique({ where: { id } });
+	if (!row) return null;
+	let status = JOB_STATUS.RUNNING;
+	if (row.status === DONE) {
+		status =
+			row.exitCode === CANCEL_CODE
+				? JOB_STATUS.CANCELLED
+				: row.exitCode === 0
+					? JOB_STATUS.DONE
+					: JOB_STATUS.ERROR;
+	}
+	return {
+		status,
+		exitCode: row.exitCode,
+		reportId: row.reportId ?? null,
+		startedAt: (row.startedAt ?? row.queuedAt).getTime()
+	};
+}
+
+async function listActive() {
+	const rows = await prisma.runQueue.findMany({
+		where: { status: { in: [QUEUED, RUNNING] } },
+		orderBy: { queuedAt: 'asc' }
+	});
+	let queuePos = 0;
+	return rows.map((r) => ({
+		runId: r.id,
+		status: r.status,
+		kind: r.kind,
+		label: r.label,
+		runnerIds: parseRunnerIds(r.runnerIds),
+		position: r.status === QUEUED ? ++queuePos : 0,
+		meta: { tag: r.tag, workers: r.workers, browser: r.browser, startedBy: r.startedBy }
+	}));
+}
+
+async function init(io) {
+	setSocketIO(io);
+	// Running rows can only be stale here — this process just started, so their
+	// child processes died with the previous one.
+	await prisma.runQueue.updateMany({
+		where: { status: RUNNING },
+		data: {
+			status: DONE,
+			exitCode: 1,
+			note: 'Interrupted by a server restart',
+			finishedAt: new Date()
+		}
+	});
+	await prune();
+	await pump();
+}
+
+module.exports = { setSocketIO, enqueue, cancel, getJob, listActive, init };