plum-e2e 2.9.7 → 2.9.8

package/backend/lib/serverBootstrap.js

@@ -43,12 +43,10 @@ function wireRealtimeServices(io, isNodeMode) {
 	if (isNodeMode) return { cronService: null, backupCronService: null };
 
 	const socketHandler = require('../websockets/socketHandler.js');
-	const nodeSocketHandler = require('../websockets/nodeSocketHandler.js');
 	const cronService = require('../services/cronService');
 	const backupCronService = require('../services/backupCronService');
 
 	socketHandler(io);
-	nodeSocketHandler(io);
 	cronService.setSocketIO(io);
 	require('../routes/trigger.routes').setSocketIO(io);
 

package/backend/logs/runner-cmtcyrx2v0007o0017vy2cym0.log

@@ -0,0 +1,6 @@
+/*
+This file is part of Plum.
+Licensed under the MIT License. See LICENSE file in the project root for details.
+*/
+📂 Loading tests from: /Users/silverlunah/Projects/plum/backend/tests
+Backend running on port 3055 (node/runner mode)

package/backend/package-lock.json

@@ -26,8 +26,7 @@
 				"node-cron": "^3.0.3",
 				"picocolors": "^1.1.1",
 				"playwright": "1.50.1",
-				"socket.io": "^4.8.1",
-				"socket.io-client": "^4.8.3"
+				"socket.io": "^4.8.1"
 			},
 			"devDependencies": {
 				"@playwright/test": "1.50.1",
@@ -2385,40 +2384,6 @@
 				"node": ">=10.2.0"
 			}
 		},
-		"node_modules/engine.io-client": {
-			"version": "6.6.6",
-			"resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.6.6.tgz",
-			"integrity": "sha512-iY6QdftLQ9pyiPoX082bpf/u1UewnOaJrtJIF9T0++QB34lZrj0uP+Q/bj8AlUsAxqhnkTV2BS8SBZSxOmoV5Q==",
-			"license": "MIT",
-			"dependencies": {
-				"@socket.io/component-emitter": "~3.1.0",
-				"debug": "~4.4.1",
-				"engine.io-parser": "~5.2.1",
-				"ws": "~8.21.0",
-				"xmlhttprequest-ssl": "~2.1.1"
-			}
-		},
-		"node_modules/engine.io-client/node_modules/ws": {
-			"version": "8.21.3",
-			"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",
-			"integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==",
-			"license": "MIT",
-			"engines": {
-				"node": ">=10.0.0"
-			},
-			"peerDependencies": {
-				"bufferutil": "^4.0.1",
-				"utf-8-validate": ">=5.0.2"
-			},
-			"peerDependenciesMeta": {
-				"bufferutil": {
-					"optional": true
-				},
-				"utf-8-validate": {
-					"optional": true
-				}
-			}
-		},
 		"node_modules/engine.io-parser": {
 			"version": "5.2.3",
 			"resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz",
@@ -4328,21 +4293,6 @@
 				}
 			}
 		},
-		"node_modules/socket.io-client": {
-			"version": "4.8.3",
-			"resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.3.tgz",
-			"integrity": "sha512-uP0bpjWrjQmUt5DTHq9RuoCBdFJF10cdX9X+a368j/Ft0wmaVgxlrjvK3kjvgCODOMMOz9lcaRzxmso0bTWZ/g==",
-			"license": "MIT",
-			"dependencies": {
-				"@socket.io/component-emitter": "~3.1.0",
-				"debug": "~4.4.1",
-				"engine.io-client": "~6.6.1",
-				"socket.io-parser": "~4.2.4"
-			},
-			"engines": {
-				"node": ">=10.0.0"
-			}
-		},
 		"node_modules/socket.io-parser": {
 			"version": "4.2.4",
 			"resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz",
@@ -4980,14 +4930,6 @@
 				"node": ">=8.0"
 			}
 		},
-		"node_modules/xmlhttprequest-ssl": {
-			"version": "2.1.2",
-			"resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz",
-			"integrity": "sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==",
-			"engines": {
-				"node": ">=0.4.0"
-			}
-		},
 		"node_modules/yallist": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

package/backend/package.json

@@ -39,7 +39,6 @@
 		"node-cron": "^3.0.3",
 		"picocolors": "^1.1.1",
 		"playwright": "1.50.1",
-		"socket.io": "^4.8.1",
-		"socket.io-client": "^4.8.3"
+		"socket.io": "^4.8.1"
 	}
 }

package/backend/scripts/manage-runners.mjs

@@ -19,6 +19,7 @@ import * as clack from '@clack/prompts';
 import pc from 'picocolors';
 import runnerProcess from '../lib/runnerProcess.js';
 import nodeRegister from '../lib/nodeRegister.js';
+import globalRegistry from '../lib/globalRegistry.js';
 
 const {
 	isLocalUrl,
@@ -37,13 +38,22 @@ const API_URL = process.env.PLUM_API_URL || 'http://localhost:3001';
 
 const cancelled = (v) => clack.isCancel(v);
 
-// The mutating /runners routes accept a registered runner's own token in place
-// of an admin session — this manager has no browser/JWT to present, but a node
-// already gets a token at registration time (see backend/middleware/jwtAuth.js
-// / runnerOrAdmin.js), so reuse that instead of a separate credential.
-const localToken = loadNodeConfig(process.cwd()).token;
+// The mutating /runners routes take a registered runner's own token in place of
+// an admin session (runnerOrAdmin.js), and this manager has no JWT — so find
+// any node token on this machine, falling back to PLUM_MCP_KEY.
+function resolveAuthHeader() {
+	const cwdToken = loadNodeConfig(process.cwd()).token;
+	if (cwdToken) return { Authorization: `Bearer ${cwdToken}` };
+	for (const dir of globalRegistry.getInstalls('node')) {
+		const token = loadNodeConfig(dir).token;
+		if (token) return { Authorization: `Bearer ${token}` };
+	}
+	if (process.env.PLUM_MCP_KEY) return { Authorization: `ApiKey ${process.env.PLUM_MCP_KEY}` };
+	return {};
+}
+const AUTH_HEADER = resolveAuthHeader();
 function authHeaders() {
-	return localToken ? { Authorization: `Bearer ${localToken}` } : {};
+	return AUTH_HEADER;
 }
 
 /**
@@ -291,6 +301,26 @@ async function runAction(r) {
 	}
 }
 
+// Polls a node's own /api/ping until it reports mode:'node' or the timeout
+// elapses — used to confirm a runner actually serves before its registration
+// is kept (see addRunner).
+async function nodeIsUp(baseUrl, token, timeoutMs) {
+	const url = `${baseUrl.replace(/\/+$/, '')}/api/ping`;
+	const headers = token ? { Authorization: `Bearer ${token}` } : {};
+	const deadline = Date.now() + timeoutMs;
+	while (Date.now() < deadline) {
+		try {
+			const res = await fetch(url, { headers, signal: AbortSignal.timeout(3000) });
+			if (res.ok) {
+				const body = await res.json().catch(() => ({}));
+				if (body.ok && body.mode === 'node') return true;
+			}
+		} catch {}
+		await new Promise((r) => setTimeout(r, 750));
+	}
+	return false;
+}
+
 async function addRunner() {
 	const suggested = `node-${generateToken().slice(0, 6)}`;
 
@@ -301,12 +331,18 @@ async function addRunner() {
 	});
 	if (cancelled(name)) return;
 
-	const port = await clack.text({
-		message: 'Local port the node listens on',
-		placeholder: '3002',
-		defaultValue: '3002'
-	});
-	if (cancelled(port)) return;
+	let port;
+	for (;;) {
+		port = await clack.text({
+			message: 'Local port the node listens on',
+			placeholder: '3002',
+			defaultValue: '3002'
+		});
+		if (cancelled(port)) return;
+		const pid = findPidOnPort(Number(port));
+		if (!pid) break;
+		clack.log.warn(pc.yellow(`Port ${port} is already in use (pid ${pid}) — choose another.`));
+	}
 
 	const defToken = process.env.NODE_TOKEN || generateToken();
 	const token = await clack.text({
@@ -325,41 +361,75 @@ async function addRunner() {
 	if (cancelled(urlInput)) return;
 
 	const url = resolveNodeUrl(urlInput || defaultUrl);
+	const local = isLocalUrl(url);
 
 	const s = clack.spinner();
 	s.start(`Registering "${name}" with the primary...`);
-	let id;
+	let id, reused;
 	try {
-		const res = await registerWithPrimary({
+		({ id, reused } = await registerWithPrimary({
 			primary: API_URL,
 			name,
 			url,
 			token,
 			browser: 'chromium'
-		});
-		id = res.id;
+		}));
 		s.stop(
-			res.reused
-				? pc.green(`Reusing existing runner "${name}"`)
-				: pc.green(`Registered "${name}" (id ${id})`)
+			reused ? pc.green(`Reusing existing runner "${name}"`) : pc.green(`Registered "${name}"`)
 		);
 	} catch (e) {
 		s.stop(pc.red(`Could not register "${name}": ${e.message}`));
 		return;
 	}
 
-	prepareNodeEnv();
+	// Verify the node actually answers before keeping the registration — start a
+	// local one here, expect a remote one to already be up. A failed check rolls
+	// the registration back so no dead runner is left behind.
+	let entry = null;
+	let ok = false;
+	if (local) {
+		prepareNodeEnv();
+		entry = startNode({ id, port, token });
+		s.start(`Waiting for "${name}" to come up on port ${port}...`);
+		ok = await nodeIsUp(`http://localhost:${port}`, token, 20000);
+		s.stop(ok ? pc.green(`"${name}" is up (pid ${entry.pid})`) : pc.red(`"${name}" did not start`));
+	} else {
+		s.start(`Checking for a Plum node at ${url}...`);
+		ok = await nodeIsUp(url, token, 8000);
+		s.stop(ok ? pc.green(`"${name}" is reachable`) : pc.red(`No Plum node answered at ${url}`));
+	}
 
-	const startNow = await clack.confirm({ message: 'Start this runner now?' });
-	if (!cancelled(startNow) && startNow) {
-		const entry = startNode({ id, port, token });
-		clack.log.success(pc.green(`Started "${name}" on port ${port} (pid ${entry.pid})`));
+	if (ok) {
+		clack.log.success(pc.green(`Runner "${name}" is ready.`));
+		return;
 	}
+
+	if (entry) stopNode(id, Number(port));
+	if (!reused) {
+		try {
+			await deleteRunner(id);
+		} catch {}
+	}
+	if (entry?.logFile) clack.note(entry.logFile, 'Node log');
+	clack.log.warn(
+		pc.yellow(
+			reused
+				? `"${name}" stays registered but is unreachable — check the port/URL.`
+				: `"${name}" was not registered — check the port/URL and try again.`
+		)
+	);
 }
 
 async function main() {
 	clack.intro(pc.bgMagenta(pc.white('  🟣 Plum — Manage Runners  ')));
 
+	if (!AUTH_HEADER.Authorization) {
+		clack.log.warn(
+			'No runner credential found on this machine — stop / restart / delete will be rejected.\n' +
+				'Run this from a folder where you started a node, or export PLUM_MCP_KEY.'
+		);
+	}
+
 	for (;;) {
 		const s = clack.spinner();
 		s.start(`Loading runners from ${API_URL}...`);

package/backend/services/nodeExecutionService.js

@@ -8,7 +8,6 @@ const os = require('os');
 const path = require('path');
 const crypto = require('crypto');
 const { spawn } = require('child_process');
-const { io: ioClient } = require('socket.io-client');
 const { startRRwebPoller } = require('../lib/rrwebPoller');
 const { TRIGGER_REMOTE } = require('../constants/triggers');
 const { DEFAULT_BROWSER } = require('../constants/defaults');
@@ -24,40 +23,17 @@ function getJob(jobId) {
 	return jobs[jobId];
 }
 
-// Opens this node's own outbound connection back to the primary for this job
-// — logs/rrweb events are pushed the moment they happen instead of waiting to
-// be polled. Uses the same token this node already validates incoming HTTP
-// calls against (see authGuard), so there's no separate credential to manage.
-// Best-effort: if the primary is unreachable this way, the job still runs —
-// dispatchAndPoll falls back to draining logs from the HTTP poll when it
-// never registered a socket relay for this jobId.
-function connectPrimaryStream(primaryUrl, jobId) {
-	try {
-		const socket = ioClient(`${primaryUrl}/node-stream`, {
-			auth: { token: process.env.NODE_TOKEN },
-			reconnectionAttempts: 5,
-			timeout: 8000
-		});
-		socket.on('connect', () => socket.emit('join', jobId));
-		return socket;
-	} catch {
-		return null;
-	}
-}
-
 // Starts a remote test job dispatched from the primary server: materializes
-// any uploaded test files, spawns `npm run test`, and tracks logs for later
-// HTTP polling (see pollJob).
+// any uploaded test files, spawns `npm run test`, and tracks logs + rrweb
+// batches for HTTP polling (see pollJob).
 function startJob({
 	tags,
 	browser = DEFAULT_BROWSER,
 	workers = 1,
 	tests = null,
-	env: userEnv = {},
-	primaryUrl = null
+	env: userEnv = {}
 }) {
 	const jobId = crypto.randomUUID();
-	const primaryStream = primaryUrl ? connectPrimaryStream(primaryUrl, jobId) : null;
 
 	// path.resolve ensures absolute even if TMPDIR env var is set to a relative path
 	const tmpdir = path.resolve(os.tmpdir());
@@ -82,8 +58,7 @@ function startJob({
 	jobs[jobId] = {
 		status: JOB_STATUS.RUNNING,
 		logs: '',
-		// Drained by pollJob — the HTTP-poll fallback for a node with no reachable
-		// notifyPublicUrl to stream these back over a socket instead.
+		// Both drained by pollJob — the primary has no socket back to this node.
 		rrwebBatches: [],
 		exitCode: null,
 		startedAt: Date.now(),
@@ -112,23 +87,17 @@ function startJob({
 
 	const ssPoller = startRRwebPoller(ssDir, (batch) => {
 		jobs[jobId].rrwebBatches.push(batch);
-		primaryStream?.emit('rrweb-batch', batch);
 	});
 
 	const proc = spawn('npm', ['run', 'test'], { env, shell: true, cwd: BACKEND_DIR });
 	proc.stdout.on('data', (d) => {
-		const text = d.toString();
-		jobs[jobId].logs += text;
-		primaryStream?.emit('log', text);
+		jobs[jobId].logs += d.toString();
 	});
 	proc.stderr.on('data', (d) => {
-		const text = d.toString();
-		jobs[jobId].logs += text;
-		primaryStream?.emit('log', text);
+		jobs[jobId].logs += d.toString();
 	});
 	proc.on('close', (code) => {
 		ssPoller.stop();
-		primaryStream?.close();
 		jobs[jobId].status = code === 0 ? JOB_STATUS.DONE : JOB_STATUS.ERROR;
 		jobs[jobId].exitCode = code;
 

package/backend/services/runnerService.js

@@ -11,8 +11,6 @@ const { BUILT_IN_RUNNER_ID } = require('../constants/triggers');
 const { DEFAULT_BROWSER } = require('../constants/defaults');
 const { bearerHeader } = require('../lib/authHeader');
 const { JOB_STATUS } = require('../constants/jobStatus');
-const settingsService = require('./settingsService');
-const nodeStreamRegistry = require('./nodeStreamRegistry');
 
 // ---------------------------------------------------------------------------
 // Runner CRUD
@@ -209,7 +207,6 @@ async function dispatchAndPoll(
 	const finish = (code, content) => {
 		if (settled) return;
 		settled = true;
-		if (jobId) nodeStreamRegistry.unregisterRelay(jobId);
 		onDone(code, content);
 	};
 
@@ -220,12 +217,6 @@ async function dispatchAndPoll(
 		return;
 	}
 
-	// The node needs to know where to open its own socket back to us — without
-	// this it has no way to reach the primary, since today only the reverse
-	// (primary knowing each node's url) is configured.
-	const { notifyPublicUrl } = await settingsService.getWebhooks();
-	const primaryUrl = notifyPublicUrl ? notifyPublicUrl.replace(/\/$/, '') : null;
-
 	let jobId;
 	try {
 		const res = await fetch(`${runner.url}/api/execute`, {
@@ -239,8 +230,7 @@ async function dispatchAndPoll(
 				browser,
 				workers,
 				tests: collectTestFiles(),
-				env: loadTestEnv(process.cwd()),
-				...(primaryUrl ? { primaryUrl } : {})
+				env: loadTestEnv(process.cwd())
 			}),
 			signal: AbortSignal.timeout(10000)
 		});
@@ -254,15 +244,11 @@ async function dispatchAndPoll(
 
 	onLog(`Connected to runner "${runner.name}" — job ${jobId}\n`);
 
-	// With a primaryUrl configured, the node streams logs/rrweb events over its
-	// own socket instead — draining them from the poll too would duplicate them.
-	if (primaryUrl) {
-		nodeStreamRegistry.registerRelay(jobId, { onRRwebBatch, onLog });
-	}
-
 	let logOffset = 0;
 	let rrwebOffset = 0;
 	let polling = false;
+	// Tight interval: the live viewer reads logs and rrweb batches straight off
+	// this poll — primary→node, so nothing has to connect the other way.
 	const poll = setInterval(async () => {
 		if (polling) return;
 		polling = true;
@@ -277,15 +263,12 @@ async function dispatchAndPoll(
 			if (!res.ok) return;
 			const body = await res.json();
 
-			// Skip if a socket relay is active — it already pushed these live.
-			if (!primaryUrl) {
-				if (body.logs) {
-					onLog(body.logs);
-					logOffset += body.logs.length;
-				}
-				for (const batch of body.rrwebBatches ?? []) onRRwebBatch?.(batch);
-				rrwebOffset += body.rrwebBatches?.length ?? 0;
+			if (body.logs) {
+				onLog(body.logs);
+				logOffset += body.logs.length;
 			}
+			for (const batch of body.rrwebBatches ?? []) onRRwebBatch?.(batch);
+			rrwebOffset += body.rrwebBatches?.length ?? 0;
 
 			if (body.status === JOB_STATUS.DONE || body.status === JOB_STATUS.ERROR) {
 				clearInterval(poll);
@@ -297,7 +280,7 @@ async function dispatchAndPoll(
 		} finally {
 			polling = false;
 		}
-	}, 2500);
+	}, 1000);
 }
 
 module.exports = {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "plum-e2e",
-	"version": "2.9.7",
+	"version": "2.9.8",
 	"repository": {
 		"type": "git",
 		"url": "https://github.com/silverlunah/plum.git"

package/backend/services/nodeStreamRegistry.js

@@ -1,24 +0,0 @@
-/*
- * This file is part of Plum.
- * Licensed under the MIT License. See LICENSE file in the project root for details.
- */
-
-// In-memory jobId -> relay callbacks, so an incoming node socket (which only
-// knows its own jobId) can find where to forward live rrweb/log events —
-// registered by whichever dispatch call (socketHandler, cronService) started
-// this job and already knows the browser-facing emit target/laneId.
-const relays = new Map();
-
-function registerRelay(jobId, { onRRwebBatch, onLog }) {
-	relays.set(jobId, { onRRwebBatch, onLog });
-}
-
-function unregisterRelay(jobId) {
-	relays.delete(jobId);
-}
-
-function getRelay(jobId) {
-	return relays.get(jobId);
-}
-
-module.exports = { registerRelay, unregisterRelay, getRelay };

package/backend/websockets/nodeSocketHandler.js

@@ -1,40 +0,0 @@
-/*
- * This file is part of Plum.
- * Licensed under the MIT License. See LICENSE file in the project root for details.
- */
-
-const runnerService = require('../services/runnerService');
-const nodeStreamRegistry = require('../services/nodeStreamRegistry');
-
-// A separate namespace from the browser-facing default one — a node connects
-// with its own runner token (the same credential it already uses to call back
-// into the primary's HTTP control routes, see runnerService.isValidToken) and
-// announces the jobId it's streaming for, which nodeStreamRegistry maps back
-// to whichever dispatch call is waiting on it.
-function nodeSocketHandler(io) {
-	const nodeIo = io.of('/node-stream');
-
-	nodeIo.use(async (socket, next) => {
-		const token = socket.handshake.auth?.token;
-		if (await runnerService.isValidToken(token)) return next();
-		next(new Error('unauthorized'));
-	});
-
-	nodeIo.on('connection', (socket) => {
-		let jobId = null;
-
-		socket.on('join', (id) => {
-			jobId = id;
-		});
-
-		socket.on('rrweb-batch', (batch) => {
-			if (jobId) nodeStreamRegistry.getRelay(jobId)?.onRRwebBatch?.(batch);
-		});
-
-		socket.on('log', (text) => {
-			if (jobId) nodeStreamRegistry.getRelay(jobId)?.onLog?.(text);
-		});
-	});
-}
-
-module.exports = nodeSocketHandler;