unprint 0.19.4 → 0.19.5

package/README.md

@@ -294,22 +294,27 @@ unprint.get({
 });
 ```
 
-### Request server
-You can run unprint as a server to execute web request remotely. This is mainly useful when you need expensive browser requests on clients with limited resources.
-If you don't need unprint-specific features, you should probably set up a dedicated proxy server instead.
+### Browser server
+You can run unprint as a Playwright browser server. This is particularly useful if the client does not have sufficient resources to run browsers locally. The browser server is not meant to replace a dedicated proxy.
 
 #### Server
 * Ensure optional dependencies are installed
-* `UNPRINT_KEY=[random] node src/app.js  --server [port|address:port]`
+* `node src/app.js --server [port|address:port]`
+
+Environment variables (optional, .env is supported):
+* `UNPRINT_HOST`: Server host, default `127.0.0.1:3333`
+* `UNPRINT_KEY`: Key required for authentication
+* `UNPRINT_MEMORY_LIMIT`: Memory threshold at which to cycle the browser in MB
+* `UNPRINT_LOG_LEVEL`: Verbosity of the console output, `info` (default), `debug`, `silly`, `warn`, `error`
 
 #### Client
 ```
 unprint.options({
 	remote: {
 		enabled: true,
-		address: '10.0.0.1:3333'
+		use: true, // use by default
+		address: 'ws://10.0.0.1:3333'
 		key: 'YOUR_UNPRINT_KEY',
-		methods: ['browser'], // browser, get, post
 	},
 });
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.19.4",
+  "version": "0.19.5",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "repository": {
@@ -26,15 +26,19 @@
     "eslint": "^8.17.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-base": "^15.0.0",
+    "express-ws": "^5.0.2",
     "jsdom": "^17.0.0",
     "minimist": "^1.2.8",
     "moment-timezone": "^0.5.34",
     "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
+    "pidtree": "^0.6.0",
+    "pidusage": "^4.0.1",
     "set-cookie-parser": "^3.0.1",
     "srcset": "^4.0.0",
     "tunnel": "^0.0.6",
-    "undici": "^7.18.2"
+    "undici": "^7.18.2",
+    "ws": "^8.19.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.56.1",

package/src/app.js

@@ -23,8 +23,8 @@ const settings = {
 	userAgent: 'unprint',
 	remote: {
 		enable: false,
-		address: 'http://127.0.0.1:3333/browser',
-		methods: ['browser'],
+		address: 'ws://127.0.0.1:3333/browser',
+		use: false,
 	},
 	limits: {
 		default: {
@@ -1219,9 +1219,21 @@ function curateResponse(res, data, options, { url, control, customOptions }) {
 /* eslint-disable no-param-reassign */
 const clients = new Map();
 
+async function getBrowserContext(browser, options, useProxy) {
+	return browser.newContext({
+		userAgent: options.browserUserAgent || options.userAgent,
+		...options.context,
+		...(useProxy && {
+			proxy: {
+				server: `${options.proxy.host}:${options.proxy.port}`,
+			},
+		}),
+	});
+}
+
 /* eslint-enable no-param-reassign */
-async function getBrowserInstance(scope, options, useProxy = false) {
-	const scopeKey = `${scope}_${useProxy ? 'proxy' : 'direct'}_${options.browser ? hashObject(options.browser) : 'default'}_${options.context ? hashObject(options.context) : 'default'}`;
+async function getBrowserInstance(scope, options, useProxy = false, useRemote = false) {
+	const scopeKey = `${scope}_${useRemote ? 'remote' : 'local'}_${useProxy ? 'proxy' : 'direct'}_${options.browser ? hashObject(options.browser) : 'default'}_${options.context ? hashObject(options.context) : 'default'}`;
 	const now = new Date();
 
 	if (clients.has(scopeKey)) {
@@ -1241,21 +1253,18 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 	}
 
 	// if we await here, we create a race condition, and a second call to getBrowserInstance would launch another browser that will overwrite the first one
-	const browserLauncher = chromium.launch({
-		headless: true,
-		...options.browser,
-	});
-
-	const contextLauncher = browserLauncher.then((browser) => browser.newContext({
-		userAgent: options.browserUserAgent || options.userAgent,
-		...options.context,
-		...(useProxy && {
-			proxy: {
-				server: `${options.proxy.host}:${options.proxy.port}`,
+	const browserLauncher = useRemote
+		? chromium.connect(options.remote.address, {
+			headers: {
+				'unprint-key': options.remote.key,
 			},
-		}),
-	}));
+		})
+		: chromium.launch({
+			headless: true,
+			...options.browser,
+		});
 
+	const contextLauncher = browserLauncher.then((browser) => getBrowserContext(browser, options, useProxy));
 	const launchers = Promise.all([browserLauncher, contextLauncher]);
 
 	const client = {
@@ -1264,6 +1273,7 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 		active: 0,
 		uses: 1,
 		retired: false,
+		isRemote: useRemote,
 		createdAt: now,
 		lastUsedAt: now,
 	};
@@ -1302,6 +1312,7 @@ async function closeAllBrowsers() {
 
 async function closeBrowser(client, options = {}) {
 	if (options.client === null // this browser is single-use
+		|| client.isRemote
 		|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
 		// this browser won't be reused, browser close DOES NOT automatically close context https://github.com/microsoft/playwright/issues/15163
 		await client.context.close();
@@ -1355,62 +1366,13 @@ function getAgent(options, url) {
 	});
 }
 
-async function remoteRequest(url, method, options, feedbackBase) {
-	const control = typeof options.control === 'function' ? options.control.toString() : null;
-
-	const res = await undici.fetch(`${options.remote.address}/request`, {
-		method: 'post',
-		body: JSON.stringify({
-			url,
-			method,
-			options: {
-				...options,
-				control: control && control.slice(control.indexOf('{') + 1, control.lastIndexOf('}')),
-			},
-		}),
-		headers: {
-			'content-type': 'application/json',
-			'unprint-key': options.remote.key,
-		},
-	});
-
-	if (res.status !== 200) {
-		return {
-			ok: false,
-			status: res.status,
-			statusText: res.statusText,
-		};
-	}
-
-	const body = await res.text();
-	const data = JSON.parse(body);
-
-	events.emit('requestSuccess', {
-		...feedbackBase,
-		status: data.status,
-		statusText: data.statusText,
-	});
-
-	return curateResponse({
-		status: data.status,
-		statusText: data.statusText,
-		headers: data.headers,
-	}, data.data, options, {
-		url,
-		customOptions: options,
-		control: data.control,
-	});
-}
-
-function useRemoteRequest(options, method) {
+function useRemoteRequest(options) {
 	if (options.remote.enable) {
-		if (options.useRemote) {
-			return true;
+		if (typeof options.useRemote === 'boolean') {
+			return options.useRemote;
 		}
 
-		if (options.remote.methods.includes(method.toLowerCase()) && options.useRemote !== false) {
-			return true;
-		}
+		return options.remote.use;
 	}
 
 	return false;
@@ -1427,7 +1389,8 @@ async function browserRequest(url, customOptions = {}) {
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 	const agent = getAgent(options, url);
-	const isRemote = useRemoteRequest(options, 'browser');
+	const useProxy = agent instanceof undici.ProxyAgent;
+	const useRemote = useRemoteRequest(options);
 
 	const feedbackBase = {
 		url,
@@ -1436,18 +1399,14 @@ async function browserRequest(url, customOptions = {}) {
 		concurrency,
 		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: true,
-		isRemote,
+		isRemote: useRemote,
 		options,
 	};
 
 	events.emit('requestInit', feedbackBase);
 
-	if (isRemote) {
-		return remoteRequest(url, 'browser', options, feedbackBase);
-	}
-
 	return limiter.schedule(async () => {
-		const client = await getBrowserInstance(options.client, options, agent instanceof undici.ProxyAgent);
+		const client = await getBrowserInstance(options.client, options, useProxy, useRemote);
 
 		events.emit('browserOpen', {
 			keys: [client.key],
@@ -1623,7 +1582,6 @@ async function request(url, customOptions = {}, redirects = 0) {
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 
 	const agent = getAgent(options, url);
-	const isRemote = useRemoteRequest(options, method);
 
 	const feedbackBase = {
 		url,
@@ -1632,15 +1590,17 @@ async function request(url, customOptions = {}, redirects = 0) {
 		concurrency,
 		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: false,
-		isRemote,
+		isRemote: false,
 		options,
 	};
 
 	events.emit('requestInit', feedbackBase);
 
+	/* only for browsers atm
 	if (isRemote) {
 		return remoteRequest(url, method, options, feedbackBase);
 	}
+	*/
 
 	const curatedBody = curateRequestBody(body, options);
 	const curatedCookie = getCookie(options);
@@ -1748,6 +1708,7 @@ const unprint = {
 	closeBrowser,
 	closeAllBrowsers,
 	getAllBrowsers,
+	getBrowserInstance,
 	initialize: init,
 	initializeAll: initAll,
 	init,
@@ -1769,7 +1730,7 @@ const unprint = {
 };
 
 if (argv.server) {
-	initServer(argv.server, unprint);
+	initServer(unprint);
 }
 
 module.exports = unprint;

package/src/server.js

@@ -1,119 +1,150 @@
 'use strict';
 
 const crypto = require('crypto');
+const os = require('os');
+const WebSocket = require('ws');
 const express = require('express');
+const expressWs = require('express-ws');
 const timers = require('timers/promises');
+const { chromium } = require('patchright');
+const pidUsage = require('pidusage');
+const pidTree = require('pidtree');
 
 require('dotenv').config();
 
 const pkg = require('../package.json');
 
+const memoryLimit = Number(process.env.UNPRINT_MEMORY_LIMIT) || Math.round(Math.max(1024, (os.totalmem() / 1024 / 1024) * 0.3)); // MB, aim for 30% of total available
+
 class HttpError extends Error {
-	constructor(message, httpCode, friendlyMessage, data) {
+	constructor(message, httpCode) {
 		super(message);
 
 		this.name = 'HttpError';
 		this.httpCode = httpCode;
-
-		if (friendlyMessage) {
-			this.friendlyMessage = friendlyMessage;
-		}
-
-		if (data) {
-			this.data = data;
-		}
 	}
 }
 
-function log(level, ...data) {
-	const now = new Date();
-
-	console.log(`${now.toISOString()} [${level.slice(0, 5).padStart(5, ' ')}] ${data.join(' ')}`);
-}
-
-const logger = Object.fromEntries([
-	'info',
-	'debug',
+const logLevels = [
 	'error',
 	'warn',
-].map((level) => [level, (...data) => log(level, ...data)]));
+	'info',
+	'debug',
+	'silly',
+];
 
-function curateOptions(options) {
-	// make sure remote unprint doesn't get configured to make request to itself
-	return {
-		...options,
-		remote: {
-			enable: false,
-		},
-		useRemote: false,
-		control: options.control
-			? async function control() {}.constructor('page', 'client', options.control) // eslint-disable-line no-eval,no-new-func,no-empty-function
-			: null,
-	};
-}
+const logLevel = process.env.UNPRINT_LOG_LEVEL || 'info';
+
+function log(level, ...data) {
+	if (logLevels.indexOf(level) <= logLevels.indexOf(logLevel)) {
+		const now = new Date();
 
-async function handleRequest(req, res, unprint, method) {
-	if (!req.body?.url) {
-		throw new HttpError('No URL provided', 400);
+		console.log(`${now.toISOString()} [${level.slice(0, 5).padStart(5, ' ')}] ${data.join(' ')}`);
 	}
+}
 
-	logger.info(`${(method || req.body.method || 'get').toLowerCase()} ${req.body.url}`);
+const logger = Object.fromEntries(logLevels.map((level) => [level, (...data) => log(level, ...data)]));
 
-	const options = curateOptions(req.body.options);
+function closeSocket(socket, code, reason) {
+	const safeCode = code >= 1000 && code <= 1015 && code !== 1006
+		? code
+		: 1000;
 
-	const unprintRes = await unprint.request(req.body.url, {
-		...options,
-		method: req.body.method,
-		body: req.body.data,
-	});
+	try {
+		socket.close(safeCode, reason);
+	} catch (error) {
+		// probably already closed
+	}
+}
 
-	res.send({
-		ok: unprintRes.ok,
-		status: unprintRes.status,
-		statusText: unprintRes.statusText,
-		data: unprintRes.data || null,
-		body: unprintRes.body || null,
-		html: unprintRes.context?.html || null,
-		headers: unprintRes.headers,
-		cookies: unprintRes.cookies,
-		control: unprintRes.control,
+async function getClient() {
+	const browser = await chromium.launchServer({
+		headless: false,
 	});
-}
 
-async function monitorBrowsers(unprint) {
-	await timers.setTimeout(60_000);
+	const endpoint = browser.wsEndpoint();
 
-	const clients = unprint.getAllBrowsers();
+	return {
+		browser,
+		endpoint,
+		active: 0,
+		isRetired: false,
+	};
+}
 
-	const checkedClients = await Promise.all(Array.from(clients.values()).map(async (client) => {
-		if (new Date() - client.lastUsedAt > 300_000) { // 5 minute expiry
-			return unprint.closeBrowser(client, { client: null });
-		}
+async function initServer() {
+	const app = express();
 
-		return false;
-	}));
+	const address = process.env.UNPRINT_HOST || '127.0.0.1:3333';
+	const [host, portString] = address.split(':');
+	const port = portString ? Number(portString) : 3333;
+
+	let client = await getClient();
+
+	async function monitorBrowser() {
+		await timers.setTimeout(60_000); // 1 minute
+
+		try {
+			const pid = client.browser.process().pid;
+			const pids = await pidTree(pid, { root: true });
+			const usages = await pidUsage(pids);
+			const memoryUsage = Math.round(Object.values(usages).reduce((acc, usage) => acc + usage.memory, 0) / 1024 / 1024); // MB
+
+			if (memoryUsage >= memoryLimit) {
+				logger.info(`Cycling browser at ${memoryUsage.toLocaleString()}MB / ${memoryLimit.toLocaleString()}MB`);
+
+				const retiredClient = client;
+				retiredClient.isRetired = true;
+
+				try {
+					client = await getClient();
+
+					if (retiredClient.active === 0) {
+						await retiredClient.browser.close();
+					} else {
+						// don't await promise timeout to maintain monitor loop pace
+						setTimeout(async () => {
+							if (retiredClient.active > 0) {
+								logger.warn(`Force closing retired browser with ${retiredClient.active} clients`);
+								await retiredClient.browser.close();
+							}
+						}, 300_000); // 5 minutes
+					}
+				} catch (error) {
+					logger.warn(`Failed to retire client: ${error.message}`);
+
+					retiredClient.isRetired = false;
+					client = retiredClient;
+				}
+			} else {
+				logger.debug(`Reusing browser at ${memoryUsage.toLocaleString()}MB / ${memoryLimit.toLocaleString()}MB`);
+			}
+		} catch (error) {
+			logger.info(`No browser PID, probably relaunching: ${error.message}`);
+		}
 
-	const closedClients = checkedClients.filter(Boolean).length;
+		monitorBrowser();
+	}
 
-	logger.info(`Closed ${closedClients}/${checkedClients.length} browsers`);
+	monitorBrowser();
 
-	monitorBrowsers(unprint);
-}
+	expressWs(app);
+	app.use(express.json());
 
-async function initServer(address, unprint) {
-	const app = express();
-	const addressComponents = typeof address === 'boolean' ? [] : String(address).split(':');
+	app.use(async (req, _res, next) => {
+		if (process.env.UNPRINT_KEY && req.path !== '/') {
+			if (!req.headers['unprint-key']) {
+				logger.warn(`Unauthenticated request from ${req.ip}`);
 
-	const host = addressComponents[1] ? addressComponents[0] : '127.0.0.1';
-	const port = addressComponents[1] || addressComponents[0] || 3000;
+				throw new HttpError('Missing key', 401);
+			}
 
-	app.use(express.json());
+			if (process.env.UNPRINT_KEY.length !== req.headers['unprint-key'].length || !crypto.timingSafeEqual(
+				Buffer.from(process.env.UNPRINT_KEY, 'utf16le'),
+				Buffer.from(req.headers['unprint-key'], 'utf16le'),
+			)) {
+				logger.warn(`Invalid key from ${req.ip}`);
 
-	app.use(async (req, _res, next) => {
-		if (process.env.UNPRINT_KEY) {
-			if (process.env.UNPRINT_KEY.length !== req.headers['unprint-key']?.length
-			|| !crypto.timingSafeEqual(Buffer.from(process.env.UNPRINT_KEY, 'utf16le'), Buffer.from(req.headers['unprint-key'], 'utf16le'))) {
-				logger.warn(`Invalid key used by ${req.ip}`);
 				throw new HttpError('Invalid key', 401);
 			}
 		}
@@ -125,21 +156,6 @@ async function initServer(address, unprint) {
 		res.send(`unprint ${pkg.version}`);
 	});
 
-	app.post('/request', async (req, res) => handleRequest(req, res, unprint));
-	app.post('/browser', async (req, res) => handleRequest(req, res, unprint, 'browser'));
-
-	app.post('/options', async (req, res) => {
-		if (!req.body) {
-			throw new HttpError('No options provided', 400);
-		}
-
-		unprint.options(curateOptions(req.body));
-
-		logger.info('Configuration updated');
-
-		res.status(204).send();
-	});
-
 	app.use((error, _req, res, _next) => {
 		logger.error(error);
 
@@ -149,16 +165,69 @@ async function initServer(address, unprint) {
 		});
 	});
 
+	app.ws('/browser', (clientSocket, _req) => {
+		const currentClient = client;
+		const browserSocket = new WebSocket(currentClient.endpoint);
+
+		let queue = [];
+
+		logger.info('Client connected');
+
+		currentClient.active += 1;
+
+		clientSocket.on('message', (data) => {
+			logger.silly(`Socket data (${browserSocket.readyState === WebSocket.OPEN ? 'sent' : 'queued'}): ${data}`);
+
+			if (browserSocket.readyState === WebSocket.OPEN) {
+				browserSocket.send(data);
+			} else {
+				queue.push(data);
+			}
+		});
+
+		browserSocket.on('open', () => {
+			logger.debug(`Browser connected, clearing ${queue.length} queue messages`);
+
+			queue.forEach((data) => browserSocket.send(data));
+			queue = [];
+
+			browserSocket.on('message', (data) => {
+				if (clientSocket.readyState === WebSocket.OPEN) {
+					clientSocket.send(data);
+				}
+			});
+		});
+
+		clientSocket.on('close', async (code, reason) => {
+			closeSocket(browserSocket, code, reason);
+
+			currentClient.active -= 1;
+
+			logger.info('Client disconnected');
+
+			if (currentClient.isRetired && currentClient.active === 0) {
+				await currentClient.browser.close();
+				logger.info('Browser retired');
+			}
+		});
+
+		browserSocket.on('close', (code, reason) => {
+			closeSocket(clientSocket, code, reason);
+			logger.debug('Browser disconnected');
+		});
+
+		clientSocket.on('error', (error) => logger.error(`Client error: ${error}`));
+		browserSocket.on('error', (error) => logger.error(`Browser error: ${error}`));
+	});
+
 	app.listen(port, host, (error) => {
 		if (error) {
 			logger.error(`Failed to start server: ${error.message}`);
 			return;
 		}
 
-		logger.info(`Started unprint server on ${host}:${port}`);
+		logger.info(`unprint server listening on http://${host}:${port}`);
 	});
-
-	monitorBrowsers(unprint);
 }
 
 module.exports = initServer;

package/tests/remote.js

@@ -1,13 +1,17 @@
 'use strict';
 
+// const { chromium } = require('patchright');
+
 const unprint = require('../src/app');
 
+const key = 'foobar';
+
 unprint.configure({
 	remote: {
 		enable: true,
-		address: 'http://127.0.0.1:3333',
-		key: 'foobar',
-		methods: [],
+		use: false,
+		address: 'ws://127.0.0.1:3333/browser',
+		key,
 	},
 });
 
@@ -18,9 +22,7 @@ async function init() {
 	const res = await unprint.browser('https://www.google.com', {
 		useRemote: true,
 		async control(page) {
-			const form = await page.locator('form');
-
-			return form.count();
+			return page.locator('form').count();
 		},
 	});
 
@@ -36,3 +38,29 @@ async function init() {
 }
 
 init();
+
+/*
+async function initRaw() {
+	const browser = await chromium.connect('ws://127.0.0.1:3333/browser', {
+		headers: {
+			'unprint-key': key,
+		},
+	});
+
+	// await timers.setTimeout(2000);
+
+	const context = await browser.newContext();
+	const page = await context.newPage();
+
+	await page.goto('https://jsonplaceholder.typicode.com');
+	await page.locator('.mb-one').hover({ trial: true, timeout: 10000, strict: false });
+
+	const content = await page.content();
+
+	console.log(content);
+
+	await page.close();
+}
+
+initRaw();
+*/