unprint 0.19.2 → 0.19.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.19.2",
+  "version": "0.19.4",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "repository": {

package/src/app.js

@@ -1222,11 +1222,13 @@ const clients = new Map();
 /* eslint-enable no-param-reassign */
 async function getBrowserInstance(scope, options, useProxy = false) {
 	const scopeKey = `${scope}_${useProxy ? 'proxy' : 'direct'}_${options.browser ? hashObject(options.browser) : 'default'}_${options.context ? hashObject(options.context) : 'default'}`;
+	const now = new Date();
 
 	if (clients.has(scopeKey)) {
 		const client = clients.get(scopeKey);
 
 		client.uses += 1;
+		client.lastUsedAt = now;
 
 		if (client.uses >= (options.clientRetirement || 20)) {
 			client.retired = true;
@@ -1262,6 +1264,8 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 		active: 0,
 		uses: 1,
 		retired: false,
+		createdAt: now,
+		lastUsedAt: now,
 	};
 
 	if (scope) {
@@ -1274,12 +1278,18 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 	return client;
 }
 
+function getAllBrowsers() {
+	return clients;
+}
+
 async function closeAllBrowsers() {
 	const closingClients = Array.from(clients.values());
 
 	await Promise.all(closingClients.map(async (client) => {
 		await client.context.close();
 		await client.browser.close();
+
+		clients.delete(client.key);
 	}));
 
 	events.emit('browserClose', {
@@ -1290,20 +1300,26 @@ async function closeAllBrowsers() {
 	});
 }
 
-async function closeBrowser(client, options) {
+async function closeBrowser(client, options = {}) {
 	if (options.client === null // this browser is single-use
 		|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
 		// this browser won't be reused, browser close DOES NOT automatically close context https://github.com/microsoft/playwright/issues/15163
 		await client.context.close();
 		await client.browser.close();
 
+		clients.delete(client.key);
+
 		events.emit('browserClose', {
 			keys: [client.key],
 			active: client.active,
 			retired: !!client.retired,
 			clients: clients.size,
 		});
+
+		return true;
 	}
+
+	return false;
 }
 
 function getAgent(options, url) {
@@ -1392,7 +1408,7 @@ function useRemoteRequest(options, method) {
 			return true;
 		}
 
-		if (options.remote.methods.includes(method.toLowerCase())) {
+		if (options.remote.methods.includes(method.toLowerCase()) && options.useRemote !== false) {
 			return true;
 		}
 	}
@@ -1729,7 +1745,9 @@ const unprint = {
 	request,
 	browserRequest,
 	browser: browserRequest,
+	closeBrowser,
 	closeAllBrowsers,
+	getAllBrowsers,
 	initialize: init,
 	initializeAll: initAll,
 	init,

package/src/server.js

@@ -2,6 +2,7 @@
 
 const crypto = require('crypto');
 const express = require('express');
+const timers = require('timers/promises');
 
 require('dotenv').config();
 
@@ -79,6 +80,26 @@ async function handleRequest(req, res, unprint, method) {
 	});
 }
 
+async function monitorBrowsers(unprint) {
+	await timers.setTimeout(60_000);
+
+	const clients = unprint.getAllBrowsers();
+
+	const checkedClients = await Promise.all(Array.from(clients.values()).map(async (client) => {
+		if (new Date() - client.lastUsedAt > 300_000) { // 5 minute expiry
+			return unprint.closeBrowser(client, { client: null });
+		}
+
+		return false;
+	}));
+
+	const closedClients = checkedClients.filter(Boolean).length;
+
+	logger.info(`Closed ${closedClients}/${checkedClients.length} browsers`);
+
+	monitorBrowsers(unprint);
+}
+
 async function initServer(address, unprint) {
 	const app = express();
 	const addressComponents = typeof address === 'boolean' ? [] : String(address).split(':');
@@ -88,7 +109,7 @@ async function initServer(address, unprint) {
 
 	app.use(express.json());
 
-	app.use(async (req, res, next) => {
+	app.use(async (req, _res, next) => {
 		if (process.env.UNPRINT_KEY) {
 			if (process.env.UNPRINT_KEY.length !== req.headers['unprint-key']?.length
 			|| !crypto.timingSafeEqual(Buffer.from(process.env.UNPRINT_KEY, 'utf16le'), Buffer.from(req.headers['unprint-key'], 'utf16le'))) {
@@ -114,6 +135,8 @@ async function initServer(address, unprint) {
 
 		unprint.options(curateOptions(req.body));
 
+		logger.info('Configuration updated');
+
 		res.status(204).send();
 	});
 
@@ -134,6 +157,8 @@ async function initServer(address, unprint) {
 
 		logger.info(`Started unprint server on ${host}:${port}`);
 	});
+
+	monitorBrowsers(unprint);
 }
 
 module.exports = initServer;

package/tests/remote.js

@@ -13,9 +13,10 @@ unprint.configure({
 
 async function init() {
 	unprint.on('requestInit', (event) => console.log('INIT', event));
+	unprint.on('browserClose', (event) => console.log('CLOSE', event));
 
 	const res = await unprint.browser('https://www.google.com', {
-		useRemote: false,
+		useRemote: true,
 		async control(page) {
 			const form = await page.locator('form');