unprint 0.17.6 → 0.17.7

package/README.md

@@ -218,7 +218,8 @@ Use Playwright with Chromium (experimental)
 
 Additional options
 * `control`: Async function to interface with Playwright page passed as argument
-* `scope`: Browser instance to (re)use, set to `null` to force new scope every request, default `main`.
+* `clientScope`: Browser instance to (re)use, set to `null` to force new scope every request, default `main`.
+* `clientRetirement`: Number of requests until a browser gets restarted for resource clean-up, default `20`.
 * `browser`: Options object passed to Playwright's `launch`.
 * `browser.headless`: Headless mode, set to `false` to launch visible browser, default `true`.
 * `context`: Options object passed to Playwright's `newContext`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.6",
+  "version": "0.17.7",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -1060,6 +1060,13 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 	if (clients.has(scopeKey)) {
 		const client = clients.get(scopeKey);
 
+		client.uses += 1;
+
+		if (client.uses >= (options.clientRetirement || 20)) {
+			client.retired = true;
+			clients.delete(scopeKey);
+		}
+
 		await client.launchers;
 
 		return client;
@@ -1082,7 +1089,14 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 	}));
 
 	const launchers = Promise.all([browserLauncher, contextLauncher]);
-	const client = { launchers };
+
+	const client = {
+		key: scopeKey,
+		launchers,
+		active: 0,
+		uses: 1,
+		retired: false,
+	};
 
 	if (scope) {
 		clients.set(scopeKey, client);
@@ -1139,7 +1153,7 @@ async function browserRequest(url, customOptions = {}) {
 	const options = merge.all([{
 		timeout: 1000,
 		extract: true,
-		scope: 'main',
+		client: 'main',
 		limiter: 'browser',
 		url,
 	}, globalOptions, customOptions]);
@@ -1160,8 +1174,11 @@ async function browserRequest(url, customOptions = {}) {
 	events.emit('requestInit', feedbackBase);
 
 	return limiter.schedule(async () => {
-		const { context, browser } = await getBrowserInstance(options.scope, options, useProxy);
-		const page = await context.newPage();
+		const client = await getBrowserInstance(options.client, options, useProxy);
+
+		client.active += 1;
+
+		const page = await client.context.newPage();
 
 		const res = await page.goto(url, {
 			...options.page,
@@ -1180,6 +1197,8 @@ async function browserRequest(url, customOptions = {}) {
 				statusText,
 			});
 
+			client.active -= 1;
+
 			return {
 				ok: false,
 				status,
@@ -1198,8 +1217,10 @@ async function browserRequest(url, customOptions = {}) {
 
 		if (customOptions.control) {
 			try {
-				control = await customOptions.control(page, { context, browser });
+				control = await customOptions.control(page, client);
 			} catch (error) {
+				client.active -= 1;
+
 				return {
 					ok: false,
 					controlError: error.message,
@@ -1218,17 +1239,20 @@ async function browserRequest(url, customOptions = {}) {
 
 		await page.close();
 
-		if (options.scope === null) {
-			// this browser won't be reused
-			await browser.close();
-		}
-
 		events.emit('requestSuccess', {
 			...feedbackBase,
 			status,
 			statusText,
 		});
 
+		client.active -= 1;
+
+		if (options.client === null // this browser is single-use
+			|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
+			// this browser won't be reused
+			await client.browser.close();
+		}
+
 		return curateResponse({
 			data,
 			status,

package/tests/browser.js

@@ -16,23 +16,20 @@ unprint.options({ // or unprint.options();
 
 async function initTest() {
 	// concurrency
-	await Promise.all([
-		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
-			browser: {
-				headless: false,
-			},
-		}),
-		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
-			browser: {
-				headless: false,
-			},
-		}),
-		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
+	await Promise.all(Array.from({ length: 20 }).map(async () => {
+		// await unprint.browser(`https://tools-httpstatus.pickup-services.com/${Math.random() < 0.2 ? '404' : '200'}?sleep=${Math.round(Math.random() * 500)}`, {
+		await unprint.browser(`https://tools-httpstatus.pickup-services.com/200?sleep=${Math.round(Math.random() * 5000)}`, {
+			// client: null,
+			interval: 100,
 			browser: {
-				headless: false,
+				headless: true,
 			},
-		}),
-	]);
+		});
+	}));
+
+	// console.log('Requests done, waiting...');
+
+	// 	await new Promise((resolve) => { setTimeout(() => resolve(), 60 * 60 * 1000); });
 
 	await Promise.all([
 		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=5000', {