unprint 0.17.6 → 0.17.8

package/README.md

@@ -218,7 +218,8 @@ Use Playwright with Chromium (experimental)
 
 Additional options
 * `control`: Async function to interface with Playwright page passed as argument
-* `scope`: Browser instance to (re)use, set to `null` to force new scope every request, default `main`.
+* `clientScope`: Browser instance to (re)use, set to `null` to force new scope every request, default `main`.
+* `clientRetirement`: Number of requests until a browser gets restarted for resource clean-up, default `20`.
 * `browser`: Options object passed to Playwright's `launch`.
 * `browser.headless`: Headless mode, set to `false` to launch visible browser, default `true`.
 * `context`: Options object passed to Playwright's `newContext`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.6",
+  "version": "0.17.8",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -191,12 +191,18 @@ function queryContents(context, selector, customOptions) {
 	const options = {
 		...context.options,
 		trim: true,
+		filter: true,
 		...customOptions,
 	};
 
 	const targets = queryElements(context, selector, options);
+	const extractedContents = targets.map((target) => extractContent(target, options));
+
+	if (options.filter) {
+		return extractedContents.filter(Boolean);
+	}
 
-	return targets.map((target) => extractContent(target, options)).filter(Boolean);
+	return extractedContents;
 }
 
 function queryAttribute(context, selector, attribute, customOptions) {
@@ -277,15 +283,24 @@ function queryNumber(context, selector, customOptions) {
 }
 
 function queryNumbers(context, selector, customOptions) {
+	const options = {
+		filter: true,
+		...customOptions,
+	};
+
 	const numberStrings = queryContents(context, selector, customOptions);
 
 	if (!numberStrings) {
 		return null;
 	}
 
-	return numberStrings
-		.map((numberString) => extractNumber(numberString, customOptions))
-		.filter(Boolean);
+	const extractedNumbers = numberStrings.map((numberString) => extractNumber(numberString, customOptions));
+
+	if (options.filter) {
+		return extractedNumbers.filter(Boolean);
+	}
+
+	return extractedNumbers;
 }
 
 function queryHtml(context, selector, customOptions) {
@@ -1060,6 +1075,13 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 	if (clients.has(scopeKey)) {
 		const client = clients.get(scopeKey);
 
+		client.uses += 1;
+
+		if (client.uses >= (options.clientRetirement || 20)) {
+			client.retired = true;
+			clients.delete(scopeKey);
+		}
+
 		await client.launchers;
 
 		return client;
@@ -1082,7 +1104,14 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 	}));
 
 	const launchers = Promise.all([browserLauncher, contextLauncher]);
-	const client = { launchers };
+
+	const client = {
+		key: scopeKey,
+		launchers,
+		active: 0,
+		uses: 1,
+		retired: false,
+	};
 
 	if (scope) {
 		clients.set(scopeKey, client);
@@ -1139,7 +1168,7 @@ async function browserRequest(url, customOptions = {}) {
 	const options = merge.all([{
 		timeout: 1000,
 		extract: true,
-		scope: 'main',
+		client: 'main',
 		limiter: 'browser',
 		url,
 	}, globalOptions, customOptions]);
@@ -1160,8 +1189,11 @@ async function browserRequest(url, customOptions = {}) {
 	events.emit('requestInit', feedbackBase);
 
 	return limiter.schedule(async () => {
-		const { context, browser } = await getBrowserInstance(options.scope, options, useProxy);
-		const page = await context.newPage();
+		const client = await getBrowserInstance(options.client, options, useProxy);
+
+		client.active += 1;
+
+		const page = await client.context.newPage();
 
 		const res = await page.goto(url, {
 			...options.page,
@@ -1180,6 +1212,8 @@ async function browserRequest(url, customOptions = {}) {
 				statusText,
 			});
 
+			client.active -= 1;
+
 			return {
 				ok: false,
 				status,
@@ -1198,8 +1232,10 @@ async function browserRequest(url, customOptions = {}) {
 
 		if (customOptions.control) {
 			try {
-				control = await customOptions.control(page, { context, browser });
+				control = await customOptions.control(page, client);
 			} catch (error) {
+				client.active -= 1;
+
 				return {
 					ok: false,
 					controlError: error.message,
@@ -1218,17 +1254,20 @@ async function browserRequest(url, customOptions = {}) {
 
 		await page.close();
 
-		if (options.scope === null) {
-			// this browser won't be reused
-			await browser.close();
-		}
-
 		events.emit('requestSuccess', {
 			...feedbackBase,
 			status,
 			statusText,
 		});
 
+		client.active -= 1;
+
+		if (options.client === null // this browser is single-use
+			|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
+			// this browser won't be reused
+			await client.browser.close();
+		}
+
 		return curateResponse({
 			data,
 			status,

package/tests/browser.js

@@ -16,23 +16,20 @@ unprint.options({ // or unprint.options();
 
 async function initTest() {
 	// concurrency
-	await Promise.all([
-		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
-			browser: {
-				headless: false,
-			},
-		}),
-		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
-			browser: {
-				headless: false,
-			},
-		}),
-		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
+	await Promise.all(Array.from({ length: 20 }).map(async () => {
+		// await unprint.browser(`https://tools-httpstatus.pickup-services.com/${Math.random() < 0.2 ? '404' : '200'}?sleep=${Math.round(Math.random() * 500)}`, {
+		await unprint.browser(`https://tools-httpstatus.pickup-services.com/200?sleep=${Math.round(Math.random() * 5000)}`, {
+			// client: null,
+			interval: 100,
 			browser: {
-				headless: false,
+				headless: true,
 			},
-		}),
-	]);
+		});
+	}));
+
+	// console.log('Requests done, waiting...');
+
+	// 	await new Promise((resolve) => { setTimeout(() => resolve(), 60 * 60 * 1000); });
 
 	await Promise.all([
 		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=5000', {