unprint 0.17.2 → 0.17.4

package/README.md

@@ -18,6 +18,10 @@ unprint.options({
 			concurrency: 10,
 			interval: 10, // ms
 		},
+		browser: {
+			concurrency: 5,
+			interval: 20,
+		},
 		[hostname]: {
 			enable: true, // enabled by default
 			concurrency: 1,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.2",
+  "version": "0.17.4",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -21,6 +21,10 @@ const settings = {
 			interval: 10,
 			concurrency: 10,
 		},
+		browser: {
+			interval: 20,
+			concurrency: 5,
+		},
 	},
 };
 
@@ -988,7 +992,7 @@ function getLimiterValue(prop, options, hostname) {
 		return options.limits[hostname][prop];
 	}
 
-	return options.limits.default[prop];
+	return options.limits[options?.limiter || 'default'][prop];
 }
 
 function getLimiter(url, options) {
@@ -1048,25 +1052,34 @@ const clients = new Map();
 /* eslint-enable no-param-reassign */
 async function getBrowserInstance(scope, options) {
 	if (clients.has(scope)) {
-		return clients.get(scope);
+		const client = clients.get(scope);
+
+		await client.launchers;
+
+		return client;
 	}
 
-	const browser = await chromium.launch({
+	// if we await here, we create a race condition, and a second call to getBrowserInstance would launch another browser that will overwrite the first one
+	const browserLauncher = chromium.launch({
 		headless: true,
 		...options.browser,
 	});
 
-	const context = await browser.newContext({
+	const contextLauncher = browserLauncher.then((browser) => browser.newContext({
 		userAgent: 'unprint',
 		...options.context,
-	});
+	}));
 
-	const client = { context, browser };
+	const launchers = Promise.all([browserLauncher, contextLauncher]);
+	const client = { launchers };
 
 	if (scope) {
 		clients.set(scope, client);
 	}
 
+	client.browser = await browserLauncher;
+	client.context = await contextLauncher;
+
 	return client;
 }
 
@@ -1116,6 +1129,7 @@ async function browserRequest(url, customOptions = {}) {
 		timeout: 1000,
 		extract: true,
 		scope: 'main',
+		limiter: 'browser',
 		url,
 	}, globalOptions, customOptions]);
 

package/tests/browser.js

@@ -3,6 +3,25 @@
 const unprint = require('../src/app');
 
 async function initTest() {
+	// concurrency
+	await Promise.all([
+		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
+			browser: {
+				headless: false,
+			},
+		}),
+		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
+			browser: {
+				headless: false,
+			},
+		}),
+		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=500', {
+			browser: {
+				headless: false,
+			},
+		}),
+	]);
+
 	await Promise.all([
 		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=5000', {
 			browser: {