unprint 0.17.3 → 0.17.5

package/README.md

@@ -18,6 +18,10 @@ unprint.options({
 			concurrency: 10,
 			interval: 10, // ms
 		},
+		browser: {
+			concurrency: 5,
+			interval: 20,
+		},
 		[hostname]: {
 			enable: true, // enabled by default
 			concurrency: 1,
@@ -215,9 +219,9 @@ Use Playwright with Chromium (experimental)
 Additional options
 * `control`: Async function to interface with Playwright page passed as argument
 * `scope`: Browser instance to (re)use, set to `null` to force new scope every request, default `main`.
-* `browser`: Options object passed to Playwright's `launch`, requires new scope.
+* `browser`: Options object passed to Playwright's `launch`.
 * `browser.headless`: Headless mode, set to `false` to launch visible browser, default `true`.
-* `context`: Options object passed to Playwright's `newContext`, requires new scope.
+* `context`: Options object passed to Playwright's `newContext`.
 * `page`: Options object passed to Playwright's `goto`.
 
 This requires you to install the Chromium executable:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.3",
+  "version": "0.17.5",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},
@@ -28,6 +28,7 @@
     "eslint-config-airbnb-base": "^15.0.0",
     "jsdom": "^17.0.0",
     "moment-timezone": "^0.5.34",
+    "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
     "srcset": "^4.0.0",
     "tunnel": "^0.0.6"

package/src/app.js

@@ -10,6 +10,7 @@ const axios = require('axios').default;
 const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
+const hashObject = require('object-hash');
 const srcset = require('srcset');
 
 const settings = {
@@ -21,6 +22,10 @@ const settings = {
 			interval: 10,
 			concurrency: 10,
 		},
+		browser: {
+			interval: 20,
+			concurrency: 5,
+		},
 	},
 };
 
@@ -988,7 +993,7 @@ function getLimiterValue(prop, options, hostname) {
 		return options.limits[hostname][prop];
 	}
 
-	return options.limits.default[prop];
+	return options.limits[options?.limiter || 'default'][prop];
 }
 
 function getLimiter(url, options) {
@@ -1020,7 +1025,6 @@ function setProxy(instance, options, url) {
 
 	if (options.proxy
 		&& options.proxy.enable !== false
-		&& options.proxy.use !== false // use is a local override for enable
 		&& (options.proxy.use
 		|| options.proxy.hostnames?.includes(hostname))
 	) {
@@ -1031,14 +1035,18 @@ function setProxy(instance, options, url) {
 			},
 		});
 
-		instance.defaults.httpAgent = proxyAgent;
-		instance.defaults.httpsAgent = proxyAgent;
+		if (instance) {
+			instance.defaults.httpAgent = proxyAgent;
+			instance.defaults.httpsAgent = proxyAgent;
+		}
 
 		return true;
 	}
 
-	instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
-	instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+	if (instance) {
+		instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
+		instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+	}
 
 	return false;
 }
@@ -1046,9 +1054,11 @@ function setProxy(instance, options, url) {
 const clients = new Map();
 
 /* eslint-enable no-param-reassign */
-async function getBrowserInstance(scope, options) {
-	if (clients.has(scope)) {
-		const client = clients.get(scope);
+async function getBrowserInstance(scope, options, useProxy = false) {
+	const scopeKey = `${scope}_${useProxy ? 'proxy' : 'direct'}_${options.browser ? hashObject(options.browser) : 'default'}_${options.context ? hashObject(options.context) : 'default'}`;
+
+	if (clients.has(scopeKey)) {
+		const client = clients.get(scopeKey);
 
 		await client.launchers;
 
@@ -1064,13 +1074,18 @@ async function getBrowserInstance(scope, options) {
 	const contextLauncher = browserLauncher.then((browser) => browser.newContext({
 		userAgent: 'unprint',
 		...options.context,
+		...(useProxy && {
+			proxy: {
+				server: `${options.proxy.host}:${options.proxy.port}`,
+			},
+		}),
 	}));
 
 	const launchers = Promise.all([browserLauncher, contextLauncher]);
 	const client = { launchers };
 
 	if (scope) {
-		clients.set(scope, client);
+		clients.set(scopeKey, client);
 	}
 
 	client.browser = await browserLauncher;
@@ -1125,22 +1140,27 @@ async function browserRequest(url, customOptions = {}) {
 		timeout: 1000,
 		extract: true,
 		scope: 'main',
+		limiter: 'browser',
 		url,
 	}, globalOptions, customOptions]);
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
+	const useProxy = setProxy(null, options, url);
 
 	const feedbackBase = {
 		url,
 		method: 'get',
 		interval,
 		concurrency,
-		isProxied: false,
+		isProxied: useProxy,
+		isBrowser: true,
 		options,
 	};
 
+	events.emit('requestInit', feedbackBase);
+
 	return limiter.schedule(async () => {
-		const { context, browser } = await getBrowserInstance(options.scope, options);
+		const { context, browser } = await getBrowserInstance(options.scope, options, useProxy);
 		const page = await context.newPage();
 
 		const res = await page.goto(url, {
@@ -1191,6 +1211,12 @@ async function browserRequest(url, customOptions = {}) {
 			await browser.close();
 		}
 
+		events.emit('requestSuccess', {
+			...feedbackBase,
+			status,
+			statusText,
+		});
+
 		return curateResponse({
 			data,
 			status,
@@ -1231,6 +1257,7 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		interval,
 		concurrency,
 		isProxied,
+		isBrowser: false,
 		options,
 	};
 

package/tests/browser.js

@@ -2,6 +2,18 @@
 
 const unprint = require('../src/app');
 
+unprint.options({ // or unprint.options();
+	proxy: {
+		enable: true,
+		use: false, // don't use for all requests by default
+		host: '192.168.1.25',
+		port: 8888,
+		hostnames: [
+			'tools-httpstatus.pickup-services.com',
+		],
+	},
+});
+
 async function initTest() {
 	// concurrency
 	await Promise.all([
@@ -41,17 +53,18 @@ async function initTest() {
 					headless: false,
 				},
 				async control(_page) {
-					//
+					// return new Promise((resolve) => { setTimeout(() => resolve(), 60000); });
 				},
 			});
 		}),
 	]);
 
 	const res = await unprint.browser('https://www.scrapingcourse.com/', {
-	// await unprint.browser('https://www.scrapingcourse.com/', {
-		headless: false,
+		browser: {
+			headless: false,
+		},
 		async control(_page) {
-			return 'test';
+			// await new Promise((resolve) => { setTimeout(() => resolve(), 60000); });
 		},
 	});