unprint 0.17.4 → 0.17.6

package/README.md

@@ -139,7 +139,7 @@ Returns the `src` from an image element (or any other specified target) as a str
 Return the contents of a `data-` attribute.
 
 #### Query a source set
-`query.sourceSet([selector], [options])` or `query.srcSet()`
+`query.sourceSet(selector, [property], [options])` or `query.srcSet()`
 
 Options:
 * `includeDescriptor`: Produce an array of `{ descriptor, url }` instead of URL strings.
@@ -219,9 +219,9 @@ Use Playwright with Chromium (experimental)
 Additional options
 * `control`: Async function to interface with Playwright page passed as argument
 * `scope`: Browser instance to (re)use, set to `null` to force new scope every request, default `main`.
-* `browser`: Options object passed to Playwright's `launch`, requires new scope.
+* `browser`: Options object passed to Playwright's `launch`.
 * `browser.headless`: Headless mode, set to `false` to launch visible browser, default `true`.
-* `context`: Options object passed to Playwright's `newContext`, requires new scope.
+* `context`: Options object passed to Playwright's `newContext`.
 * `page`: Options object passed to Playwright's `goto`.
 
 This requires you to install the Chromium executable:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.4",
+  "version": "0.17.6",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},
@@ -28,6 +28,7 @@
     "eslint-config-airbnb-base": "^15.0.0",
     "jsdom": "^17.0.0",
     "moment-timezone": "^0.5.34",
+    "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
     "srcset": "^4.0.0",
     "tunnel": "^0.0.6"

package/src/app.js

@@ -10,6 +10,7 @@ const axios = require('axios').default;
 const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
+const hashObject = require('object-hash');
 const srcset = require('srcset');
 
 const settings = {
@@ -1024,7 +1025,6 @@ function setProxy(instance, options, url) {
 
 	if (options.proxy
 		&& options.proxy.enable !== false
-		&& options.proxy.use !== false // use is a local override for enable
 		&& (options.proxy.use
 		|| options.proxy.hostnames?.includes(hostname))
 	) {
@@ -1035,14 +1035,18 @@ function setProxy(instance, options, url) {
 			},
 		});
 
-		instance.defaults.httpAgent = proxyAgent;
-		instance.defaults.httpsAgent = proxyAgent;
+		if (instance) {
+			instance.defaults.httpAgent = proxyAgent;
+			instance.defaults.httpsAgent = proxyAgent;
+		}
 
 		return true;
 	}
 
-	instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
-	instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+	if (instance) {
+		instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
+		instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+	}
 
 	return false;
 }
@@ -1050,9 +1054,11 @@ function setProxy(instance, options, url) {
 const clients = new Map();
 
 /* eslint-enable no-param-reassign */
-async function getBrowserInstance(scope, options) {
-	if (clients.has(scope)) {
-		const client = clients.get(scope);
+async function getBrowserInstance(scope, options, useProxy = false) {
+	const scopeKey = `${scope}_${useProxy ? 'proxy' : 'direct'}_${options.browser ? hashObject(options.browser) : 'default'}_${options.context ? hashObject(options.context) : 'default'}`;
+
+	if (clients.has(scopeKey)) {
+		const client = clients.get(scopeKey);
 
 		await client.launchers;
 
@@ -1068,13 +1074,18 @@ async function getBrowserInstance(scope, options) {
 	const contextLauncher = browserLauncher.then((browser) => browser.newContext({
 		userAgent: 'unprint',
 		...options.context,
+		...(useProxy && {
+			proxy: {
+				server: `${options.proxy.host}:${options.proxy.port}`,
+			},
+		}),
 	}));
 
 	const launchers = Promise.all([browserLauncher, contextLauncher]);
 	const client = { launchers };
 
 	if (scope) {
-		clients.set(scope, client);
+		clients.set(scopeKey, client);
 	}
 
 	client.browser = await browserLauncher;
@@ -1134,18 +1145,22 @@ async function browserRequest(url, customOptions = {}) {
 	}, globalOptions, customOptions]);
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
+	const useProxy = setProxy(null, options, url);
 
 	const feedbackBase = {
 		url,
 		method: 'get',
 		interval,
 		concurrency,
-		isProxied: false,
+		isProxied: useProxy,
+		isBrowser: true,
 		options,
 	};
 
+	events.emit('requestInit', feedbackBase);
+
 	return limiter.schedule(async () => {
-		const { context, browser } = await getBrowserInstance(options.scope, options);
+		const { context, browser } = await getBrowserInstance(options.scope, options, useProxy);
 		const page = await context.newPage();
 
 		const res = await page.goto(url, {
@@ -1182,7 +1197,19 @@ async function browserRequest(url, customOptions = {}) {
 		let control = null;
 
 		if (customOptions.control) {
-			control = await customOptions.control(page, { context, browser });
+			try {
+				control = await customOptions.control(page, { context, browser });
+			} catch (error) {
+				return {
+					ok: false,
+					controlError: error.message,
+					status,
+					statusText,
+					headers,
+					response: res,
+					res,
+				};
+			}
 		}
 
 		events.emit('controlSuccess', feedbackBase);
@@ -1196,6 +1223,12 @@ async function browserRequest(url, customOptions = {}) {
 			await browser.close();
 		}
 
+		events.emit('requestSuccess', {
+			...feedbackBase,
+			status,
+			statusText,
+		});
+
 		return curateResponse({
 			data,
 			status,
@@ -1236,6 +1269,7 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		interval,
 		concurrency,
 		isProxied,
+		isBrowser: false,
 		options,
 	};
 

package/tests/browser.js

@@ -2,6 +2,18 @@
 
 const unprint = require('../src/app');
 
+unprint.options({ // or unprint.options();
+	proxy: {
+		enable: true,
+		use: false, // don't use for all requests by default
+		host: '192.168.1.25',
+		port: 8888,
+		hostnames: [
+			'tools-httpstatus.pickup-services.com',
+		],
+	},
+});
+
 async function initTest() {
 	// concurrency
 	await Promise.all([
@@ -41,17 +53,18 @@ async function initTest() {
 					headless: false,
 				},
 				async control(_page) {
-					//
+					// return new Promise((resolve) => { setTimeout(() => resolve(), 60000); });
 				},
 			});
 		}),
 	]);
 
 	const res = await unprint.browser('https://www.scrapingcourse.com/', {
-	// await unprint.browser('https://www.scrapingcourse.com/', {
-		headless: false,
+		browser: {
+			headless: false,
+		},
 		async control(_page) {
-			return 'test';
+			// await new Promise((resolve) => { setTimeout(() => resolve(), 60000); });
 		},
 	});