unprint 0.14.1 → 0.14.3

package/README.md

@@ -209,9 +209,24 @@ Returns
 
 ### Proxy
 ```javascript
-unprint.get({ // or unprint.options();
-	host: '127.0.0.1',
-	port: 8888,
+unprint.options({ // or unprint.options();
+	proxy: {
+		enable: true,
+		use: false, // don't use for all requests by default
+		host: '127.0.0.1',
+		port: 8888,
+		hostnames: [
+			'www.google.com',
+			'www.example.com',
+		],
+	}
+});
+
+unprint.get({
+	proxy: {
+		use: true, // use proxy for this request
+		// all other proxy options can be supplied here
+	},
 });
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.14.1",
+  "version": "0.14.3",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {

package/src/app.js

@@ -930,6 +930,36 @@ function getLimiter(url, options) {
 	};
 }
 
+/* eslint-disable no-param-reassign */
+function setProxy(instance, options, url) {
+	const { hostname } = new URL(url);
+
+	if (options.proxy
+		&& options.proxy.enable !== false
+		&& options.proxy.use !== false // use is a local override for enable
+		&& (options.proxy.use
+		|| options.proxy.hostnames?.includes(hostname))
+	) {
+		const proxyAgent = tunnel.httpsOverHttp({
+			proxy: {
+				host: options.proxy.host,
+				port: options.proxy.port,
+			},
+		});
+
+		instance.defaults.httpAgent = proxyAgent;
+		instance.defaults.httpsAgent = proxyAgent;
+
+		return true;
+	}
+
+	instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
+	instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+
+	return false;
+}
+/* eslint-enable no-param-reassign */
+
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
 		timeout: 1000,
@@ -939,16 +969,6 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 
-	const feedbackBase = {
-		url,
-		method,
-		interval,
-		concurrency,
-		options,
-	};
-
-	events.emit('requestInit', feedbackBase);
-
 	const instance = axios.create({
 		data: body,
 		validateStatus: null,
@@ -959,20 +979,18 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		// httpAgent: options.httpAgent || new http.Agent({ ...options.agent }),
 	});
 
-	if (options.proxy) {
-		const proxyAgent = tunnel.httpsOverHttp({
-			proxy: {
-				host: options.proxy.host,
-				port: options.proxy.port,
-			},
-		});
+	const isProxied = setProxy(instance, options, url);
 
-		instance.defaults.httpAgent = proxyAgent;
-		instance.defaults.httpsAgent = proxyAgent;
-	} else {
-		instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
-		instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
-	}
+	const feedbackBase = {
+		url,
+		method,
+		interval,
+		concurrency,
+		isProxied,
+		options,
+	};
+
+	events.emit('requestInit', feedbackBase);
 
 	const res = await limiter.schedule(async () => instance.get(url));
 

package/tests/init.js

@@ -18,12 +18,11 @@ async function initTest() {
 				interval: 100,
 			},
 		},
-		/*
 		proxy: {
 			host: '192.168.178.25',
 			port: 8888,
+			hostnames: ['127.0.0.2'],
 		},
-		*/
 	});
 
 	unprint.on('requestInit', (initData) => console.log('init', initData));