unprint 0.13.2 → 0.14.0

package/README.md

@@ -207,6 +207,14 @@ Returns
 }
 ```
 
+### Proxy
+```javascript
+unprint.get({ // or unprint.options();
+	host: '127.0.0.1',
+	port: 8888,
+});
+```
+
 ### Feedback events
 Usage:
 * `unprint.on('trigger', callbackFn)`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.13.2",
+  "version": "0.14.0",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {
@@ -28,6 +28,7 @@
     "eslint": "^8.17.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-base": "^15.0.0",
+    "https-proxy-agent": "^7.0.5",
     "jsdom": "^17.0.0",
     "moment-timezone": "^0.5.34"
   },

package/src/app.js

@@ -5,6 +5,7 @@ const EventEmitter = require('events');
 const http = require('http');
 const https = require('https');
 const axios = require('axios').default;
+const { HttpsProxyAgent } = require('https-proxy-agent');
 const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
@@ -803,7 +804,6 @@ function initQueryFns(fns, context) {
 			events.emit('query', {
 				key,
 				args,
-				origin: context.options.origin,
 			});
 
 			return fn(...args);
@@ -816,7 +816,6 @@ function initQueryFns(fns, context) {
 			events.emit('query', {
 				key,
 				args,
-				origin: context.options.origin,
 			});
 
 			return fn({
@@ -950,17 +949,31 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 
 	events.emit('requestInit', feedbackBase);
 
-	const res = await limiter.schedule(async () => axios({
-		url,
-		method,
+	const instance = axios.create({
 		data: body,
 		validateStatus: null,
-		...options,
+		headers: options.headers,
 		timeout: options.timeout,
 		signal: options.abortSignal,
-		httpAgent: options.httpAgent || new http.Agent({ ...options.agent }),
-		httpsAgent: options.httpsAgent || new https.Agent({ ...options.agent }),
-	}));
+		// ...options,
+		// httpAgent: options.httpAgent || new http.Agent({ ...options.agent }),
+	});
+
+	if (options.proxy) {
+		// instance.defaults.httpAgent = new HttpsProxyAgent(`http://${options.proxy.host}:${options.proxy.port}`, { ...options.agent });
+		instance.defaults.httpsAgent = options.httpsAgent || new HttpsProxyAgent(`http://${options.proxy.host}:${options.proxy.port}`, { ...options.agent });
+
+		instance.defaults.proxy = {
+			protocol: 'http',
+			host: options.proxy.host,
+			port: options.proxy.port,
+		};
+	} else {
+		instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
+		instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+	}
+
+	const res = await limiter.schedule(async () => instance.get(url));
 
 	if (!(res.status >= 200 && res.status < 300)) {
 		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${res.data}`), 'HTTP_NOT_OK');

package/tests/init.js

@@ -18,6 +18,12 @@ async function initTest() {
 				interval: 100,
 			},
 		},
+		/*
+		proxy: {
+			host: '192.168.178.25',
+			port: 8888,
+		},
+		*/
 	});
 
 	unprint.on('requestInit', (initData) => console.log('init', initData));