unprint 0.11.13 → 0.13.0

package/README.md

@@ -13,6 +13,17 @@ unprint.options({
 	headers: {
 		'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
 	},
+	limits: { // request throttling
+		default: {
+			concurrency: 10,
+			interval: 10, // ms
+		},
+		[hostname]: {
+			enable: true, // enabled by default
+			concurrency: 1,
+			interval: 1000,
+		},
+	},
 })
 ```
 
@@ -195,3 +206,13 @@ Returns
 	res,				// (object)		alias for 'response'
 }
 ```
+
+### Feedback events
+Usage:
+* `unprint.on('trigger', callbackFn)`
+* `unprint.off('trigger', callbackFn)`
+
+Triggers:
+* `requestInit`: A HTTP request is about to be made
+* `requestSuccess`: The HTTP request completed with an OK status code
+* `requestError`: The HTTP request completed with an error status code

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.11.13",
+  "version": "0.13.0",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {

package/src/app.js

@@ -1,9 +1,11 @@
 'use strict';
 
 const { JSDOM, VirtualConsole } = require('jsdom');
+const EventEmitter = require('events');
 const http = require('http');
 const https = require('https');
 const axios = require('axios').default;
+const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
 
@@ -11,17 +13,33 @@ const settings = {
 	throwErrors: false,
 	logErrors: true,
 	requestTimeout: 30000,
+	limits: {
+		default: {
+			interval: 10,
+			concurrency: 10,
+		},
+	},
 };
 
 const virtualConsole = new VirtualConsole();
 const { window: globalWindow } = new JSDOM('', { virtualConsole });
 
+let globalOptions = {
+	...settings,
+};
+
+const events = new EventEmitter();
+
+function configure(newOptions) {
+	globalOptions = merge(globalOptions, newOptions);
+}
+
 function handleError(error, code) {
-	if (settings.logErrors) {
+	if (globalOptions.logErrors) {
 		console.error(`unprint encountered an error (${code}): ${error.message}`);
 	}
 
-	if (settings.throwErrors) {
+	if (globalOptions.throwErrors) {
 		throw Object.assign(error, { code });
 	}
 
@@ -31,12 +49,6 @@ function handleError(error, code) {
 virtualConsole.on('error', (message) => handleError(message, 'JSDOM'));
 virtualConsole.on('jsdomError', (message) => handleError(message, 'JSDOM'));
 
-let globalOptions = {};
-
-function configure(newOptions) {
-	globalOptions = newOptions;
-}
-
 function trim(string) {
 	if (typeof string === 'string') {
 		return string.trim().replace(/\s+/g, ' ');
@@ -773,13 +785,27 @@ function isDomObject(element) {
 
 function initQueryFns(fns, context) {
 	if (context) {
-		return Object.fromEntries(Object.entries(fns).map(([key, fn]) => [key, (...args) => fn(context, ...args)]));
+		return Object.fromEntries(Object.entries(fns).map(([key, fn]) => [key, (...args) => {
+			events.emit('query', {
+				key,
+				args,
+				origin: context.options.origin,
+			});
+
+			return fn(context, ...args);
+		}]));
 	}
 
 	// context is passed directly to query method
 	return Object.fromEntries(Object.entries(fns).map(([key, fn]) => [key, (...args) => {
 		// first argument is already an unprint context. this seems like a convoluted approach, but there is little reason not to allow it
 		if (args[0]?.isUnprint) {
+			events.emit('query', {
+				key,
+				args,
+				origin: context.options.origin,
+			});
+
 			return fn(...args);
 		}
 
@@ -787,6 +813,12 @@ function initQueryFns(fns, context) {
 		if (isDomObject(args[0])) {
 			const element = args[0];
 
+			events.emit('query', {
+				key,
+				args,
+				origin: context.options.origin,
+			});
+
 			return fn({
 				element,
 				html: element.outerHTML || element.body?.outerHTML,
@@ -860,6 +892,44 @@ function initAll(context, selector, options = {}) {
 		.map((element) => init(element, null, options));
 }
 
+const limiters = {
+	default: new Bottleneck(),
+};
+
+function getLimiterValue(prop, options, hostname) {
+	if (options[prop] !== undefined) {
+		return options[prop];
+	}
+
+	if (options.limits[hostname]?.enable !== false && options.limits[hostname]?.[prop] !== undefined) {
+		return options.limits[hostname][prop];
+	}
+
+	return options.limits.default[prop];
+}
+
+function getLimiter(url, options) {
+	const { hostname } = new URL(url);
+
+	const interval = getLimiterValue('interval', options, hostname);
+	const concurrency = getLimiterValue('concurrency', options, hostname);
+
+	if (!limiters[interval]?.[concurrency]) {
+		limiters[interval] = limiters[interval] || {};
+
+		limiters[interval][concurrency] = new Bottleneck({
+			minTime: interval,
+			maxConcurrent: concurrency,
+			timeout: options.timeout + 10000, // timeout 10 seconds after axious should
+		});
+	}
+
+	return {
+		limiter: limiters[interval][concurrency],
+		interval: concurrency,
+	};
+}
+
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
 		timeout: 1000,
@@ -867,7 +937,19 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		url,
 	}, globalOptions, customOptions]);
 
-	const res = await axios({
+	const { limiter, interval, concurrency } = getLimiter(url, options);
+
+	const feedbackBase = {
+		url,
+		method,
+		interval,
+		concurrency,
+		options,
+	};
+
+	events.emit('requestInit', feedbackBase);
+
+	const res = await limiter.schedule(async () => axios({
 		url,
 		method,
 		data: body,
@@ -877,11 +959,17 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		signal: options.abortSignal,
 		httpAgent: options.httpAgent || new http.Agent({ ...options.agent }),
 		httpsAgent: options.httpsAgent || new https.Agent({ ...options.agent }),
-	});
+	}));
 
 	if (!(res.status >= 200 && res.status < 300)) {
 		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${res.data}`), 'HTTP_NOT_OK');
 
+		events.emit('requestError', {
+			...feedbackBase,
+			status: res.status,
+			statusText: res.statusText,
+		});
+
 		return {
 			ok: false,
 			status: res.status,
@@ -899,6 +987,12 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		res,
 	};
 
+	events.emit('requestSuccess', {
+		...feedbackBase,
+		status: res.status,
+		statusText: res.statusText,
+	});
+
 	if (res.headers['content-type'].includes('application/json') && typeof res.data === 'object') {
 		return {
 			...base,
@@ -933,8 +1027,19 @@ async function post(url, body, options) {
 	return request(url, body, options, 'POST');
 }
 
+function on(trigger, fn) {
+	events.on(trigger, fn);
+}
+
+function off(trigger, fn) {
+	events.off(trigger, fn);
+}
+
 module.exports = {
 	configure,
+	on,
+	off,
+	events,
 	get,
 	post,
 	request,

package/tests/init.js

@@ -10,7 +10,20 @@ const data = require('./data.json');
 const port = process.env.PORT || 3101;
 
 async function initTest() {
-	unprint.options({ headers: { 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' } });
+	unprint.options({
+		headers: { 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' },
+		limits: {
+			default: {
+				concurrency: 1,
+				interval: 100,
+			},
+		},
+	});
+
+	unprint.on('requestInit', (initData) => console.log('init', initData));
+	unprint.on('requestError', (errorData) => console.error('error', errorData));
+	unprint.on('requestSuccess', (successData) => console.log('success', successData));
+	unprint.on('query', (queryData) => console.log('query', queryData));
 
 	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
 	// const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);