unprint 0.17.9 → 0.18.1

package/.eslintrc

@@ -2,7 +2,7 @@
     "extends": "airbnb-base",
     "parserOptions": {
         "sourceType": "script",
-		"ecmaVersion": 2020
+		"ecmaVersion": "latest"
     },
     "rules": {
         "strict": 0,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.9",
+  "version": "0.18.1",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},
@@ -20,8 +20,8 @@
   },
   "homepage": "https://github.com/ThePendulum/unprint#readme",
   "dependencies": {
-    "axios": "^0.27.2",
     "bottleneck": "^2.19.5",
+    "cookie": "^1.1.1",
     "deepmerge": "^4.2.2",
     "eslint": "^8.17.0",
     "eslint-config-airbnb": "^19.0.4",
@@ -31,7 +31,8 @@
     "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
     "srcset": "^4.0.0",
-    "tunnel": "^0.0.6"
+    "tunnel": "^0.0.6",
+    "undici": "^7.18.2"
   },
   "devDependencies": {
     "@playwright/test": "^1.56.1",

package/src/app.js

@@ -3,10 +3,9 @@
 const { JSDOM, VirtualConsole } = require('jsdom');
 const { chromium } = require('patchright');
 const EventEmitter = require('events');
-const http = require('http');
-const https = require('https');
-const tunnel = require('tunnel');
-const axios = require('axios').default;
+const undici = require('undici');
+const qs = require('node:querystring');
+const cookie = require('cookie');
 const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
@@ -1034,38 +1033,96 @@ function getLimiter(url, options) {
 	};
 }
 
-/* eslint-disable no-param-reassign */
-function setProxy(instance, options, url) {
-	const { hostname } = new URL(url);
+function getCookie(options) {
+	const headerCookieData = options.headers?.cookie || options.headers?.Cookie || null;
+	const headerCookies = headerCookieData && cookie.parseCookie(headerCookieData);
 
-	if (options.proxy
-		&& options.proxy.enable !== false
-		&& (options.proxy.use
-		|| options.proxy.hostnames?.includes(hostname))
-	) {
-		const proxyAgent = tunnel.httpsOverHttp({
-			proxy: {
-				host: options.proxy.host,
-				port: options.proxy.port,
-			},
+	if (typeof options.cookies === 'object') {
+		return cookie.stringifyCookie({
+			...headerCookies,
+			...options.cookies,
 		});
+	}
+
+	if (typeof options.cookies === 'string') {
+		const cookieData = cookie.parseCookie(options.cookies);
+
+		return cookie.stringifyCookie({
+			...headerCookies,
+			...cookieData,
+		});
+	}
+
+	return headerCookieData;
+}
 
-		if (instance) {
-			instance.defaults.httpAgent = proxyAgent;
-			instance.defaults.httpsAgent = proxyAgent;
+function filterHeaders(headers, options) {
+	if (headers && options.defaultHeaders !== false) {
+		return Object.fromEntries(Object.entries(headers).filter(([_key, value]) => value !== null));
+	}
+
+	return headers;
+}
+
+function curateResponse(res, data, options, { url, control, customOptions }) {
+	const base = {
+		ok: true,
+		data,
+		status: res.statusCode || res.status,
+		statusText: res.statusText,
+		headers: res.headers,
+		response: res,
+		res,
+		control,
+	};
+
+	if (['application/json', 'application/javascript'].some((type) => {
+		if (typeof res.headers.get === 'function') {
+			return res.headers.get('content-type')?.includes(type);
+		}
+
+		return res.headers['content-type']?.includes(type);
+	})) {
+		if (typeof data === 'object') {
+			return {
+				...base,
+				data,
+			};
 		}
 
-		return true;
+		try {
+			return {
+				...base,
+				data: JSON.parse(data),
+			};
+		} catch (error) {
+			return {
+				...base,
+				data,
+			};
+		}
 	}
 
-	if (instance) {
-		instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
-		instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+	if (!options.extract) {
+		return base;
 	}
 
-	return false;
+	const contextOptions = {
+		...customOptions,
+		origin: url,
+	};
+
+	const context = options.selectAll
+		? initAll(data, options.selectAll, contextOptions)
+		: init(data, options.select, contextOptions);
+
+	return {
+		...base,
+		context,
+	};
 }
 
+/* eslint-disable no-param-reassign */
 const clients = new Map();
 
 /* eslint-enable no-param-reassign */
@@ -1127,43 +1184,6 @@ async function closeAllBrowsers() {
 	await Promise.all(Array.from(clients.values()).map(async (client) => client.browser.close()));
 }
 
-function curateResponse(res, options, { url, control, customOptions }) {
-	const base = {
-		ok: true,
-		status: res.status,
-		statusText: res.statusText,
-		headers: res.headers,
-		response: res,
-		res,
-		control,
-	};
-
-	if (['application/json', 'application/javascript'].some((type) => res.headers['content-type']?.includes(type)) && typeof res.data === 'object') {
-		return {
-			...base,
-			data: res.data,
-		};
-	}
-
-	if (!options.extract) {
-		return base;
-	}
-
-	const contextOptions = {
-		...customOptions,
-		origin: url,
-	};
-
-	const context = options.selectAll
-		? initAll(res.data, options.selectAll, contextOptions)
-		: init(res.data, options.select, contextOptions);
-
-	return {
-		...base,
-		context,
-	};
-}
-
 async function closeBrowser(client, options) {
 	if (options.client === null // this browser is single-use
 		|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
@@ -1172,9 +1192,28 @@ async function closeBrowser(client, options) {
 	}
 }
 
+function getAgent(options, url) {
+	const { hostname } = new URL(url);
+
+	if (options.proxy
+		&& options.proxy.enable !== false
+		&& (options.useProxy // defined locally
+		|| options.proxy.use // defined globally
+		|| options.proxy.hostnames?.includes(hostname))
+	) {
+		return new undici.ProxyAgent(`http://${options.proxy.host}:${options.proxy.port}/`, {
+			bodyTimeout: options.timeout,
+		});
+	}
+
+	return new undici.Agent({
+		bodyTimeout: options.timeout,
+	});
+}
+
 async function browserRequest(url, customOptions = {}) {
 	const options = merge.all([{
-		timeout: 1000,
+		timeout: 10000,
 		extract: true,
 		client: 'main',
 		limiter: 'browser',
@@ -1182,14 +1221,14 @@ async function browserRequest(url, customOptions = {}) {
 	}, globalOptions, customOptions]);
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
-	const useProxy = setProxy(null, options, url);
+	const agent = getAgent(options, url);
 
 	const feedbackBase = {
 		url,
 		method: 'get',
 		interval,
 		concurrency,
-		isProxied: useProxy,
+		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: true,
 		options,
 	};
@@ -1197,12 +1236,24 @@ async function browserRequest(url, customOptions = {}) {
 	events.emit('requestInit', feedbackBase);
 
 	return limiter.schedule(async () => {
-		const client = await getBrowserInstance(options.client, options, useProxy);
+		const client = await getBrowserInstance(options.client, options, agent instanceof undici.ProxyAgent);
 
 		client.active += 1;
 
 		const page = await client.context.newPage();
 
+		await page.route(url, async (route) => {
+			const headers = route.request().headers();
+
+			route.continue({
+				headers: filterHeaders({
+					...headers,
+					...options.headers,
+					cookie: getCookie(options),
+				}, options),
+			});
+		});
+
 		const res = await page.goto(url, {
 			...options.page,
 		}).catch((error) => error);
@@ -1220,7 +1271,9 @@ async function browserRequest(url, customOptions = {}) {
 		const headers = await res.allHeaders();
 
 		if (!(status >= 200 && status < 300)) {
-			handleError(new Error(`HTTP response from ${url} not OK (${status} ${statusText}): ${res.data}`), 'HTTP_NOT_OK');
+			const data = await page.content();
+
+			handleError(new Error(`HTTP response from ${url} not OK (${status} ${statusText}): ${data}`), 'HTTP_NOT_OK');
 
 			events.emit('requestError', {
 				...feedbackBase,
@@ -1285,11 +1338,10 @@ async function browserRequest(url, customOptions = {}) {
 		await closeBrowser(client, options);
 
 		return curateResponse({
-			data,
 			status,
 			statusText,
 			headers,
-		}, options, {
+		}, data, options, {
 			url,
 			customOptions,
 			control,
@@ -1297,46 +1349,80 @@ async function browserRequest(url, customOptions = {}) {
 	});
 }
 
+function curateRequestBody(body) {
+	if (!body) {
+		return { body };
+	}
+
+	if (body instanceof undici.FormData) {
+		return {
+			body: qs.stringify(body),
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+			},
+		};
+	}
+
+	if (typeof body === 'object') {
+		return {
+			body: JSON.stringify(body),
+			headers: {
+				'content-type': 'application/json',
+			},
+		};
+	}
+
+	return { body };
+}
+
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
-		timeout: 1000,
+		timeout: 10000,
 		extract: true,
 		url,
 	}, globalOptions, customOptions]);
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 
-	const instance = axios.create({
-		method,
-		validateStatus: null,
-		headers: options.headers,
-		timeout: options.timeout,
-		signal: options.abortSignal,
-		// ...options,
-		// httpAgent: options.httpAgent || new http.Agent({ ...options.agent }),
-	});
-
-	const isProxied = setProxy(instance, options, url);
+	const agent = getAgent(options, url);
 
 	const feedbackBase = {
 		url,
 		method,
 		interval,
 		concurrency,
-		isProxied,
+		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: false,
 		options,
 	};
 
 	events.emit('requestInit', feedbackBase);
 
-	const res = await limiter.schedule(async () => instance.request({
-		url,
-		data: body,
+	const curatedBody = curateRequestBody(body);
+	const curatedCookie = getCookie(options);
+
+	const headers = filterHeaders({
+		...curatedBody.headers,
+		...options.headers,
+		cookie: curatedCookie,
+	}, options);
+
+	const res = await limiter.schedule(async () => undici.fetch(url, {
+		dispatcher: agent,
+		method,
+		body: curatedBody.body,
+		headers,
+		signal: options.abortSignal,
+	})).catch((error) => ({ // tends to happen when proxy can't reach host
+		status: 500,
+		statusText: 'Request aborted',
+		async text() { return error.cause?.cause?.message || 'Request aborted'; },
 	}));
 
 	if (!(res.status >= 200 && res.status < 300)) {
-		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${res.data}`), 'HTTP_NOT_OK');
+		const data = await res.text();
+
+		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
 
 		events.emit('requestError', {
 			...feedbackBase,
@@ -1360,7 +1446,9 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		statusText: res.statusText,
 	});
 
-	return curateResponse(res, options, { url, customOptions });
+	const data = await res.text();
+
+	return curateResponse(res, data, options, { url, customOptions });
 }
 
 async function get(url, options) {

package/tests/init.js

@@ -19,7 +19,7 @@ async function initTest() {
 			},
 		},
 		proxy: {
-			host: '192.168.178.25',
+			host: '192.168.1.25',
 			port: 8888,
 			hostnames: ['127.0.0.2'],
 		},
@@ -31,8 +31,22 @@ async function initTest() {
 	// unprint.on('query', (queryData) => console.log('query', queryData));
 
 	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
-	// const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
-	// const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
+
+	const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
+	const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
+	const cookiesRes = await unprint.get(`http://127.0.0.1:${port}/json`, {
+		headers: {
+			cookie: 'foo=bar',
+			'User-Agent': null,
+		},
+		cookies: {
+			hello: 'world',
+		},
+	});
+
+	console.log('JSON RES', jsonRes);
+	console.log('ERROR RES', errorRes);
+	console.log('COOKIES RES', cookiesRes);
 
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));