unprint 0.17.8 → 0.18.0

package/.eslintrc

@@ -2,7 +2,7 @@
     "extends": "airbnb-base",
     "parserOptions": {
         "sourceType": "script",
-		"ecmaVersion": 2020
+		"ecmaVersion": "latest"
     },
     "rules": {
         "strict": 0,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.8",
+  "version": "0.18.0",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},
@@ -20,8 +20,8 @@
   },
   "homepage": "https://github.com/ThePendulum/unprint#readme",
   "dependencies": {
-    "axios": "^0.27.2",
     "bottleneck": "^2.19.5",
+    "cookie": "^1.1.1",
     "deepmerge": "^4.2.2",
     "eslint": "^8.17.0",
     "eslint-config-airbnb": "^19.0.4",
@@ -31,7 +31,8 @@
     "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
     "srcset": "^4.0.0",
-    "tunnel": "^0.0.6"
+    "tunnel": "^0.0.6",
+    "undici": "^7.18.2"
   },
   "devDependencies": {
     "@playwright/test": "^1.56.1",

package/src/app.js

@@ -3,10 +3,9 @@
 const { JSDOM, VirtualConsole } = require('jsdom');
 const { chromium } = require('patchright');
 const EventEmitter = require('events');
-const http = require('http');
-const https = require('https');
-const tunnel = require('tunnel');
-const axios = require('axios').default;
+const undici = require('undici');
+const qs = require('node:querystring');
+const cookie = require('cookie');
 const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
@@ -1034,38 +1033,88 @@ function getLimiter(url, options) {
 	};
 }
 
-/* eslint-disable no-param-reassign */
-function setProxy(instance, options, url) {
-	const { hostname } = new URL(url);
+function getCookie(options) {
+	const headerCookieData = options.headers?.cookie || options.headers?.Cookie || null;
+	const headerCookies = headerCookieData && cookie.parseCookie(headerCookieData);
 
-	if (options.proxy
-		&& options.proxy.enable !== false
-		&& (options.proxy.use
-		|| options.proxy.hostnames?.includes(hostname))
-	) {
-		const proxyAgent = tunnel.httpsOverHttp({
-			proxy: {
-				host: options.proxy.host,
-				port: options.proxy.port,
-			},
+	if (typeof options.cookies === 'object') {
+		return cookie.stringifyCookie({
+			...headerCookies,
+			...options.cookies,
 		});
+	}
+
+	if (typeof options.cookies === 'string') {
+		const cookieData = cookie.parseCookie(options.cookies);
+
+		return cookie.stringifyCookie({
+			...headerCookies,
+			...cookieData,
+		});
+	}
+
+	return headerCookieData;
+}
 
-		if (instance) {
-			instance.defaults.httpAgent = proxyAgent;
-			instance.defaults.httpsAgent = proxyAgent;
+function curateResponse(res, data, options, { url, control, customOptions }) {
+	const base = {
+		ok: true,
+		data,
+		status: res.statusCode || res.status,
+		statusText: res.statusText,
+		headers: res.headers,
+		response: res,
+		res,
+		control,
+	};
+
+	if (['application/json', 'application/javascript'].some((type) => {
+		if (typeof res.headers.get === 'function') {
+			return res.headers.get('content-type')?.includes(type);
 		}
 
-		return true;
+		return res.headers['content-type']?.includes(type);
+	})) {
+		if (typeof data === 'object') {
+			return {
+				...base,
+				data,
+			};
+		}
+
+		try {
+			return {
+				...base,
+				data: JSON.parse(data),
+			};
+		} catch (error) {
+			return {
+				...base,
+				data,
+			};
+		}
 	}
 
-	if (instance) {
-		instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
-		instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+	if (!options.extract) {
+		return base;
 	}
 
-	return false;
+	const contextOptions = {
+		...customOptions,
+		origin: url,
+	};
+
+	const context = options.selectAll
+		? initAll(data, options.selectAll, contextOptions)
+		: init(data, options.select, contextOptions);
+
+	return {
+		...base,
+		context,
+	};
 }
 
+/* eslint-disable no-param-reassign */
 const clients = new Map();
 
 /* eslint-enable no-param-reassign */
@@ -1127,46 +1176,36 @@ async function closeAllBrowsers() {
 	await Promise.all(Array.from(clients.values()).map(async (client) => client.browser.close()));
 }
 
-function curateResponse(res, options, { url, control, customOptions }) {
-	const base = {
-		ok: true,
-		status: res.status,
-		statusText: res.statusText,
-		headers: res.headers,
-		response: res,
-		res,
-		control,
-	};
-
-	if (['application/json', 'application/javascript'].some((type) => res.headers['content-type']?.includes(type)) && typeof res.data === 'object') {
-		return {
-			...base,
-			data: res.data,
-		};
-	}
-
-	if (!options.extract) {
-		return base;
+async function closeBrowser(client, options) {
+	if (options.client === null // this browser is single-use
+		|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
+		// this browser won't be reused
+		await client.browser.close();
 	}
+}
 
-	const contextOptions = {
-		...customOptions,
-		origin: url,
-	};
+function getAgent(options, url) {
+	const { hostname } = new URL(url);
 
-	const context = options.selectAll
-		? initAll(res.data, options.selectAll, contextOptions)
-		: init(res.data, options.select, contextOptions);
+	if (options.proxy
+		&& options.proxy.enable !== false
+		&& (options.useProxy // defined locally
+		|| options.proxy.use // defined globally
+		|| options.proxy.hostnames?.includes(hostname))
+	) {
+		return new undici.ProxyAgent(`http://${options.proxy.host}:${options.proxy.port}/`, {
+			bodyTimeout: options.timeout,
+		});
+	}
 
-	return {
-		...base,
-		context,
-	};
+	return new undici.Agent({
+		bodyTimeout: options.timeout,
+	});
 }
 
 async function browserRequest(url, customOptions = {}) {
 	const options = merge.all([{
-		timeout: 1000,
+		timeout: 10000,
 		extract: true,
 		client: 'main',
 		limiter: 'browser',
@@ -1174,14 +1213,14 @@ async function browserRequest(url, customOptions = {}) {
 	}, globalOptions, customOptions]);
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
-	const useProxy = setProxy(null, options, url);
+	const agent = getAgent(options, url);
 
 	const feedbackBase = {
 		url,
 		method: 'get',
 		interval,
 		concurrency,
-		isProxied: useProxy,
+		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: true,
 		options,
 	};
@@ -1189,22 +1228,44 @@ async function browserRequest(url, customOptions = {}) {
 	events.emit('requestInit', feedbackBase);
 
 	return limiter.schedule(async () => {
-		const client = await getBrowserInstance(options.client, options, useProxy);
+		const client = await getBrowserInstance(options.client, options, agent instanceof undici.ProxyAgent);
 
 		client.active += 1;
 
 		const page = await client.context.newPage();
 
+		await page.route(url, async (route) => {
+			const headers = route.request().headers();
+
+			route.continue({
+				headers: {
+					...headers,
+					...options.headers,
+					cookie: getCookie(options),
+				},
+			});
+		});
+
 		const res = await page.goto(url, {
 			...options.page,
-		});
+		}).catch((error) => error);
+
+		if (res instanceof Error) {
+			return {
+				ok: false,
+				status: null,
+				statusText: res.name,
+			};
+		}
 
 		const status = res.status();
 		const statusText = res.statusText();
 		const headers = await res.allHeaders();
 
 		if (!(status >= 200 && status < 300)) {
-			handleError(new Error(`HTTP response from ${url} not OK (${status} ${statusText}): ${res.data}`), 'HTTP_NOT_OK');
+			const data = await page.content();
+
+			handleError(new Error(`HTTP response from ${url} not OK (${status} ${statusText}): ${data}`), 'HTTP_NOT_OK');
 
 			events.emit('requestError', {
 				...feedbackBase,
@@ -1214,6 +1275,8 @@ async function browserRequest(url, customOptions = {}) {
 
 			client.active -= 1;
 
+			await closeBrowser(client, options);
+
 			return {
 				ok: false,
 				status,
@@ -1236,6 +1299,8 @@ async function browserRequest(url, customOptions = {}) {
 			} catch (error) {
 				client.active -= 1;
 
+				await closeBrowser(client, options);
+
 				return {
 					ok: false,
 					controlError: error.message,
@@ -1262,18 +1327,13 @@ async function browserRequest(url, customOptions = {}) {
 
 		client.active -= 1;
 
-		if (options.client === null // this browser is single-use
-			|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
-			// this browser won't be reused
-			await client.browser.close();
-		}
+		await closeBrowser(client, options);
 
 		return curateResponse({
-			data,
 			status,
 			statusText,
 			headers,
-		}, options, {
+		}, data, options, {
 			url,
 			customOptions,
 			control,
@@ -1281,46 +1341,78 @@ async function browserRequest(url, customOptions = {}) {
 	});
 }
 
+function curateRequestBody(body) {
+	if (!body) {
+		return { body };
+	}
+
+	if (body instanceof undici.FormData) {
+		return {
+			body: qs.stringify(body),
+			headers: {
+				'content-type': 'application/x-www-form-urlencoded',
+			},
+		};
+	}
+
+	if (typeof body === 'object') {
+		return {
+			body: JSON.stringify(body),
+			headers: {
+				'content-type': 'application/json',
+			},
+		};
+	}
+
+	return { body };
+}
+
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
-		timeout: 1000,
+		timeout: 10000,
 		extract: true,
 		url,
 	}, globalOptions, customOptions]);
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 
-	const instance = axios.create({
-		method,
-		validateStatus: null,
-		headers: options.headers,
-		timeout: options.timeout,
-		signal: options.abortSignal,
-		// ...options,
-		// httpAgent: options.httpAgent || new http.Agent({ ...options.agent }),
-	});
-
-	const isProxied = setProxy(instance, options, url);
+	const agent = getAgent(options, url);
 
 	const feedbackBase = {
 		url,
 		method,
 		interval,
 		concurrency,
-		isProxied,
+		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: false,
 		options,
 	};
 
 	events.emit('requestInit', feedbackBase);
 
-	const res = await limiter.schedule(async () => instance.request({
-		url,
-		data: body,
+	const curatedBody = curateRequestBody(body);
+	const curatedCookie = getCookie(options);
+
+	const res = await limiter.schedule(async () => undici.fetch(url, {
+		dispatcher: agent,
+		method,
+		body: curatedBody.body,
+		headers: {
+			...curatedBody.headers,
+			...options.headers,
+			cookie: curatedCookie,
+		},
+		signal: options.abortSignal,
+	})).catch((error) => ({ // tends to happen when proxy can't reach host
+		status: 500,
+		statusText: 'Request aborted',
+		async text() { return error.cause?.cause?.message || 'Request aborted'; },
 	}));
 
 	if (!(res.status >= 200 && res.status < 300)) {
-		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${res.data}`), 'HTTP_NOT_OK');
+		const data = await res.text();
+
+		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
 
 		events.emit('requestError', {
 			...feedbackBase,
@@ -1344,7 +1436,9 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		statusText: res.statusText,
 	});
 
-	return curateResponse(res, options, { url, customOptions });
+	const data = await res.text();
+
+	return curateResponse(res, data, options, { url, customOptions });
 }
 
 async function get(url, options) {

package/tests/browser.js

@@ -16,6 +16,8 @@ unprint.options({ // or unprint.options();
 
 async function initTest() {
 	// concurrency
+	/*
+	console.log('TEST CONCURRENCY');
 	await Promise.all(Array.from({ length: 20 }).map(async () => {
 		// await unprint.browser(`https://tools-httpstatus.pickup-services.com/${Math.random() < 0.2 ? '404' : '200'}?sleep=${Math.round(Math.random() * 500)}`, {
 		await unprint.browser(`https://tools-httpstatus.pickup-services.com/200?sleep=${Math.round(Math.random() * 5000)}`, {
@@ -26,11 +28,24 @@ async function initTest() {
 			},
 		});
 	}));
+	*/
 
 	// console.log('Requests done, waiting...');
 
 	// 	await new Promise((resolve) => { setTimeout(() => resolve(), 60 * 60 * 1000); });
+	// timeout
+	console.log('TEST TIMEOUT');
+	await unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=30000', {
+		// client: null,
+		browser: {
+			headless: true,
+		},
+		page: {
+			timeout: 5000,
+		},
+	});
 
+	/*
 	await Promise.all([
 		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=5000', {
 			browser: {
@@ -56,6 +71,7 @@ async function initTest() {
 		}),
 	]);
 
+	console.log('TEST SCRAPE');
 	const res = await unprint.browser('https://www.scrapingcourse.com/', {
 		browser: {
 			headless: false,
@@ -69,6 +85,8 @@ async function initTest() {
 
 	console.log('CARD TITLES', cards);
 	console.log('CONTROL OUT', res.control);
+	*/
+	console.log('CLOSING ALL BROWSERS');
 
 	await unprint.closeAllBrowsers();
 }

package/tests/init.js

@@ -19,7 +19,7 @@ async function initTest() {
 			},
 		},
 		proxy: {
-			host: '192.168.178.25',
+			host: '192.168.1.25',
 			port: 8888,
 			hostnames: ['127.0.0.2'],
 		},
@@ -31,8 +31,21 @@ async function initTest() {
 	// unprint.on('query', (queryData) => console.log('query', queryData));
 
 	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
-	// const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
-	// const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
+
+	const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
+	const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
+	const cookiesRes = await unprint.get(`http://127.0.0.1:${port}/json`, {
+		headers: {
+			cookie: 'foo=bar',
+		},
+		cookies: {
+			hello: 'world',
+		},
+	});
+
+	console.log('JSON RES', jsonRes);
+	console.log('ERROR RES', errorRes);
+	console.log('COOKIES RES', cookiesRes);
 
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));