unprint 0.18.0 → 0.18.2

package/README.md

@@ -211,9 +211,10 @@ Extracts the CSS `url()` background from a style attribute. Alias for `query.sty
 Options
 * `select`: Pre-query and initialize a specific element on the page.
 * `selectAll`: Pre-query and initialize multiple specific element on the page.
+* `interface`: Use undici `fetch` (browser-like, default) or `request` (raw)
 
 Use Playwright with Chromium (experimental)
-* `unprint.browserRequest(url, [options])`
+* `unprint.browserRequest(url, [options])` or `unprint.browser(url, [options])`
 * `unprint.closeAllBrowsers()`: Close reused browser instances.
 
 Additional options

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.18.0",
+  "version": "0.18.2",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -1056,6 +1056,14 @@ function getCookie(options) {
 	return headerCookieData;
 }
 
+function filterHeaders(headers, options) {
+	if (headers && options.defaultHeaders !== false) {
+		return Object.fromEntries(Object.entries(headers).filter(([_key, value]) => value !== null));
+	}
+
+	return headers;
+}
+
 function curateResponse(res, data, options, { url, control, customOptions }) {
 	const base = {
 		ok: true,
@@ -1238,11 +1246,11 @@ async function browserRequest(url, customOptions = {}) {
 			const headers = route.request().headers();
 
 			route.continue({
-				headers: {
+				headers: filterHeaders({
 					...headers,
 					...options.headers,
 					cookie: getCookie(options),
-				},
+				}, options),
 			});
 		});
 
@@ -1369,6 +1377,7 @@ function curateRequestBody(body) {
 
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
+		interface: 'fetch', // fetch or request
 		timeout: 10000,
 		extract: true,
 		url,
@@ -1393,15 +1402,17 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 	const curatedBody = curateRequestBody(body);
 	const curatedCookie = getCookie(options);
 
-	const res = await limiter.schedule(async () => undici.fetch(url, {
+	const headers = filterHeaders({
+		...curatedBody.headers,
+		...options.headers,
+		cookie: curatedCookie,
+	}, options);
+
+	const res = await limiter.schedule(async () => undici[options.interface](url, {
 		dispatcher: agent,
 		method,
 		body: curatedBody.body,
-		headers: {
-			...curatedBody.headers,
-			...options.headers,
-			cookie: curatedCookie,
-		},
+		headers,
 		signal: options.abortSignal,
 	})).catch((error) => ({ // tends to happen when proxy can't reach host
 		status: 500,
@@ -1409,20 +1420,24 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		async text() { return error.cause?.cause?.message || 'Request aborted'; },
 	}));
 
-	if (!(res.status >= 200 && res.status < 300)) {
-		const data = await res.text();
+	const data = options.interface === 'fetch'
+		? await res.text()
+		: await res.body.text();
+
+	const status = res.statusCode || res.status;
 
-		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
+	if (!(status >= 200 && status < 300)) {
+		handleError(new Error(`HTTP response from ${url} not OK (${status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
 
 		events.emit('requestError', {
 			...feedbackBase,
-			status: res.status,
+			status,
 			statusText: res.statusText,
 		});
 
 		return {
 			ok: false,
-			status: res.status,
+			status,
 			statusText: res.statusText,
 			headers: res.headers,
 			response: res,
@@ -1432,12 +1447,10 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 
 	events.emit('requestSuccess', {
 		...feedbackBase,
-		status: res.status,
+		status,
 		statusText: res.statusText,
 	});
 
-	const data = await res.text();
-
 	return curateResponse(res, data, options, { url, customOptions });
 }
 

package/tests/init.js

@@ -30,23 +30,31 @@ async function initTest() {
 	unprint.on('requestSuccess', (successData) => console.log('success', successData));
 	// unprint.on('query', (queryData) => console.log('query', queryData));
 
-	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
+	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body', interface: 'request' });
 
 	const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
 	const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
 	const cookiesRes = await unprint.get(`http://127.0.0.1:${port}/json`, {
 		headers: {
 			cookie: 'foo=bar',
+			'User-Agent': null,
 		},
 		cookies: {
 			hello: 'world',
 		},
 	});
 
+	const proxyRes = await unprint.get('https://api.ipify.org?format=json', {
+		interface: 'request',
+		useProxy: true,
+	});
+
 	console.log('JSON RES', jsonRes);
 	console.log('ERROR RES', errorRes);
 	console.log('COOKIES RES', cookiesRes);
 
+	console.log('PROXY RES', proxyRes.data);
+
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));
 	console.log('date xpath', res.context.query.date('//div[contains(text(), "Today:")]', 'MMM DD, YYYY'));