unprint 0.18.1 → 0.18.2

package/README.md

@@ -211,9 +211,10 @@ Extracts the CSS `url()` background from a style attribute. Alias for `query.sty
 Options
 * `select`: Pre-query and initialize a specific element on the page.
 * `selectAll`: Pre-query and initialize multiple specific element on the page.
+* `interface`: Use undici `fetch` (browser-like, default) or `request` (raw)
 
 Use Playwright with Chromium (experimental)
-* `unprint.browserRequest(url, [options])`
+* `unprint.browserRequest(url, [options])` or `unprint.browser(url, [options])`
 * `unprint.closeAllBrowsers()`: Close reused browser instances.
 
 Additional options

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.18.1",
+  "version": "0.18.2",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -1377,6 +1377,7 @@ function curateRequestBody(body) {
 
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
+		interface: 'fetch', // fetch or request
 		timeout: 10000,
 		extract: true,
 		url,
@@ -1407,7 +1408,7 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		cookie: curatedCookie,
 	}, options);
 
-	const res = await limiter.schedule(async () => undici.fetch(url, {
+	const res = await limiter.schedule(async () => undici[options.interface](url, {
 		dispatcher: agent,
 		method,
 		body: curatedBody.body,
@@ -1419,20 +1420,24 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		async text() { return error.cause?.cause?.message || 'Request aborted'; },
 	}));
 
-	if (!(res.status >= 200 && res.status < 300)) {
-		const data = await res.text();
+	const data = options.interface === 'fetch'
+		? await res.text()
+		: await res.body.text();
 
-		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
+	const status = res.statusCode || res.status;
+
+	if (!(status >= 200 && status < 300)) {
+		handleError(new Error(`HTTP response from ${url} not OK (${status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
 
 		events.emit('requestError', {
 			...feedbackBase,
-			status: res.status,
+			status,
 			statusText: res.statusText,
 		});
 
 		return {
 			ok: false,
-			status: res.status,
+			status,
 			statusText: res.statusText,
 			headers: res.headers,
 			response: res,
@@ -1442,12 +1447,10 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 
 	events.emit('requestSuccess', {
 		...feedbackBase,
-		status: res.status,
+		status,
 		statusText: res.statusText,
 	});
 
-	const data = await res.text();
-
 	return curateResponse(res, data, options, { url, customOptions });
 }
 

package/tests/init.js

@@ -30,7 +30,7 @@ async function initTest() {
 	unprint.on('requestSuccess', (successData) => console.log('success', successData));
 	// unprint.on('query', (queryData) => console.log('query', queryData));
 
-	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
+	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body', interface: 'request' });
 
 	const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
 	const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
@@ -44,10 +44,17 @@ async function initTest() {
 		},
 	});
 
+	const proxyRes = await unprint.get('https://api.ipify.org?format=json', {
+		interface: 'request',
+		useProxy: true,
+	});
+
 	console.log('JSON RES', jsonRes);
 	console.log('ERROR RES', errorRes);
 	console.log('COOKIES RES', cookiesRes);
 
+	console.log('PROXY RES', proxyRes.data);
+
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));
 	console.log('date xpath', res.context.query.date('//div[contains(text(), "Today:")]', 'MMM DD, YYYY'));