unprint 0.18.1 → 0.18.3

package/README.md

@@ -211,9 +211,13 @@ Extracts the CSS `url()` background from a style attribute. Alias for `query.sty
 Options
 * `select`: Pre-query and initialize a specific element on the page.
 * `selectAll`: Pre-query and initialize multiple specific element on the page.
+* `interface`: Use undici `fetch` (browser-like, default) or `request` (raw)
+* `userAgent`: The default user agent header
+* `browserUserAgent`: The default user agent header for browser-like requests (`get` interface `fetch` and `browserRequest`)
+* `apiUserAgent`: The default user agent header for raw requests (`get` interface `request`)
 
 Use Playwright with Chromium (experimental)
-* `unprint.browserRequest(url, [options])`
+* `unprint.browserRequest(url, [options])` or `unprint.browser(url, [options])`
 * `unprint.closeAllBrowsers()`: Close reused browser instances.
 
 Additional options

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.18.1",
+  "version": "0.18.3",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -16,6 +16,7 @@ const settings = {
 	throwErrors: false,
 	logErrors: true,
 	requestTimeout: 30000,
+	userAgent: 'unprint',
 	limits: {
 		default: {
 			interval: 10,
@@ -1056,9 +1057,11 @@ function getCookie(options) {
 	return headerCookieData;
 }
 
-function filterHeaders(headers, options) {
+function curateHeaders(headers, options) {
 	if (headers && options.defaultHeaders !== false) {
-		return Object.fromEntries(Object.entries(headers).filter(([_key, value]) => value !== null));
+		return Object.fromEntries(Object.entries(headers)
+			.map(([key, value]) => [key.toLowerCase(), value])
+			.filter(([_key, value]) => value !== null));
 	}
 
 	return headers;
@@ -1246,8 +1249,9 @@ async function browserRequest(url, customOptions = {}) {
 			const headers = route.request().headers();
 
 			route.continue({
-				headers: filterHeaders({
+				headers: curateHeaders({
 					...headers,
+					'user-agent': options.browserUserAgent || options.userAgent,
 					...options.headers,
 					cookie: getCookie(options),
 				}, options),
@@ -1377,6 +1381,7 @@ function curateRequestBody(body) {
 
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
+		interface: 'fetch', // fetch or request
 		timeout: 10000,
 		extract: true,
 		url,
@@ -1401,13 +1406,14 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 	const curatedBody = curateRequestBody(body);
 	const curatedCookie = getCookie(options);
 
-	const headers = filterHeaders({
+	const headers = curateHeaders({
 		...curatedBody.headers,
+		'user-agent': (options.interface === 'fetch' ? options.browserUserAgent : options.apiUserAgent) || options.userAgent,
 		...options.headers,
 		cookie: curatedCookie,
 	}, options);
 
-	const res = await limiter.schedule(async () => undici.fetch(url, {
+	const res = await limiter.schedule(async () => undici[options.interface](url, {
 		dispatcher: agent,
 		method,
 		body: curatedBody.body,
@@ -1419,20 +1425,24 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		async text() { return error.cause?.cause?.message || 'Request aborted'; },
 	}));
 
-	if (!(res.status >= 200 && res.status < 300)) {
-		const data = await res.text();
+	const data = options.interface === 'fetch'
+		? await res.text()
+		: await res.body.text();
 
-		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
+	const status = res.statusCode || res.status;
+
+	if (!(status >= 200 && status < 300)) {
+		handleError(new Error(`HTTP response from ${url} not OK (${status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
 
 		events.emit('requestError', {
 			...feedbackBase,
-			status: res.status,
+			status,
 			statusText: res.statusText,
 		});
 
 		return {
 			ok: false,
-			status: res.status,
+			status,
 			statusText: res.statusText,
 			headers: res.headers,
 			response: res,
@@ -1442,12 +1452,10 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 
 	events.emit('requestSuccess', {
 		...feedbackBase,
-		status: res.status,
+		status,
 		statusText: res.statusText,
 	});
 
-	const data = await res.text();
-
 	return curateResponse(res, data, options, { url, customOptions });
 }
 

package/tests/init.js

@@ -11,7 +11,9 @@ const port = process.env.PORT || 3101;
 
 async function initTest() {
 	unprint.options({
-		headers: { 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' },
+		userAgent: 'unprint',
+		apiUserAgent: 'unprint',
+		browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
 		limits: {
 			default: {
 				concurrency: 1,
@@ -30,7 +32,7 @@ async function initTest() {
 	unprint.on('requestSuccess', (successData) => console.log('success', successData));
 	// unprint.on('query', (queryData) => console.log('query', queryData));
 
-	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
+	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body', interface: 'request' });
 
 	const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
 	const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
@@ -44,9 +46,15 @@ async function initTest() {
 		},
 	});
 
+	const proxyRes = await unprint.get('https://api.ipify.org?format=json', {
+		interface: 'request',
+		useProxy: true,
+	});
+
 	console.log('JSON RES', jsonRes);
 	console.log('ERROR RES', errorRes);
 	console.log('COOKIES RES', cookiesRes);
+	console.log('PROXY RES', proxyRes.data);
 
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));