unprint 0.18.2 → 0.18.4

package/README.md

@@ -212,6 +212,9 @@ Options
 * `select`: Pre-query and initialize a specific element on the page.
 * `selectAll`: Pre-query and initialize multiple specific element on the page.
 * `interface`: Use undici `fetch` (browser-like, default) or `request` (raw)
+* `userAgent`: The default user agent header
+* `browserUserAgent`: The default user agent header for browser-like requests (`get` interface `fetch` and `browserRequest`)
+* `apiUserAgent`: The default user agent header for raw requests (`get` interface `request`)
 
 Use Playwright with Chromium (experimental)
 * `unprint.browserRequest(url, [options])` or `unprint.browser(url, [options])`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.18.2",
+  "version": "0.18.4",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -16,6 +16,7 @@ const settings = {
 	throwErrors: false,
 	logErrors: true,
 	requestTimeout: 30000,
+	userAgent: 'unprint',
 	limits: {
 		default: {
 			interval: 10,
@@ -1056,9 +1057,11 @@ function getCookie(options) {
 	return headerCookieData;
 }
 
-function filterHeaders(headers, options) {
+function curateHeaders(headers, options) {
 	if (headers && options.defaultHeaders !== false) {
-		return Object.fromEntries(Object.entries(headers).filter(([_key, value]) => value !== null));
+		return Object.fromEntries(Object.entries(headers)
+			.map(([key, value]) => [key.toLowerCase(), value])
+			.filter(([_key, value]) => value !== null));
 	}
 
 	return headers;
@@ -1151,7 +1154,7 @@ async function getBrowserInstance(scope, options, useProxy = false) {
 	});
 
 	const contextLauncher = browserLauncher.then((browser) => browser.newContext({
-		userAgent: 'unprint',
+		userAgent: options.browserUserAgent || options.userAgent,
 		...options.context,
 		...(useProxy && {
 			proxy: {
@@ -1246,8 +1249,9 @@ async function browserRequest(url, customOptions = {}) {
 			const headers = route.request().headers();
 
 			route.continue({
-				headers: filterHeaders({
+				headers: curateHeaders({
 					...headers,
+					'user-agent': options.browserUserAgent || options.userAgent,
 					...options.headers,
 					cookie: getCookie(options),
 				}, options),
@@ -1402,8 +1406,9 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 	const curatedBody = curateRequestBody(body);
 	const curatedCookie = getCookie(options);
 
-	const headers = filterHeaders({
+	const headers = curateHeaders({
 		...curatedBody.headers,
+		'user-agent': (options.interface === 'fetch' ? options.browserUserAgent : options.apiUserAgent) || options.userAgent,
 		...options.headers,
 		cookie: curatedCookie,
 	}, options);

package/tests/init.js

@@ -11,7 +11,9 @@ const port = process.env.PORT || 3101;
 
 async function initTest() {
 	unprint.options({
-		headers: { 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' },
+		userAgent: 'unprint',
+		apiUserAgent: 'unprint',
+		browserUserAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
 		limits: {
 			default: {
 				concurrency: 1,
@@ -52,7 +54,6 @@ async function initTest() {
 	console.log('JSON RES', jsonRes);
 	console.log('ERROR RES', errorRes);
 	console.log('COOKIES RES', cookiesRes);
-
 	console.log('PROXY RES', proxyRes.data);
 
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));