unprint 0.18.31 → 0.18.33

package/README.md

@@ -234,6 +234,7 @@ Options
 * `browserUserAgent`: The default user agent header for browser-like requests (`get` interface `fetch` and `browserRequest`)
 * `apiUserAgent`: The default user agent header for raw requests (`get` interface `request`)
 * `useBrowser`: Forward the call to `unprint.browser()` (see below), only for GET-requests
+* `fullCookies`: Return cookie metadata in `res.cookies`, instead of just values
 
 Use Playwright with Chromium (experimental)
 * `unprint.browser(url, [options])`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.18.31",
+  "version": "0.18.33",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},
@@ -30,6 +30,7 @@
     "moment-timezone": "^0.5.34",
     "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
+    "set-cookie-parser": "^3.0.1",
     "srcset": "^4.0.0",
     "tunnel": "^0.0.6",
     "undici": "^7.18.2"

package/src/app.js

@@ -6,6 +6,7 @@ const EventEmitter = require('events');
 const undici = require('undici');
 const qs = require('node:querystring');
 const cookie = require('cookie');
+const { parseSetCookie } = require('set-cookie-parser');
 const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
@@ -1122,15 +1123,21 @@ function curateHeaders(headers, options) {
 	return headers;
 }
 
-function curateCookies(headers) {
-	if (headers) {
-		const setCookie = typeof headers.get === 'function'
-			? headers.get('set-cookie')
-			: headers['set-cookie'];
+function curateCookies(res, options) {
+	if (res) {
+		const setCookie = typeof res.headers.get === 'function'
+			? res.headers.get('set-cookie')
+			: res.headers['set-cookie'];
 
 		if (setCookie) {
 			try {
-				return cookie.parseCookie(setCookie);
+				const cookies = parseSetCookie(res, { map: true, decode: false });
+
+				if (options.fullCookies) {
+					return cookies;
+				}
+
+				return Object.fromEntries(Object.entries(cookies).map(([_key, value]) => [value.name, value.value]));
 			} catch (_error) {
 				// invalid cookie
 			}
@@ -1148,7 +1155,7 @@ function curateResponse(res, data, options, { url, control, customOptions }) {
 		status: res.statusCode || res.status,
 		statusText: res.statusText,
 		headers: res.headers,
-		cookies: curateCookies(res.headers),
+		cookies: curateCookies(res, customOptions),
 		response: res,
 		res,
 		control,
@@ -1401,15 +1408,15 @@ async function browserRequest(url, customOptions = {}) {
 
 			await closeBrowser(client, options);
 
-			return {
+			return curateResponse({
 				ok: false,
 				status,
 				statusText,
 				headers,
-				cookies: curateCookies(headers),
+				cookies: curateCookies(res, customOptions),
 				response: res,
 				res,
-			};
+			}, data, options, { url, customOptions });
 		}
 
 		events.emit('requestSuccess', feedbackBase);
@@ -1437,7 +1444,7 @@ async function browserRequest(url, customOptions = {}) {
 					status,
 					statusText,
 					headers,
-					cookies: curateCookies(headers),
+					cookies: curateCookies(res, customOptions),
 					response: res,
 					res,
 				};
@@ -1565,6 +1572,10 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 		return request(newUrl, body, options, method, redirects + 1);
 	}
 
+	const data = options.interface === 'fetch'
+		? await res.text()
+		: await res.body.text();
+
 	if (!(status >= 200 && status < 300)) {
 		handleError(new Error(`HTTP response from ${url} not OK (${status} ${res.statusText})`), 'HTTP_NOT_OK');
 
@@ -1574,21 +1585,17 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 			statusText: res.statusText,
 		});
 
-		return {
+		return curateResponse({
 			ok: false,
 			status,
 			statusText: res.statusText,
 			headers: res.headers,
-			cookies: curateCookies(res.headers),
+			cookies: curateCookies(res, customOptions),
 			response: res,
 			res,
-		};
+		}, data, options, { url, customOptions });
 	}
 
-	const data = options.interface === 'fetch'
-		? await res.text()
-		: await res.body.text();
-
 	events.emit('requestSuccess', {
 		...feedbackBase,
 		status,

package/tests/init.js

@@ -51,10 +51,13 @@ async function initTest() {
 		useProxy: false,
 	});
 
+	const setCookiesRes = await unprint.get(`http://127.0.0.1:${port}/cookies`);
+
 	console.log('JSON RES', jsonRes);
 	console.log('ERROR RES', errorRes);
 	console.log('COOKIES RES', cookiesRes);
 	console.log('PROXY RES', proxyRes.data);
+	console.log('SET COOKIES RES', setCookiesRes.cookies);
 
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));
@@ -123,6 +126,13 @@ async function initServer() {
 		res.send(data);
 	});
 
+	app.get('/cookies', (req, res) => {
+		res.cookie('hello', 'world', { httpOnly: true });
+		res.cookie('foo', 'bar');
+
+		res.send(data);
+	});
+
 	app.get('/error/:code', (req, res) => {
 		res.status(Number(req.params.code)).send();
 	});