unprint 0.18.31 → 0.18.32

package/README.md

@@ -234,6 +234,7 @@ Options
 * `browserUserAgent`: The default user agent header for browser-like requests (`get` interface `fetch` and `browserRequest`)
 * `apiUserAgent`: The default user agent header for raw requests (`get` interface `request`)
 * `useBrowser`: Forward the call to `unprint.browser()` (see below), only for GET-requests
+* `fullCookies`: Return cookie metadata in `res.cookies`, instead of just values
 
 Use Playwright with Chromium (experimental)
 * `unprint.browser(url, [options])`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.18.31",
+  "version": "0.18.32",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},
@@ -30,6 +30,7 @@
     "moment-timezone": "^0.5.34",
     "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
+    "set-cookie-parser": "^3.0.1",
     "srcset": "^4.0.0",
     "tunnel": "^0.0.6",
     "undici": "^7.18.2"

package/src/app.js

@@ -6,6 +6,7 @@ const EventEmitter = require('events');
 const undici = require('undici');
 const qs = require('node:querystring');
 const cookie = require('cookie');
+const { parseSetCookie } = require('set-cookie-parser');
 const Bottleneck = require('bottleneck');
 const moment = require('moment-timezone');
 const merge = require('deepmerge');
@@ -1122,15 +1123,21 @@ function curateHeaders(headers, options) {
 	return headers;
 }
 
-function curateCookies(headers) {
-	if (headers) {
-		const setCookie = typeof headers.get === 'function'
-			? headers.get('set-cookie')
-			: headers['set-cookie'];
+function curateCookies(res, options) {
+	if (res) {
+		const setCookie = typeof res.headers.get === 'function'
+			? res.headers.get('set-cookie')
+			: res.headers['set-cookie'];
 
 		if (setCookie) {
 			try {
-				return cookie.parseCookie(setCookie);
+				const cookies = parseSetCookie(res, { map: true, decode: false });
+
+				if (options.fullCookies) {
+					return cookies;
+				}
+
+				return Object.fromEntries(Object.entries(cookies).map(([_key, value]) => [value.name, value.value]));
 			} catch (_error) {
 				// invalid cookie
 			}
@@ -1148,7 +1155,7 @@ function curateResponse(res, data, options, { url, control, customOptions }) {
 		status: res.statusCode || res.status,
 		statusText: res.statusText,
 		headers: res.headers,
-		cookies: curateCookies(res.headers),
+		cookies: curateCookies(res, customOptions),
 		response: res,
 		res,
 		control,
@@ -1406,7 +1413,7 @@ async function browserRequest(url, customOptions = {}) {
 				status,
 				statusText,
 				headers,
-				cookies: curateCookies(headers),
+				cookies: curateCookies(res, customOptions),
 				response: res,
 				res,
 			};
@@ -1437,7 +1444,7 @@ async function browserRequest(url, customOptions = {}) {
 					status,
 					statusText,
 					headers,
-					cookies: curateCookies(headers),
+					cookies: curateCookies(res, customOptions),
 					response: res,
 					res,
 				};
@@ -1579,7 +1586,7 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 			status,
 			statusText: res.statusText,
 			headers: res.headers,
-			cookies: curateCookies(res.headers),
+			cookies: curateCookies(res, customOptions),
 			response: res,
 			res,
 		};

package/tests/init.js

@@ -51,10 +51,13 @@ async function initTest() {
 		useProxy: false,
 	});
 
+	const setCookiesRes = await unprint.get(`http://127.0.0.1:${port}/cookies`);
+
 	console.log('JSON RES', jsonRes);
 	console.log('ERROR RES', errorRes);
 	console.log('COOKIES RES', cookiesRes);
 	console.log('PROXY RES', proxyRes.data);
+	console.log('SET COOKIES RES', setCookiesRes.cookies);
 
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));
@@ -123,6 +126,13 @@ async function initServer() {
 		res.send(data);
 	});
 
+	app.get('/cookies', (req, res) => {
+		res.cookie('hello', 'world', { httpOnly: true });
+		res.cookie('foo', 'bar');
+
+		res.send(data);
+	});
+
 	app.get('/error/:code', (req, res) => {
 		res.status(Number(req.params.code)).send();
 	});