unprint 0.18.11 → 0.18.13

package/README.md

@@ -228,6 +228,7 @@ Options
 * `select`: Pre-query and initialize a specific element on the page.
 * `selectAll`: Pre-query and initialize multiple specific element on the page.
 * `interface`: Use undici `fetch` (browser-like, default) or `request` (raw)
+* `form`: Encode POST body as urlencoded form rather than JSON
 * `userAgent`: The default user agent header
 * `browserUserAgent`: The default user agent header for browser-like requests (`get` interface `fetch` and `browserRequest`)
 * `apiUserAgent`: The default user agent header for raw requests (`get` interface `request`)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.18.11",
+  "version": "0.18.13",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -1115,15 +1115,17 @@ function curateHeaders(headers, options) {
 }
 
 function curateCookies(headers) {
-	const setCookie = typeof headers.get === 'function'
-		? headers.get('set-cookie')
-		: headers['set-cookie'];
+	if (headers) {
+		const setCookie = typeof headers.get === 'function'
+			? headers.get('set-cookie')
+			: headers['set-cookie'];
 
-	if (setCookie) {
-		try {
-			return cookie.parseCookie(setCookie);
-		} catch (_error) {
-			// invalid cookie
+		if (setCookie) {
+			try {
+				return cookie.parseCookie(setCookie);
+			} catch (_error) {
+				// invalid cookie
+			}
 		}
 	}
 
@@ -1134,6 +1136,7 @@ function curateResponse(res, data, options, { url, control, customOptions }) {
 	const base = {
 		ok: true,
 		data,
+		body: data,
 		status: res.statusCode || res.status,
 		statusText: res.statusText,
 		headers: res.headers,
@@ -1433,21 +1436,21 @@ async function browserRequest(url, customOptions = {}) {
 	});
 }
 
-function curateRequestBody(body) {
+function curateRequestBody(body, options) {
 	if (!body) {
 		return { body };
 	}
 
-	if (body instanceof undici.FormData) {
-		return {
-			body: qs.stringify(body),
-			headers: {
-				'content-type': 'application/x-www-form-urlencoded',
-			},
-		};
-	}
-
 	if (typeof body === 'object') {
+		if (options.form) {
+			return {
+				body: qs.stringify(body),
+				headers: {
+					'content-type': 'application/x-www-form-urlencoded',
+				},
+			};
+		}
+
 		return {
 			body: JSON.stringify(body),
 			headers: {
@@ -1485,7 +1488,7 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 
 	events.emit('requestInit', feedbackBase);
 
-	const curatedBody = curateRequestBody(body);
+	const curatedBody = curateRequestBody(body, options);
 	const curatedCookie = getCookie(options);
 
 	const headers = curateHeaders({
@@ -1522,12 +1525,8 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 		return request(newUrl, body, options, method, redirects + 1);
 	}
 
-	const data = options.interface === 'fetch'
-		? await res.text()
-		: await res.body.text();
-
 	if (!(status >= 200 && status < 300)) {
-		handleError(new Error(`HTTP response from ${url} not OK (${status} ${res.statusText}): ${data}`), 'HTTP_NOT_OK');
+		handleError(new Error(`HTTP response from ${url} not OK (${status} ${res.statusText})`), 'HTTP_NOT_OK');
 
 		events.emit('requestError', {
 			...feedbackBase,
@@ -1546,6 +1545,10 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 		};
 	}
 
+	const data = options.interface === 'fetch'
+		? await res.text()
+		: await res.body.text();
+
 	events.emit('requestSuccess', {
 		...feedbackBase,
 		status,

package/tests/init.js

@@ -48,7 +48,7 @@ async function initTest() {
 
 	const proxyRes = await unprint.get('https://api.ipify.org?format=json', {
 		interface: 'request',
-		useProxy: true,
+		useProxy: false,
 	});
 
 	console.log('JSON RES', jsonRes);