unprint 0.17.7 → 0.17.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.17.7",
+  "version": "0.17.9",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {},

package/src/app.js

@@ -191,12 +191,18 @@ function queryContents(context, selector, customOptions) {
 	const options = {
 		...context.options,
 		trim: true,
+		filter: true,
 		...customOptions,
 	};
 
 	const targets = queryElements(context, selector, options);
+	const extractedContents = targets.map((target) => extractContent(target, options));
+
+	if (options.filter) {
+		return extractedContents.filter(Boolean);
+	}
 
-	return targets.map((target) => extractContent(target, options)).filter(Boolean);
+	return extractedContents;
 }
 
 function queryAttribute(context, selector, attribute, customOptions) {
@@ -277,15 +283,24 @@ function queryNumber(context, selector, customOptions) {
 }
 
 function queryNumbers(context, selector, customOptions) {
+	const options = {
+		filter: true,
+		...customOptions,
+	};
+
 	const numberStrings = queryContents(context, selector, customOptions);
 
 	if (!numberStrings) {
 		return null;
 	}
 
-	return numberStrings
-		.map((numberString) => extractNumber(numberString, customOptions))
-		.filter(Boolean);
+	const extractedNumbers = numberStrings.map((numberString) => extractNumber(numberString, customOptions));
+
+	if (options.filter) {
+		return extractedNumbers.filter(Boolean);
+	}
+
+	return extractedNumbers;
 }
 
 function queryHtml(context, selector, customOptions) {
@@ -1149,6 +1164,14 @@ function curateResponse(res, options, { url, control, customOptions }) {
 	};
 }
 
+async function closeBrowser(client, options) {
+	if (options.client === null // this browser is single-use
+		|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
+		// this browser won't be reused
+		await client.browser.close();
+	}
+}
+
 async function browserRequest(url, customOptions = {}) {
 	const options = merge.all([{
 		timeout: 1000,
@@ -1182,7 +1205,15 @@ async function browserRequest(url, customOptions = {}) {
 
 		const res = await page.goto(url, {
 			...options.page,
-		});
+		}).catch((error) => error);
+
+		if (res instanceof Error) {
+			return {
+				ok: false,
+				status: null,
+				statusText: res.name,
+			};
+		}
 
 		const status = res.status();
 		const statusText = res.statusText();
@@ -1199,6 +1230,8 @@ async function browserRequest(url, customOptions = {}) {
 
 			client.active -= 1;
 
+			await closeBrowser(client, options);
+
 			return {
 				ok: false,
 				status,
@@ -1221,6 +1254,8 @@ async function browserRequest(url, customOptions = {}) {
 			} catch (error) {
 				client.active -= 1;
 
+				await closeBrowser(client, options);
+
 				return {
 					ok: false,
 					controlError: error.message,
@@ -1247,11 +1282,7 @@ async function browserRequest(url, customOptions = {}) {
 
 		client.active -= 1;
 
-		if (options.client === null // this browser is single-use
-			|| (client.retired && client.active === 0)) { // this browser is retired to minimize garbage build-up
-			// this browser won't be reused
-			await client.browser.close();
-		}
+		await closeBrowser(client, options);
 
 		return curateResponse({
 			data,

package/tests/browser.js

@@ -16,6 +16,8 @@ unprint.options({ // or unprint.options();
 
 async function initTest() {
 	// concurrency
+	/*
+	console.log('TEST CONCURRENCY');
 	await Promise.all(Array.from({ length: 20 }).map(async () => {
 		// await unprint.browser(`https://tools-httpstatus.pickup-services.com/${Math.random() < 0.2 ? '404' : '200'}?sleep=${Math.round(Math.random() * 500)}`, {
 		await unprint.browser(`https://tools-httpstatus.pickup-services.com/200?sleep=${Math.round(Math.random() * 5000)}`, {
@@ -26,11 +28,24 @@ async function initTest() {
 			},
 		});
 	}));
+	*/
 
 	// console.log('Requests done, waiting...');
 
 	// 	await new Promise((resolve) => { setTimeout(() => resolve(), 60 * 60 * 1000); });
+	// timeout
+	console.log('TEST TIMEOUT');
+	await unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=30000', {
+		// client: null,
+		browser: {
+			headless: true,
+		},
+		page: {
+			timeout: 5000,
+		},
+	});
 
+	/*
 	await Promise.all([
 		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=5000', {
 			browser: {
@@ -56,6 +71,7 @@ async function initTest() {
 		}),
 	]);
 
+	console.log('TEST SCRAPE');
 	const res = await unprint.browser('https://www.scrapingcourse.com/', {
 		browser: {
 			headless: false,
@@ -69,6 +85,8 @@ async function initTest() {
 
 	console.log('CARD TITLES', cards);
 	console.log('CONTROL OUT', res.control);
+	*/
+	console.log('CLOSING ALL BROWSERS');
 
 	await unprint.closeAllBrowsers();
 }