unprint 0.16.2 → 0.16.4-beta

package/README.md

@@ -59,6 +59,9 @@ The selector can be a CSS selector, an XPath selector starting with `/` or `(`,
 #### Querying multiple elements
 Most methods can be used in plural, returning an array of results, i.e. `query.elements()`, `query.dates()`.
 
+Options
+* `filterDuplicates`: When an array of selectors results in the same element being selected multiple times, ensure each element is only returned once, default `true`.
+
 #### Query an element
 * `query.element([selector], [options])`
 
@@ -199,11 +202,19 @@ Extracts the CSS `url()` background from a style attribute. Alias for `query.sty
 ### HTTP request
 * `unprint.get(url, [options])`
 * `unprint.post(url, body, [options])`
+* `unprint.request(url, body, [options], [method])`
 
 Options
 * `select`: Pre-query and initialize a specific element on the page
 * `selectAll`: Pre-query and initialize multiple specific element on the page
 
+Use Playwright with Chromium (experimental)
+* `unprint.browserRequest(url, [options])`
+* `unprint.closeAllBrowsers()`
+
+Additional options
+* `browser`: Options object passed to Playwright
+
 Returns
 ```javascript
 {

package/package.json

@@ -1,11 +1,9 @@
 {
   "name": "unprint",
-  "version": "0.16.2",
+  "version": "0.16.4-beta",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
+  "scripts": {},
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ThePendulum/unprint.git"
@@ -30,10 +28,13 @@
     "eslint-config-airbnb-base": "^15.0.0",
     "jsdom": "^17.0.0",
     "moment-timezone": "^0.5.34",
+    "patchright": "^1.56.1",
     "srcset": "^4.0.0",
     "tunnel": "^0.0.6"
   },
   "devDependencies": {
+    "@playwright/test": "^1.56.1",
+    "@types/node": "^24.10.0",
     "express": "^4.18.1"
   }
 }

package/playwright.config.js

@@ -0,0 +1,81 @@
+// @ts-check
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * @see https://playwright.dev/docs/test-configuration
+ */
+export default defineConfig({
+  testDir: './e2e',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://localhost:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});
+

package/src/app.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { JSDOM, VirtualConsole } = require('jsdom');
+const { chromium } = require('patchright');
 const EventEmitter = require('events');
 const http = require('http');
 const https = require('https');
@@ -102,14 +103,20 @@ function queryElement(context, selectors, _customOptions) {
 	return target || null;
 }
 
-function queryElements(context, selectors, _customOptions) {
+function queryElements(context, selectors, customOptions = {}) {
 	if (!selectors) {
 		return context.element;
 	}
 
-	const targets = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, false), null);
+	const options = customOptions;
+	const targets = [].concat(selectors).reduce((acc, selector) => acc.concat(getElements(context, selector, false)), []).filter(Boolean);
 
-	return targets || [];
+	if (options.filterDuplicates === false) {
+		return targets || [];
+	}
+
+	// findIndex always finds first index, if current index is not the first index, it's a dupe
+	return targets.filter((target, index, array) => index === array.findIndex((dupe) => target === dupe));
 }
 
 function queryExistence(context, selector, customOptions) {
@@ -1035,7 +1042,143 @@ function setProxy(instance, options, url) {
 
 	return false;
 }
+
+const clients = new Map();
+
 /* eslint-enable no-param-reassign */
+async function getBrowserInstance(scope) {
+	if (clients.has(scope)) {
+		return clients.get(scope);
+	}
+
+	const browser = await chromium.launch({
+		headless: false,
+	});
+
+	const context = await browser.newContext({
+		userAgent: 'unprint',
+	});
+
+	const client = { context, browser };
+
+	clients.set(scope, client);
+
+	return client;
+}
+
+async function closeAllBrowsers() {
+	await Promise.all(Array.from(clients.values()).map(async (client) => client.browser.close()));
+}
+
+function curateResponse(res, options, { url, customOptions }) {
+	const base = {
+		ok: true,
+		status: res.status,
+		statusText: res.statusText,
+		headers: res.headers,
+		response: res,
+		res,
+	};
+
+	if (['application/json', 'application/javascript'].some((type) => res.headers['content-type']?.includes(type)) && typeof res.data === 'object') {
+		return {
+			...base,
+			data: res.data,
+		};
+	}
+
+	if (!options.extract) {
+		return base;
+	}
+
+	const contextOptions = {
+		...customOptions,
+		origin: url,
+	};
+
+	const context = options.selectAll
+		? initAll(res.data, options.selectAll, contextOptions)
+		: init(res.data, options.select, contextOptions);
+
+	return {
+		...base,
+		context,
+	};
+}
+
+async function browserRequest(url, customOptions = {}) {
+	const options = merge.all([{
+		timeout: 1000,
+		extract: true,
+		scope: 'main',
+		url,
+	}, globalOptions, customOptions]);
+
+	const { limiter, interval, concurrency } = getLimiter(url, options);
+
+	const feedbackBase = {
+		url,
+		method: 'get',
+		interval,
+		concurrency,
+		isProxied: false,
+		options,
+	};
+
+	return limiter.schedule(async () => {
+		const { context, browser } = await getBrowserInstance(options.scope);
+		const page = await context.newPage();
+
+		const res = await page.goto(url, {
+			...options.browser,
+		});
+
+		const status = res.status();
+		const statusText = res.statusText();
+		const headers = await res.allHeaders();
+
+		if (!(status >= 200 && status < 300)) {
+			handleError(new Error(`HTTP response from ${url} not OK (${status} ${statusText}): ${res.data}`), 'HTTP_NOT_OK');
+
+			events.emit('requestError', {
+				...feedbackBase,
+				status,
+				statusText,
+			});
+
+			return {
+				ok: false,
+				status,
+				statusText,
+				headers,
+				response: res,
+				res,
+			};
+		}
+
+		events.emit('requestSuccess', feedbackBase);
+
+		await page.waitForLoadState();
+
+		if (customOptions.control) {
+			await customOptions.control(page, { context, browser });
+		}
+
+		events.emit('controlSuccess', feedbackBase);
+
+		const data = await page.content();
+
+		await page.close();
+		// await browser.close();
+
+		return curateResponse({
+			data,
+			status,
+			statusText,
+			headers,
+		}, options, { url, customOptions });
+	});
+}
 
 async function request(url, body, customOptions = {}, method = 'GET') {
 	const options = merge.all([{
@@ -1093,45 +1236,13 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		};
 	}
 
-	const base = {
-		ok: true,
-		status: res.status,
-		statusText: res.statusText,
-		headers: res.headers,
-		response: res,
-		res,
-	};
-
 	events.emit('requestSuccess', {
 		...feedbackBase,
 		status: res.status,
 		statusText: res.statusText,
 	});
 
-	if (['application/json', 'application/javascript'].some((type) => res.headers['content-type'].includes(type)) && typeof res.data === 'object') {
-		return {
-			...base,
-			data: res.data,
-		};
-	}
-
-	if (!options.extract) {
-		return base;
-	}
-
-	const contextOptions = {
-		...customOptions,
-		origin: url,
-	};
-
-	const context = options.selectAll
-		? initAll(res.data, options.selectAll, contextOptions)
-		: init(res.data, options.select, contextOptions);
-
-	return {
-		...base,
-		context,
-	};
+	return curateResponse(res, options, { url, customOptions });
 }
 
 async function get(url, options) {
@@ -1158,6 +1269,9 @@ module.exports = {
 	get,
 	post,
 	request,
+	browserRequest,
+	browser: browserRequest,
+	closeAllBrowsers,
 	initialize: init,
 	initializeAll: initAll,
 	init,

package/tests/browser.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const unprint = require('../src/app');
+
+async function initTest() {
+	await Promise.all([
+		unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=5000', {
+			headless: false,
+			async control(_page) {
+				//
+			},
+		}),
+		new Promise((resolve) => {
+			setTimeout(() => {
+				resolve();
+			}, 1000);
+		}).then(async () => {
+			await unprint.browser('https://tools-httpstatus.pickup-services.com/200?sleep=2000', {
+				headless: false,
+				async control(_page) {
+					//
+				},
+			});
+		}),
+	]);
+
+	const res = await unprint.browser('https://www.scrapingcourse.com/', {
+	// await unprint.browser('https://www.scrapingcourse.com/', {
+		headless: false,
+		async control(_page) {
+			//
+		},
+	});
+
+	const cards = res.context.query.contents('h2');
+
+	console.log('CARD TITLES', cards);
+
+	await unprint.closeAllBrowsers();
+}
+
+initTest();

package/tests/init.js

@@ -47,6 +47,7 @@ async function initTest() {
 	console.log('number indexed', res.context.query.number('.number', { match: /(\d+)/, matchIndex: 1 }));
 	console.log('data', res.context.query.json('#json'));
 	console.log('items', res.context.query.contents('.item'));
+	console.log('items css xpath array', res.context.query.contents(['.item', '//li[contains(@class, "number")]']));
 	console.log('link', res.context.query.url('#link'));
 	console.log('links', res.context.query.urls('.link'));
 	console.log('text', res.context.query.text('.text'));