unprint 0.19.0 → 0.19.2

package/README.md

@@ -223,9 +223,11 @@ Extracts the CSS `url()` background from a style attribute. Alias for `query.sty
 ### HTTP request
 * `unprint.get(url, [options])`
 * `unprint.post(url, body, [options])`
-* `unprint.request(url, body, [options], [method])`
+* `unprint.request(url, [options])`
 
 Options
+* `method`: `get` (default)`, `post`, `browser` (same as `useBrowser`)
+* `body`: POST body, ignored for `get` and `browser` requests
 * `select`: Pre-query and initialize a specific element on the page.
 * `selectAll`: Pre-query and initialize multiple specific element on the page.
 * `interface`: Use undici `fetch` (browser-like, default) or `request` (raw)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.19.0",
+  "version": "0.19.2",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "repository": {

package/src/app.js

@@ -22,7 +22,7 @@ const settings = {
 	requestTimeout: 30000,
 	userAgent: 'unprint',
 	remote: {
-		enabled: false,
+		enable: false,
 		address: 'http://127.0.0.1:3333/browser',
 		methods: ['browser'],
 	},
@@ -46,9 +46,10 @@ let globalOptions = {
 };
 
 const events = new EventEmitter();
+const arrayMerge = (_destinationArray, sourceArray) => sourceArray;
 
 function configure(newOptions) {
-	globalOptions = merge(globalOptions, newOptions);
+	globalOptions = merge(globalOptions, newOptions, { arrayMerge });
 }
 
 function handleError(error, code) {
@@ -1338,7 +1339,7 @@ function getAgent(options, url) {
 	});
 }
 
-async function remoteRequest(url, method, options) {
+async function remoteRequest(url, method, options, feedbackBase) {
 	const control = typeof options.control === 'function' ? options.control.toString() : null;
 
 	const res = await undici.fetch(`${options.remote.address}/request`, {
@@ -1368,6 +1369,12 @@ async function remoteRequest(url, method, options) {
 	const body = await res.text();
 	const data = JSON.parse(body);
 
+	events.emit('requestSuccess', {
+		...feedbackBase,
+		status: data.status,
+		statusText: data.statusText,
+	});
+
 	return curateResponse({
 		status: data.status,
 		statusText: data.statusText,
@@ -1380,7 +1387,7 @@ async function remoteRequest(url, method, options) {
 }
 
 function useRemoteRequest(options, method) {
-	if (options.remote.enabled) {
+	if (options.remote.enable) {
 		if (options.useRemote) {
 			return true;
 		}
@@ -1402,12 +1409,9 @@ async function browserRequest(url, customOptions = {}) {
 		url,
 	}, globalOptions, customOptions]);
 
-	if (useRemoteRequest(options, 'browser')) {
-		return remoteRequest(url, 'browser', options);
-	}
-
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 	const agent = getAgent(options, url);
+	const isRemote = useRemoteRequest(options, 'browser');
 
 	const feedbackBase = {
 		url,
@@ -1416,11 +1420,16 @@ async function browserRequest(url, customOptions = {}) {
 		concurrency,
 		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: true,
+		isRemote,
 		options,
 	};
 
 	events.emit('requestInit', feedbackBase);
 
+	if (isRemote) {
+		return remoteRequest(url, 'browser', options, feedbackBase);
+	}
+
 	return limiter.schedule(async () => {
 		const client = await getBrowserInstance(options.client, options, agent instanceof undici.ProxyAgent);
 
@@ -1576,8 +1585,10 @@ function curateRequestBody(body, options) {
 	return { body };
 }
 
-async function request(url, body, customOptions = {}, method = 'GET', redirects = 0) {
+async function request(url, customOptions = {}, redirects = 0) {
 	const options = merge.all([{
+		method: 'get',
+		body: null,
 		interface: 'fetch', // fetch or request
 		timeout: 10000,
 		extract: true,
@@ -1586,17 +1597,17 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 		url,
 	}, globalOptions, customOptions]);
 
-	if (options.useBrowser) {
-		return browserRequest(url, options);
-	}
+	const method = options.method.toUpperCase(); // uppercase required by undici
+	const body = options.body;
 
-	if (useRemoteRequest(options, method)) {
-		return remoteRequest(url, method, options);
+	if (options.useBrowser || options.method === 'browser') {
+		return browserRequest(url, options);
 	}
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 
 	const agent = getAgent(options, url);
+	const isRemote = useRemoteRequest(options, method);
 
 	const feedbackBase = {
 		url,
@@ -1605,11 +1616,16 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 		concurrency,
 		isProxied: agent instanceof undici.ProxyAgent,
 		isBrowser: false,
+		isRemote,
 		options,
 	};
 
 	events.emit('requestInit', feedbackBase);
 
+	if (isRemote) {
+		return remoteRequest(url, method, options, feedbackBase);
+	}
+
 	const curatedBody = curateRequestBody(body, options);
 	const curatedCookie = getCookie(options);
 
@@ -1681,11 +1697,18 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 }
 
 async function get(url, options) {
-	return request(url, null, options, 'GET');
+	return request(url, {
+		...options,
+		method: 'GET',
+	});
 }
 
 async function post(url, body, options) {
-	return request(url, body, options, 'POST');
+	return request(url, {
+		...options,
+		method: 'POST',
+		body,
+	});
 }
 
 function on(trigger, fn) {

package/src/server.js

@@ -42,7 +42,7 @@ function curateOptions(options) {
 	return {
 		...options,
 		remote: {
-			enabled: false,
+			enable: false,
 		},
 		useRemote: false,
 		control: options.control
@@ -60,9 +60,11 @@ async function handleRequest(req, res, unprint, method) {
 
 	const options = curateOptions(req.body.options);
 
-	const unprintRes = req.body.method === 'post'
-		? await unprint.post(req.body.url, req.body.data, options)
-		: await unprint[(method || req.body.method || 'get').toLowerCase()](req.body.url, options);
+	const unprintRes = await unprint.request(req.body.url, {
+		...options,
+		method: req.body.method,
+		body: req.body.data,
+	});
 
 	res.send({
 		ok: unprintRes.ok,
@@ -124,7 +126,12 @@ async function initServer(address, unprint) {
 		});
 	});
 
-	app.listen(port, host, () => {
+	app.listen(port, host, (error) => {
+		if (error) {
+			logger.error(`Failed to start server: ${error.message}`);
+			return;
+		}
+
 		logger.info(`Started unprint server on ${host}:${port}`);
 	});
 }

package/tests/init.js

@@ -34,6 +34,7 @@ async function initTest() {
 
 	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body', interface: 'request' });
 
+	/*
 	const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
 	const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
 	const cookiesRes = await unprint.get(`http://127.0.0.1:${port}/json`, {
@@ -58,6 +59,7 @@ async function initTest() {
 	console.log('COOKIES RES', cookiesRes);
 	console.log('PROXY RES', proxyRes.data);
 	console.log('SET COOKIES RES', setCookiesRes.cookies);
+	*/
 
 	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
 	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));
@@ -137,7 +139,12 @@ async function initServer() {
 		res.status(Number(req.params.code)).send();
 	});
 
-	const server = app.listen(port, async () => {
+	const server = app.listen(port, async (error) => {
+		if (error) {
+			console.error(error);
+			return;
+		}
+
 		const { address } = server.address();
 
 		console.log(`Test server listening on ${address}:${port}`);

package/tests/remote.js

@@ -4,15 +4,18 @@ const unprint = require('../src/app');
 
 unprint.configure({
 	remote: {
-		enabled: true,
+		enable: true,
 		address: 'http://127.0.0.1:3333',
 		key: 'foobar',
+		methods: [],
 	},
 });
 
 async function init() {
+	unprint.on('requestInit', (event) => console.log('INIT', event));
+
 	const res = await unprint.browser('https://www.google.com', {
-		useRemote: true,
+		useRemote: false,
 		async control(page) {
 			const form = await page.locator('form');