unprint 0.18.34 → 0.19.0

package/.eslintrc

@@ -15,6 +15,7 @@
         "template-curly-spacing": "off",
         "object-curly-newline": "off",
 		"default-param-last": "off",
-        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
+        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}],
+		"import/no-extraneous-dependencies": ["error", {"devDependencies": true}]
     }
 }

package/README.md

@@ -272,17 +272,6 @@ Returns
 }
 ```
 
-### Helpers
-* `initialize(source, [selector], [options])` (`init`): Initialize element or HTML as unprint context
-* `initializeAll(source, [selector], [options])` (`initAll`): Initialize element or HTML as multiple contexts
-* `extractDate(string, [format], [options])`: Parse date with moment and some curation
-* `extractDateAgo(string, [options])`: Extract relative date (e.g. 4 months ago)
-* `extractDuration(timestamp, [matchRegex])`: Parse duration (e.g. 04:11:05) to seconds
-* `extractTimestamp(string)`: Parse timestamp (e.g. 4H11M5S) to seconds
-* `extractNumber(string, [options])`: Parse string as number
-* `extractSourceSet(string, [options])`: Parse source set to object
-* `formatDate(date, format, inputFormat)`: Format date with moment
-
 ### Proxy
 ```javascript
 unprint.options({ // or unprint.options();
@@ -299,13 +288,61 @@ unprint.options({ // or unprint.options();
 });
 
 unprint.get({
-	proxy: {
-		use: true, // use proxy for this request
-		// all other proxy options can be supplied here
+	useProxy: true, // use proxy for this request
+});
+```
+
+### Request server
+You can run unprint as a server to execute web request remotely. This is mainly useful when you need expensive browser requests on clients with limited resources.
+If you don't need unprint-specific features, you should probably set up a dedicated proxy server instead.
+
+#### Server
+* Ensure optional dependencies are installed
+* `UNPRINT_KEY=[random] node src/app.js  --server [port|address:port]`
+
+#### Client
+```
+unprint.options({
+	remote: {
+		enabled: true,
+		address: '10.0.0.1:3333'
+		key: 'YOUR_UNPRINT_KEY',
+		methods: ['browser'], // browser, get, post
 	},
 });
 ```
 
+If the remote is enabled, you can utilize it for individual requests regardless of method configuration:
+```
+unprint.get([url], {
+	useRemote: true,
+})
+```
+
+#### API
+```
+POST /request {
+	url: 'https://example.com', // required
+	method: 'get', // default get, post, browser
+	options: {}, // request options as passed to native method
+}
+
+POST /browser // same as /request, but with browser method implied
+
+* `options.control` is passed as a function body string, i.e. `control: "return 'hello world';"`
+```
+
+### Helpers
+* `initialize(source, [selector], [options])` (`init`): Initialize element or HTML as unprint context
+* `initializeAll(source, [selector], [options])` (`initAll`): Initialize element or HTML as multiple contexts
+* `extractDate(string, [format], [options])`: Parse date with moment and some curation
+* `extractDateAgo(string, [options])`: Extract relative date (e.g. 4 months ago)
+* `extractDuration(timestamp, [matchRegex])`: Parse duration (e.g. 04:11:05) to seconds
+* `extractTimestamp(string)`: Parse timestamp (e.g. 4H11M5S) to seconds
+* `extractNumber(string, [options])`: Parse string as number
+* `extractSourceSet(string, [options])`: Parse source set to object
+* `formatDate(date, format, inputFormat)`: Format date with moment
+
 ### Feedback events
 Usage:
 * `unprint.on('trigger', callbackFn)`

package/package.json

@@ -1,9 +1,8 @@
 {
   "name": "unprint",
-  "version": "0.18.34",
+  "version": "0.19.0",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
-  "scripts": {},
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ThePendulum/unprint.git"
@@ -23,10 +22,12 @@
     "bottleneck": "^2.19.5",
     "cookie": "^1.1.1",
     "deepmerge": "^4.2.2",
+    "dotenv": "^17.3.1",
     "eslint": "^8.17.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-base": "^15.0.0",
     "jsdom": "^17.0.0",
+    "minimist": "^1.2.8",
     "moment-timezone": "^0.5.34",
     "object-hash": "^3.0.0",
     "patchright": "^1.56.1",
@@ -38,6 +39,9 @@
   "devDependencies": {
     "@playwright/test": "^1.56.1",
     "@types/node": "^24.10.0",
-    "express": "^4.18.1"
+    "express": "^5.2.1"
+  },
+  "optionalDependencies": {
+    "express": "^5.2.1"
   }
 }

package/src/app.js

@@ -12,12 +12,20 @@ const moment = require('moment-timezone');
 const merge = require('deepmerge');
 const hashObject = require('object-hash');
 const srcset = require('srcset');
+const argv = require('minimist')(process.argv.slice(2));
+
+const initServer = require('./server');
 
 const settings = {
 	throwErrors: false,
 	logErrors: true,
 	requestTimeout: 30000,
 	userAgent: 'unprint',
+	remote: {
+		enabled: false,
+		address: 'http://127.0.0.1:3333/browser',
+		methods: ['browser'],
+	},
 	limits: {
 		default: {
 			interval: 10,
@@ -1149,7 +1157,7 @@ function curateCookies(res, options) {
 
 function curateResponse(res, data, options, { url, control, customOptions }) {
 	const base = {
-		ok: true,
+		ok: res.ok ?? true,
 		data,
 		body: data,
 		status: res.statusCode || res.status,
@@ -1330,6 +1338,61 @@ function getAgent(options, url) {
 	});
 }
 
+async function remoteRequest(url, method, options) {
+	const control = typeof options.control === 'function' ? options.control.toString() : null;
+
+	const res = await undici.fetch(`${options.remote.address}/request`, {
+		method: 'post',
+		body: JSON.stringify({
+			url,
+			method,
+			options: {
+				...options,
+				control: control && control.slice(control.indexOf('{') + 1, control.lastIndexOf('}')),
+			},
+		}),
+		headers: {
+			'content-type': 'application/json',
+			'unprint-key': options.remote.key,
+		},
+	});
+
+	if (res.status !== 200) {
+		return {
+			ok: false,
+			status: res.status,
+			statusText: res.statusText,
+		};
+	}
+
+	const body = await res.text();
+	const data = JSON.parse(body);
+
+	return curateResponse({
+		status: data.status,
+		statusText: data.statusText,
+		headers: data.headers,
+	}, data.data, options, {
+		url,
+		customOptions: options,
+		control: data.control,
+	});
+}
+
+function useRemoteRequest(options, method) {
+	if (options.remote.enabled) {
+		if (options.useRemote) {
+			return true;
+		}
+
+		if (options.remote.methods.includes(method.toLowerCase())) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
 async function browserRequest(url, customOptions = {}) {
 	const options = merge.all([{
 		timeout: 10000,
@@ -1339,6 +1402,10 @@ async function browserRequest(url, customOptions = {}) {
 		url,
 	}, globalOptions, customOptions]);
 
+	if (useRemoteRequest(options, 'browser')) {
+		return remoteRequest(url, 'browser', options);
+	}
+
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 	const agent = getAgent(options, url);
 
@@ -1523,6 +1590,10 @@ async function request(url, body, customOptions = {}, method = 'GET', redirects
 		return browserRequest(url, options);
 	}
 
+	if (useRemoteRequest(options, method)) {
+		return remoteRequest(url, method, options);
+	}
+
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 
 	const agent = getAgent(options, url);
@@ -1625,7 +1696,7 @@ function off(trigger, fn) {
 	events.off(trigger, fn);
 }
 
-module.exports = {
+const unprint = {
 	configure,
 	on,
 	off,
@@ -1655,3 +1726,9 @@ module.exports = {
 	options: configure,
 	query: initQueryFns(queryFns),
 };
+
+if (argv.server) {
+	initServer(argv.server, unprint);
+}
+
+module.exports = unprint;

package/src/server.js

@@ -0,0 +1,132 @@
+'use strict';
+
+const crypto = require('crypto');
+const express = require('express');
+
+require('dotenv').config();
+
+const pkg = require('../package.json');
+
+class HttpError extends Error {
+	constructor(message, httpCode, friendlyMessage, data) {
+		super(message);
+
+		this.name = 'HttpError';
+		this.httpCode = httpCode;
+
+		if (friendlyMessage) {
+			this.friendlyMessage = friendlyMessage;
+		}
+
+		if (data) {
+			this.data = data;
+		}
+	}
+}
+
+function log(level, ...data) {
+	const now = new Date();
+
+	console.log(`${now.toISOString()} [${level.slice(0, 5).padStart(5, ' ')}] ${data.join(' ')}`);
+}
+
+const logger = Object.fromEntries([
+	'info',
+	'debug',
+	'error',
+	'warn',
+].map((level) => [level, (...data) => log(level, ...data)]));
+
+function curateOptions(options) {
+	// make sure remote unprint doesn't get configured to make request to itself
+	return {
+		...options,
+		remote: {
+			enabled: false,
+		},
+		useRemote: false,
+		control: options.control
+			? async function control() {}.constructor('page', 'client', options.control) // eslint-disable-line no-eval,no-new-func,no-empty-function
+			: null,
+	};
+}
+
+async function handleRequest(req, res, unprint, method) {
+	if (!req.body?.url) {
+		throw new HttpError('No URL provided', 400);
+	}
+
+	logger.info(`${(method || req.body.method || 'get').toLowerCase()} ${req.body.url}`);
+
+	const options = curateOptions(req.body.options);
+
+	const unprintRes = req.body.method === 'post'
+		? await unprint.post(req.body.url, req.body.data, options)
+		: await unprint[(method || req.body.method || 'get').toLowerCase()](req.body.url, options);
+
+	res.send({
+		ok: unprintRes.ok,
+		status: unprintRes.status,
+		statusText: unprintRes.statusText,
+		data: unprintRes.data || null,
+		body: unprintRes.body || null,
+		html: unprintRes.context?.html || null,
+		headers: unprintRes.headers,
+		cookies: unprintRes.cookies,
+		control: unprintRes.control,
+	});
+}
+
+async function initServer(address, unprint) {
+	const app = express();
+	const addressComponents = typeof address === 'boolean' ? [] : String(address).split(':');
+
+	const host = addressComponents[1] ? addressComponents[0] : '127.0.0.1';
+	const port = addressComponents[1] || addressComponents[0] || 3000;
+
+	app.use(express.json());
+
+	app.use(async (req, res, next) => {
+		if (process.env.UNPRINT_KEY) {
+			if (process.env.UNPRINT_KEY.length !== req.headers['unprint-key']?.length
+			|| !crypto.timingSafeEqual(Buffer.from(process.env.UNPRINT_KEY, 'utf16le'), Buffer.from(req.headers['unprint-key'], 'utf16le'))) {
+				logger.warn(`Invalid key used by ${req.ip}`);
+				throw new HttpError('Invalid key', 401);
+			}
+		}
+
+		next();
+	});
+
+	app.get('/', (_req, res) => {
+		res.send(`unprint ${pkg.version}`);
+	});
+
+	app.post('/request', async (req, res) => handleRequest(req, res, unprint));
+	app.post('/browser', async (req, res) => handleRequest(req, res, unprint, 'browser'));
+
+	app.post('/options', async (req, res) => {
+		if (!req.body) {
+			throw new HttpError('No options provided', 400);
+		}
+
+		unprint.options(curateOptions(req.body));
+
+		res.status(204).send();
+	});
+
+	app.use((error, _req, res, _next) => {
+		logger.error(error);
+
+		res.status(error.httpCode || 500).send({
+			statusCode: error.httpCode || 500,
+			statusMessage: error.message,
+		});
+	});
+
+	app.listen(port, host, () => {
+		logger.info(`Started unprint server on ${host}:${port}`);
+	});
+}
+
+module.exports = initServer;

package/tests/remote.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const unprint = require('../src/app');
+
+unprint.configure({
+	remote: {
+		enabled: true,
+		address: 'http://127.0.0.1:3333',
+		key: 'foobar',
+	},
+});
+
+async function init() {
+	const res = await unprint.browser('https://www.google.com', {
+		useRemote: true,
+		async control(page) {
+			const form = await page.locator('form');
+
+			return form.count();
+		},
+	});
+
+	if (!res.ok) {
+		console.log(res);
+		return;
+	}
+
+	const form = res.context.query.element('form');
+
+	console.log('control', res.control);
+	console.log('form', form);
+}
+
+init();