unprint 0.0.0 → 0.3.0

package/.editorconfig

@@ -0,0 +1,14 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = tab
+indent_size = 4
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*.js]
+charset = utf-8

package/.eslintrc

@@ -0,0 +1,20 @@
+{
+    "extends": "airbnb-base",
+    "parserOptions": {
+        "sourceType": "script",
+		"ecmaVersion": 2020
+    },
+    "rules": {
+        "strict": 0,
+        "indent": "off",
+        "no-tabs": "off",
+        "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
+        "no-console": 0,
+        "no-underscore-dangle": 0,
+        "prefer-destructuring": "off",
+        "template-curly-spacing": "off",
+        "object-curly-newline": "off",
+		"default-param-last": "off",
+        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
+    }
+}

package/README.md

@@ -1 +1,76 @@
 # unprint
+unprint is a web scraping utility built around JSDOM, providing convenience methods for quickly extracting common data types.
+
+## Install
+`npm install unprint`
+
+## Usage
+`const unprint = require('unprint');`
+
+### Querying
+For optimal flexibility, unprint query methods can be used with or without initialization. If you already have access to DOM elements using another library or unprint instance, you can query it by using the uninitialized `query` methods provided directly from the library, and passing the element as the first argument, as such:
+
+`unprint.query.element(element, 'h1#title')` // HTMLHeadingElement
+
+Both `unprint.get()` and `unprint.init()` return its `query` methods pre-initialized, removing the element argument in favor of the element retrieved or received. Initialized query methods therefore will *not* accept a custom element, usually expecting the selector as the first argument instead.
+
+```javascript
+const result = await unprint.get('http://localhot:3101/html');
+const { query } = result.context;
+
+query.element('h1#title'); // HTMLHeadingElement
+```
+
+```javascript
+const result = await fetch('http://localhot:3101/html');
+const body = await res.text();
+const { query } = await unprint.init(body);
+
+query.element('h1#title'); // HTMLHeadingElement
+```
+
+**From here on, the query methods will be described in their initialized form.** The API for the *uninitialized* methods is identical, except for the element passed as the first argument
+
+#### Selector
+The selector can be a CSS selector, an XPath selector starting with `//`, or an array of either or both acting as fallbacks. If the selector is falsy, the input element will be used.
+
+#### Query an element
+* `query.element([selector], [options])`
+
+Returns the element node directly.
+
+#### Query a date
+`query.date(selector, format, [options])`
+
+Arguments
+* `format` (string, array): The input format as a string or array of strings described by the [Moment.js docs](https://momentjs.com/docs/#/displaying/format/).
+
+Options
+* `match (RegExp): The text to extract before attempting to parse it as a date. The default expression will attempt to extract any of 01-01-1970, 1970-01-01, 01/01/1970 or January 1, 1970 with optional 00:00[:00] time.
+* `timezone` (string): The name of the input timezone, defaults to 'UTC'.
+
+Returns a Date object.
+
+#### Querying multiple elements
+Most methods can be used in plural, returning an array of results, i.e. `query.elements()`, `query.dates()`.
+
+### HTTP request
+* `unprint.get(url, [options])`
+* `unprint.post(url, body, [options])`
+
+Options
+* `select`: Pre-query and initialize a specific element on the page
+* `selectAll`: Pre-query and initialize multiple specific element on the page
+
+Returns
+```javascript
+{
+	query,				// (object)		unprint querying methods
+	html,				// (string)		HTML body
+	data,				// (object)		parsed JSON response
+	status,				// (number)		HTTP status code
+	ok,					// (boolean)	status code >= 200 and < 300
+	response,			// (object)		the original axios response object, alias 'res'
+	res,				// (object)		alias for 'response'
+}
+```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.0.0",
+  "version": "0.3.0",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {
@@ -22,8 +22,15 @@
   },
   "homepage": "https://github.com/ThePendulum/unprint#readme",
   "dependencies": {
-    "bhttp": "^1.2.8",
-    "dayjs": "^1.10.7",
-    "jsdom": "^17.0.0"
+    "axios": "^0.27.2",
+    "bottleneck": "^2.19.5",
+    "eslint": "^8.17.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "jsdom": "^17.0.0",
+    "moment-timezone": "^0.5.34"
+  },
+  "devDependencies": {
+    "express": "^4.18.1"
   }
 }

package/src/app.js

@@ -1,3 +1,563 @@
 'use strict';
 
-console.log('unprint');
+const { JSDOM, VirtualConsole } = require('jsdom');
+const axios = require('axios').default;
+const moment = require('moment-timezone');
+
+const settings = {
+	throwErrors: false,
+	logErrors: true,
+	requestTimeout: 30000,
+};
+
+const virtualConsole = new VirtualConsole();
+const { window: globalWindow } = new JSDOM('', { virtualConsole });
+
+function handleError(error, code) {
+	if (settings.logErrors) {
+		console.error(`unprint encountered an error (${code}): ${error.message}`);
+	}
+
+	if (settings.throwErrors) {
+		throw Object.assign(error, { code });
+	}
+
+	return null;
+}
+
+virtualConsole.on('error', (message) => handleError(message, 'JSDOM'));
+virtualConsole.on('jsdomError', (message) => handleError(message, 'JSDOM'));
+
+const defaultOptions = {
+	trim: true,
+};
+
+function trim(string) {
+	if (typeof string === 'string') {
+		return string.trim().replace(/\s+/g, ' ');
+	}
+
+	return string;
+}
+
+function iterateXpathResult(iterator, results = []) {
+	const element = iterator.iterateNext();
+
+	if (element) {
+		return iterateXpathResult(iterator, results.concat(element));
+	}
+
+	return results;
+}
+
+function getElements(context, selector, firstOnly = false) {
+	if (!selector) {
+		return context.element;
+	}
+
+	if (/^\/\//.test(selector)) {
+		// XPath selector
+		const iterator = globalWindow.document.evaluate(selector, context.element, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
+
+		if (firstOnly) {
+			return iterator.iterateNext();
+		}
+
+		return iterateXpathResult(iterator);
+	}
+
+	if (firstOnly) {
+		return context.element.querySelector(selector);
+	}
+
+	return Array.from(context.element.querySelectorAll(selector));
+}
+
+function queryElement(context, selectors, _customOptions) {
+	if (!selectors && context.element.nodeName === '#document') {
+		return null;
+	}
+
+	const target = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, true), null);
+
+	return target || null;
+}
+
+function queryElements(context, selectors, _customOptions) {
+	if (!selectors) {
+		return context.element;
+	}
+
+	const targets = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, false), null);
+
+	return targets || [];
+}
+
+function queryExistence(context, selector, customOptions) {
+	return !!queryElement(context, selector, customOptions);
+}
+
+function queryCount(context, selector, customOptions) {
+	return queryElements(context, selector, customOptions)?.length || 0;
+}
+
+function getAttributeKey(options) {
+	if (!options) {
+		return null;
+	}
+
+	if (Object.hasOwn(options, 'attr')) {
+		return options.attr;
+	}
+
+	if (Object.hasOwn(options, 'attribute')) {
+		return options.attribute;
+	}
+
+	return null;
+}
+
+function extractContent(element, options) {
+	if (!element) {
+		return null;
+	}
+
+	const attributeKey = getAttributeKey(options);
+
+	if (attributeKey) {
+		// handle attribute extraction in content method so all methods can easily optionally query a specific attribute
+		const attribute = element[attributeKey] || element.getAttribute(attributeKey);
+
+		if (attribute && options.trim) {
+			return trim(attribute);
+		}
+
+		return attribute;
+	}
+
+	if (options.trim) {
+		return trim(element.textContent);
+	}
+
+	return element.textContent;
+}
+
+function queryContent(context, selector, customOptions) {
+	const options = { ...context.options, ...customOptions };
+	const target = queryElement(context, selector, options);
+
+	return extractContent(target, options);
+}
+
+function queryContents(context, selector, customOptions) {
+	const options = { ...context.options, ...customOptions };
+	const targets = queryElements(context, selector, options);
+
+	return targets.map((target) => extractContent(target, options)).filter(Boolean);
+}
+
+function queryAttribute(context, selector, attribute, customOptions) {
+	return queryContent(context, selector, {
+		...customOptions,
+		attribute,
+	});
+}
+
+function queryAttributes(context, selector, attribute, customOptions) {
+	return queryContents(context, selector, {
+		...customOptions,
+		attribute,
+	});
+}
+
+function queryHtml(context, selector, customOptions) {
+	const target = queryElement(context, selector, customOptions);
+
+	if (target) {
+		return trim(target.innerHTML);
+	}
+
+	return null;
+}
+
+function queryHtmls(context, selector, customOptions) {
+	const targets = queryElements(context, selector, customOptions);
+
+	return targets.map((target) => trim(target.innerHTML));
+}
+
+function prefixUrl(urlPath, originUrl, customOptions) {
+	if (!urlPath) {
+		return null;
+	}
+
+	if (!originUrl) {
+		return urlPath;
+	}
+
+	const options = {
+		protocol: 'https',
+		...customOptions,
+	};
+
+	const { origin, protocol } = new URL(originUrl);
+
+	if (/^http/.test(urlPath)) {
+		// this is already a complete URL
+		return urlPath;
+	}
+
+	if (options.protocol && /^\/\//.test(urlPath)) {
+		return `${options.protocol.replace(/:$/, '')}:${urlPath}`; // allow protocol to be defined either as 'https' or 'https:'
+	}
+
+	if (protocol && /^\/\//.test(urlPath)) {
+		return `${protocol}${urlPath}`;
+	}
+
+	if (/^\//.test(urlPath)) {
+		return `${origin}${urlPath}`;
+	}
+
+	if (/^\.\//.test(urlPath)) {
+		return `${originUrl.replace(/\/+$/, '')}${urlPath.slice(1)}`;
+	}
+
+	return `${origin}/${urlPath}`;
+}
+
+function queryUrl(context, selector = 'a', customOptions) {
+	const options = {
+		...context.options,
+		attribute: 'href',
+		...customOptions,
+	};
+
+	const url = queryContent(context, selector, options);
+	const curatedUrl = prefixUrl(url, options.origin, customOptions);
+
+	return curatedUrl;
+}
+
+function getImageUrl(context, selector, options) {
+	const attributeKey = getAttributeKey(options);
+
+	if (attributeKey) {
+		return queryAttribute(context, selector, attributeKey, options);
+	}
+
+	return queryAttribute(context, selector, 'data-src', options)
+		|| queryAttribute(context, selector, 'src', options);
+}
+
+function getImageUrls(context, selector, options) {
+	const attributeKey = getAttributeKey(options);
+
+	if (attributeKey) {
+		return queryAttributes(context, selector, attributeKey, options);
+	}
+
+	const dataLinks = queryAttributes(context, selector, 'data-src', options);
+
+	if (dataLinks.lenght > 0) {
+		return dataLinks;
+	}
+
+	return queryAttributes(context, selector, 'src', options);
+}
+
+function queryImage(context, selector = 'img', customOptions) {
+	const options = {
+		...context.options,
+		...customOptions,
+	};
+
+	const imageUrl = getImageUrl(context, selector, options);
+
+	return prefixUrl(imageUrl, options.origin, options);
+}
+
+function queryImages(context, selector = 'img', customOptions) {
+	const options = {
+		...context.options,
+		...customOptions,
+	};
+
+	const imageUrls = getImageUrls(context, selector, options);
+
+	return imageUrls.map((imageUrl) => prefixUrl(imageUrl, options.origin, options));
+}
+
+function extractJson(element) {
+	if (!element) {
+		return null;
+	}
+
+	try {
+		return JSON.parse(element.innerHTML);
+	} catch (error) {
+		return null;
+	}
+}
+
+function queryJson(context, selector, customOptions) {
+	const target = queryElement(context, selector, customOptions);
+
+	return extractJson(target);
+}
+
+function queryJsons(context, selector, customOptions) {
+	const targets = queryElements(context, selector, customOptions);
+
+	return targets.map((target) => extractJson(target)).filter(Boolean);
+}
+
+function extractDate(dateString, format, customOptions) {
+	if (!dateString) {
+		return null;
+	}
+
+	if (!format) {
+		return handleError(new Error('Missing required date format parameter'), 'NO_DATE_FORMAT');
+	}
+
+	const options = {
+		match: /((\d{1,4}[/-]\d{1,2}[/-]\d{1,4})|(\w+\s+\d{1,2},?\s+\d{4}))(\s+\d{1,2}:\d{2}(:\d{2})?)?/g, // matches any of 01-01-1970, 1970-01-01 and January 1, 1970 with optional 00:00[:00] time
+		timezone: 'UTC',
+		...customOptions,
+	};
+
+	const dateStamp = options.match
+		? trim(dateString).match(options.match)
+		: trim(dateString);
+
+	if (dateStamp) {
+		const dateValue = moment.tz(options.match ? dateStamp[0] : dateStamp, format, options.timezone);
+
+		if (dateValue.isValid()) {
+			return dateValue.toDate();
+		}
+	}
+
+	return null;
+}
+
+function queryDate(context, selector, format, customOptions) {
+	const dateString = queryContent(context, selector, customOptions);
+
+	return extractDate(dateString, format, {
+		...context.options,
+		...customOptions,
+	});
+}
+
+function queryDates(context, selector, format, customOptions) {
+	const dateStrings = queryContents(context, selector, customOptions);
+
+	return dateStrings.map((dateString) => extractDate(dateString, format, {
+		...context.options,
+		customOptions,
+	}));
+}
+
+const queryFns = {
+	element: queryElement,
+	elements: queryElements,
+	el: queryElement,
+	els: queryElements,
+	all: queryElements,
+	content: queryContent,
+	contents: queryContents,
+	attribute: queryAttribute,
+	attributes: queryAttributes,
+	attr: queryAttribute,
+	attrs: queryAttributes,
+	exists: queryExistence,
+	count: queryCount,
+	html: queryHtml,
+	htmls: queryHtmls,
+	image: queryImage,
+	images: queryImages,
+	img: queryImage,
+	imgs: queryImages,
+	json: queryJson,
+	jsons: queryJsons,
+	date: queryDate,
+	dates: queryDates,
+	url: queryUrl,
+};
+
+function isDomObject(element) {
+	if (!element) {
+		return false;
+	}
+
+	return typeof element.nodeType !== 'undefined';
+}
+
+function initQueryFns(fns, context) {
+	if (context) {
+		return Object.fromEntries(Object.entries(fns).map(([key, fn]) => [key, (...args) => fn(context, ...args)]));
+	}
+
+	// context is passed directly to query method
+	return Object.fromEntries(Object.entries(fns).map(([key, fn]) => [key, (...args) => {
+		// first argument is already an unprint context. this seems like a convoluted approach, but there is little reason not to allow it
+		if (args[0]?.isUnprint) {
+			return fn(...args);
+		}
+
+		// most common usage is to pass an element directly, convert to context
+		if (isDomObject(args[0])) {
+			const element = args[0];
+
+			return fn({
+				element,
+				html: element.outerHTML || element.body?.outerHTML,
+				isUnprint: true,
+			}, ...args.slice(1));
+		}
+
+		return handleError(new Error('Context is not provided or initialized'), 'INVALID_CONTEXT');
+	}]));
+}
+
+function init(elementOrHtml, selector, options) {
+	if (!elementOrHtml) {
+		return null;
+	}
+
+	if (typeof elementOrHtml === 'string') {
+		// the context should be raw HTML
+		const { window } = new JSDOM(elementOrHtml, { virtualConsole, ...options.parser });
+
+		return init(window.document, selector, { ...options, window });
+	}
+
+	if (!isDomObject(elementOrHtml)) {
+		// the context is not a valid
+		return handleError(new Error('Init context is not a DOM element, HTML or an array'), 'INVALID_CONTEXT');
+	}
+
+	const element = selector
+		? elementOrHtml.querySelector(selector)
+		: elementOrHtml;
+
+	if (!element) {
+		return null;
+	}
+
+	const context = {
+		element,
+		html: element.outerHTML || element.body?.outerHTML,
+		...(options.window && {
+			window: options.window,
+			document: options.window.document,
+		}),
+		options,
+		isUnprint: true,
+	};
+
+	context.query = initQueryFns(queryFns, context);
+
+	return context;
+}
+
+function initAll(context, selector, options) {
+	if (Array.isArray(context)) {
+		return context.map((element) => init(element, selector, options));
+	}
+
+	if (typeof context === 'string') {
+		// the context should be raw HTML
+		const { window } = new JSDOM(context, { virtualConsole, ...options.parser });
+
+		return initAll(window.document, selector, { ...options, window });
+	}
+
+	if (!(context instanceof globalWindow.HTMLElement)) {
+		// the context is not a valid
+		return handleError(new Error('Init context is not a DOM element, HTML or an array'), 'INVALID_CONTEXT');
+	}
+
+	return Array.from(context.querySelectorAll(options.select))
+		.map((element) => init(element, selector, options));
+}
+
+async function request(url, body, customOptions = {}, method = 'GET') {
+	const options = {
+		timeout: 1000,
+		extract: true,
+		url,
+		...customOptions,
+	};
+
+	const res = await axios({
+		url,
+		method,
+		data: body,
+		validateStatus: null,
+		timeout: options.timeout,
+		signal: options.abortSignal,
+		...options,
+	});
+
+	if (!(res.status >= 200 && res.status < 300)) {
+		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${res.data}`), 'HTTP_NOT_OK');
+
+		return res.status;
+	}
+
+	const base = {
+		ok: true,
+		status: res.status,
+		statusText: res.statusText,
+		response: res,
+		res,
+	};
+
+	if (res.headers['content-type'].includes('application/json') && typeof res.data === 'object') {
+		return {
+			...base,
+			data: res.data,
+		};
+	}
+
+	if (!options.extract) {
+		return base;
+	}
+
+	const contextOptions = {
+		...defaultOptions,
+		origin: url,
+	};
+
+	const context = options.selectAll
+		? initAll(res.data, options.selectAll, contextOptions)
+		: init(res.data, options.select, contextOptions);
+
+	return {
+		...base,
+		context,
+	};
+}
+
+async function get(url, options) {
+	return request(url, null, options, 'GET');
+}
+
+async function post(url, body, options) {
+	return request(url, body, options, 'POST');
+}
+
+module.exports = {
+	get,
+	post,
+	request,
+	initialize: init,
+	initializeAll: initAll,
+	init,
+	initAll,
+	extractDate,
+	query: initQueryFns(queryFns),
+};

package/tests/data.json

@@ -0,0 +1,5 @@
+{
+    "name": "Testos Theron",
+    "age": 42,
+    "hobbies": ["skydiving", "knitting"]
+}

package/tests/index.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+		<meta charset="UTF-8">
+		<title>Test page</title>
+    </head>
+
+    <body>
+		<h1 id="title">Test page</h1>
+
+		<ul id="items">
+			<li class="item">Item 1</li>
+			<li class="item">Item 2</li>
+			<li class="item">Item 3</li>
+		</ul>
+
+		<a id="link" href="http://localhost:3101/html">Get HTML</a>
+		<a id="path" href="/json">Get data</a>
+		<a id="relativePath" href="./json">Get data</a>
+
+		<div id="date">Date: 22-07-2022 02:00</div>
+		<div id="date2">Date: 13-05-2022 18:00</div>
+
+		<img class="image" src="https://i.redd.it/vn9h981hlx281.png">
+		<img class="image" src="https://i.redd.it/1s22dsrqy0181.jpg">
+		<img class="image" src="https://i.redd.it/e91oo4ueyeb71.jpg">
+
+		<video id="video"><source src="https://i.imgur.com/eDQmLys.mp4"></video>
+
+		<script id="json" type="application/js">{"foo": "bar", "lorem": "ipsum", "hello": "world"}</script>
+    </body>
+</html>

package/tests/init.js

@@ -0,0 +1,66 @@
+'use strict';
+
+const path = require('path');
+const express = require('express');
+// const unprint = require('unprint');
+
+const unprint = require('../src/app');
+const data = require('./data.json');
+
+const port = process.env.PORT || 3101;
+
+async function initTest() {
+	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
+	// const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
+	// const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
+
+	console.log('title', res.context.query.content('//*[contains(text(), "Test")]'));
+	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));
+	console.log('data', res.context.query.json('#json'));
+	console.log('items', res.context.query.contents('.item'));
+	console.log('link', res.context.query.url('#link'));
+	console.log('image', res.context.query.img('.image'));
+	console.log('images', res.context.query.imgs('.image'));
+	console.log('path', res.context.query.url('#path'));
+	console.log('relative path', res.context.query.url('#relativePath'));
+	console.log('exists', res.context.query.exists('#title'));
+	console.log('count', res.context.query.count('.item'), res.context.query.count('.foo'));
+}
+
+async function initServer() {
+	const app = express();
+
+	app.use((req, res, next) => {
+		if (req.query.delay) {
+			setTimeout(() => {
+				next();
+			}, req.query.delay);
+
+			return;
+		}
+
+		next();
+	});
+
+	app.get('/html', (req, res) => {
+		res.sendFile(path.resolve(__dirname, 'index.html'));
+	});
+
+	app.get('/json', (req, res) => {
+		res.send(data);
+	});
+
+	app.get('/error/:code', (req, res) => {
+		res.status(Number(req.params.code)).send();
+	});
+
+	const server = app.listen(port, async () => {
+		const { address } = server.address();
+
+		console.log(`Test server listening on ${address}:${port}`);
+
+		await initTest();
+	});
+}
+
+initServer();