unprint 0.0.0 → 0.2.1

package/.editorconfig

@@ -0,0 +1,14 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = tab
+indent_size = 4
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*.js]
+charset = utf-8

package/.eslintrc

@@ -0,0 +1,20 @@
+{
+    "extends": "airbnb-base",
+    "parserOptions": {
+        "sourceType": "script",
+		"ecmaVersion": 2020
+    },
+    "rules": {
+        "strict": 0,
+        "indent": "off",
+        "no-tabs": "off",
+        "no-unused-vars": ["error", {"argsIgnorePattern": "^_"}],
+        "no-console": 0,
+        "no-underscore-dangle": 0,
+        "prefer-destructuring": "off",
+        "template-curly-spacing": "off",
+        "object-curly-newline": "off",
+		"default-param-last": "off",
+        "max-len": [2, {"code": 300, "tabWidth": 4, "ignoreUrls": true}]
+    }
+}

package/README.md

@@ -1 +1,74 @@
 # unprint
+unprint is a web scraping utility built around JSDOM, providing convenience methods for quickly extracting common data types.
+
+## Install
+`npm install unprint`
+
+## Usage
+`const unprint = require('unprint');`
+
+### Querying
+For optimal flexibility, unprint query methods can be used with or without initialization. If you already have access to DOM elements using another library or unprint instance, you can query it by using the uninitialized `query` methods provided directly from the library, and passing the element as the first argument, as such:
+
+`unprint.query.element(element, 'h1#title')` // HTMLHeadingElement
+
+Both `http.get()` and `http.init()` return its `query` methods pre-initialized, removing the element argument in favor of the element retrieved or received. Initialized query methods therefore will *not* accept a custom element, usually expecting the selector as the first argument instead.
+
+```javascript
+const result = await unprint.get('http://localhot:3101/html');
+result.context.query.element('h1#title'); // HTMLHeadingElement
+```
+
+```javascript
+const result = await fetch('http://localhot:3101/html');
+const body = await res.text();
+const context = await unprint.init(body);
+
+context.query.element('h1#title'); // HTMLHeadingElement
+```
+
+**From here on, the query methods will be described in their initialized form.** The API for the *uninitialized* methods is identical, except for the element passed as the first argument
+
+#### Selector
+The selector can be a CSS selector, an XPath selector starting with `//`, or an array of either or both acting as fallbacks. If the selector is falsy, the input element will be used.
+
+#### Query an element
+* `query.element([selector], [options])`
+
+Returns the element node directly.
+
+#### Query a date
+`query.date([selector], format, [options])`
+
+Arguments
+* `format` (string, array): The input format as a string or array of strings described by the [Moment.js docs](https://momentjs.com/docs/#/displaying/format/).
+
+Options
+* `match (RegExp): The text to extract before attempting to parse it as a date. The default expression will attempt to extract any of 01-01-1970, 1970-01-01, 01/01/1970 or January 1, 1970 with optional 00:00[:00] time.
+* `timezone` (string): The name of the input timezone, defaults to 'UTC'.
+
+Returns a Date object.
+
+#### Querying multiple elements
+Most methods can be used in plural, returning an array of results, i.e. `query.elements()`, `query.dates()`.
+
+### HTTP request
+`unprint.get(url, [options])`
+`unprint.post(url, body, [options])`
+
+Options
+* `select`: Pre-query and initialize a specific element on the page
+* `selectAll`: Pre-query and initialize multiple specific element on the page
+
+Returns
+```javascript
+{
+	query,				// (object)		unprint querying methods
+	html,				// (string)		HTML body
+	data,				// (object)		parsed JSON response
+	status,				// (number)		HTTP status code
+	ok,					// (boolean)	status code >= 200 and < 300
+	response,			// (object)		the original axios response object, alias 'res'
+	res,				// (object)		alias for 'response'
+}
+```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.0.0",
+  "version": "0.2.1",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {
@@ -22,8 +22,14 @@
   },
   "homepage": "https://github.com/ThePendulum/unprint#readme",
   "dependencies": {
-    "bhttp": "^1.2.8",
+    "axios": "^0.27.2",
+    "bottleneck": "^2.19.5",
     "dayjs": "^1.10.7",
-    "jsdom": "^17.0.0"
+    "eslint": "^8.17.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "jsdom": "^17.0.0",
+    "moment-timezone": "^0.5.34",
+    "tunnel": "0.0.6"
   }
 }

package/src/app.js

@@ -1,3 +1,376 @@
 'use strict';
 
-console.log('unprint');
+const { JSDOM, VirtualConsole } = require('jsdom');
+const axios = require('axios').default;
+const moment = require('moment-timezone');
+
+const settings = {
+	throwErrors: false,
+	logErrors: true,
+	requestTimeout: 30000,
+};
+
+const virtualConsole = new VirtualConsole();
+const { window: globalWindow } = new JSDOM('', { virtualConsole });
+
+function handleError(error, code) {
+	if (settings.logErrors) {
+		console.error(`unprint encountered an error (${code}): ${error.message}`);
+	}
+
+	if (settings.throwErrors) {
+		throw Object.assign(error, { code });
+	}
+
+	return null;
+}
+
+virtualConsole.on('error', (message) => handleError(message, 'JSDOM'));
+virtualConsole.on('jsdomError', (message) => handleError(message, 'JSDOM'));
+
+const defaultOptions = {
+	trim: true,
+};
+
+function trim(string) {
+	if (typeof string === 'string') {
+		return string.trim().replace(/\s+/g, ' ');
+	}
+
+	return string;
+}
+
+function iterateXpathResult(iterator, results = []) {
+	const element = iterator.iterateNext();
+
+	if (element) {
+		return iterateXpathResult(iterator, results.concat(element));
+	}
+
+	return results;
+}
+
+function getElements(element, selector, firstOnly = false) {
+	if (!selector) {
+		return element;
+	}
+
+	if (/^\/\//.test(selector)) {
+		// XPath selector
+		const iterator = globalWindow.document.evaluate(selector, element, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
+
+		if (firstOnly) {
+			return iterator.iterateNext();
+		}
+
+		return iterateXpathResult(iterator);
+	}
+
+	if (firstOnly) {
+		return element.querySelector(selector);
+	}
+
+	return Array.from(element.querySelectorAll(selector));
+}
+
+function queryElement(element, selectors, _customOptions) {
+	if (!selectors && element.nodeName === '#document') {
+		return null;
+	}
+
+	const target = [].concat(selectors).reduce((acc, selector) => acc || getElements(element, selector, true), null);
+
+	return target || null;
+}
+
+function queryElements(element, selectors, _customOptions) {
+	if (!selectors) {
+		return element;
+	}
+
+	const targets = [].concat(selectors).reduce((acc, selector) => acc || getElements(element, selector, false), null);
+
+	return targets || [];
+}
+
+function queryExistence(element, selector, customOptions) {
+	return !!queryElement(element, selector, customOptions);
+}
+
+function extractContent(element, options) {
+	if (options.attribute) {
+		const attribute = element[options.attribute] || element.getAttribute(options.attribute);
+
+		if (attribute && options.trim) {
+			return trim(attribute);
+		}
+
+		return attribute;
+	}
+
+	if (options.trim) {
+		return trim(element.textContent);
+	}
+
+	return element.textContent;
+}
+
+function queryContent(element, selector, customOptions) {
+	const options = { ...defaultOptions, ...customOptions };
+	const target = queryElement(element, selector, options);
+
+	return extractContent(target, options);
+}
+
+function queryContents(element, selector, customOptions) {
+	const options = { ...defaultOptions, ...customOptions };
+	const targets = queryElements(element, selector, options);
+
+	return targets.map((target) => extractContent(target, options));
+}
+
+function queryAttribute(element, selector, attribute, customOptions) {
+	return queryContent(element, selector, {
+		...customOptions,
+		attribute,
+	});
+}
+
+function queryAttributes(element, selector, attribute, customOptions) {
+	return queryContents(element, selector, {
+		...customOptions,
+		attribute,
+	});
+}
+
+function queryHtml(element, selector, customOptions) {
+	const target = queryElement(element, selector, customOptions);
+
+	if (target) {
+		return trim(target.innerHTML);
+	}
+
+	return null;
+}
+
+function queryHtmls(element, selector, customOptions) {
+	const targets = queryElements(element, selector, customOptions);
+
+	return targets.map((target) => trim(target.innerHTML));
+}
+
+function extractJson(element) {
+	if (!element) {
+		return null;
+	}
+
+	try {
+		return JSON.parse(element.innerHTML);
+	} catch (error) {
+		return null;
+	}
+}
+
+function queryJson(element, selector, customOptions) {
+	const target = queryElement(element, selector, customOptions);
+
+	return extractJson(target);
+}
+
+function queryJsons(element, selector, customOptions) {
+	const targets = queryElements(element, selector, customOptions);
+
+	return targets.map((target) => extractJson(target)).filter(Boolean);
+}
+
+function extractDate(dateString, format, customOptions) {
+	if (!dateString) {
+		return null;
+	}
+
+	if (!format) {
+		return handleError(new Error('Missing required date format parameter'), 'NO_DATE_FORMAT');
+	}
+
+	const options = {
+		...defaultOptions,
+		match: /((\d{1,4}[/-]\d{1,2}[/-]\d{1,4})|(\w+\s+\d{1,2},?\s+\d{4}))(\s+\d{1,2}:\d{2}(:\d{2})?)?/g, // matches any of 01-01-1970, 1970-01-01 and January 1, 1970 with optional 00:00[:00] time
+		timezone: 'UTC',
+		...customOptions,
+	};
+
+	const dateStamp = options.match
+		? trim(dateString).match(options.match)
+		: trim(dateString);
+
+	if (dateStamp) {
+		const dateValue = moment.tz(options.match ? dateStamp[0] : dateStamp, format, options.timezone);
+
+		if (dateValue.isValid()) {
+			return dateValue.toDate();
+		}
+	}
+
+	return null;
+}
+
+function queryDate(element, selector, format, customOptions) {
+	const dateString = queryContent(element, selector, customOptions);
+
+	return extractDate(dateString, format, customOptions);
+}
+
+function queryDates(element, selector, format, customOptions) {
+	const dateStrings = queryContents(element, selector, customOptions);
+
+	return dateStrings.map((dateString) => extractDate(dateString, format, customOptions));
+}
+
+const queryFns = {
+	element: queryElement,
+	elements: queryElements,
+	el: queryElement,
+	els: queryElements,
+	all: queryElements,
+	content: queryContent,
+	contents: queryContents,
+	attribute: queryAttribute,
+	attributes: queryAttributes,
+	attr: queryAttribute,
+	attrs: queryAttributes,
+	exists: queryExistence,
+	html: queryHtml,
+	htmls: queryHtmls,
+	json: queryJson,
+	jsons: queryJsons,
+	date: queryDate,
+	dates: queryDates,
+	extractDate,
+};
+
+function initFns(context, fns) {
+	return Object.fromEntries(Object.entries(fns).map(([key, fn]) => [key, (...args) => fn(context, ...args)]));
+}
+
+function init(context, selector, options) {
+	if (!context) {
+		return null;
+	}
+
+	if (typeof context === 'string') {
+		// the context should be raw HTML
+		const { window } = new JSDOM(context, { virtualConsole, ...options.parser });
+
+		return init(window.document, selector, { ...options, window });
+	}
+
+	if (!context.querySelector) {
+		// the context is not a valid
+		return null;
+	}
+
+	const element = selector
+		? context.querySelector(selector)
+		: context;
+
+	if (!element) {
+		return null;
+	}
+
+	return {
+		element,
+		html: element.outerHTML || element.body.outerHTML,
+		...(options.window && {
+			window: options.window,
+			document: options.window.document,
+		}),
+		query: initFns(context, queryFns),
+	};
+}
+
+function initAll(context, selector, options) {
+	if (Array.isArray(context)) {
+		return context.map((element) => init(element, selector, options));
+	}
+
+	if (typeof context === 'string') {
+		// the context should be raw HTML
+		const { window } = new JSDOM(context, { virtualConsole, ...options.parser });
+
+		return initAll(window.document, selector, { ...options, window });
+	}
+
+	if (!(context instanceof globalWindow.HTMLElement)) {
+		handleError(new Error('Init context is not a DOM element, HTML or an array'), 'INVALID_CONTEXT');
+	}
+
+	return Array.from(context.querySelectorAll(options.select))
+		.map((element) => init(element, selector, options));
+}
+
+async function request(url, data, customOptions = {}, method = 'GET') {
+	const options = {
+		timeout: 1000,
+		extract: true,
+		...customOptions,
+	};
+
+	const res = await axios({
+		url,
+		method,
+		data,
+		validateStatus: null,
+		timeout: options.timeout,
+		signal: options.abortSignal,
+		...options,
+	});
+
+	if (!(res.status >= 200 && res.status < 300)) {
+		handleError(new Error(`HTTP response from ${url} not OK (${res.status} ${res.statusText}): ${res.data}`), 'HTTP_NOT_OK');
+
+		return res.status;
+	}
+
+	if (res.headers['content-type'].includes('application/json') && typeof res.data === 'object') {
+		return {
+			data,
+			ok: true,
+			status: res.status,
+			statusText: res.statusText,
+			response: res,
+			res,
+		};
+	}
+
+	const context = options.selectAll
+		? initAll(res.data, options.selectAll, options)
+		: init(res.data, options.select, options);
+
+	return {
+		context,
+		html: res.data,
+		ok: true,
+		status: res.status,
+		response: res,
+		res,
+	};
+}
+
+async function get(url, options) {
+	return request(url, null, options, 'GET');
+}
+
+async function post(url, body, options) {
+	return request(url, body, options, 'POST');
+}
+
+module.exports = {
+	get,
+	post,
+	request,
+	initialize: init,
+	initializeAll: initAll,
+	init,
+	initAll,
+	query: queryFns,
+};

package/tests/data.json

@@ -0,0 +1,5 @@
+{
+    "name": "Testos Theron",
+    "age": 42,
+    "hobbies": ["skydiving", "knitting"]
+}

package/tests/index.html

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+		<meta charset="UTF-8">
+		<title>Test page</title>
+    </head>
+
+    <body>
+		<h1 id="title">Test page</h1>
+
+		<ul id="items">
+			<li class="item">Item 1</li>
+			<li class="item">Item 2</li>
+			<li class="item">Item 3</li>
+		</ul>
+
+		<div id="date">Date: 22-07-2022 02:00</div>
+		<div id="date2">Date: 13-05-2022 18:00</div>
+
+		<script id="json" type="application/js">{"foo": "bar", "lorem": "ipsum", "hello": "world"}</script>
+    </body>
+</html>

package/tests/init.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const path = require('path');
+const express = require('express');
+
+const unprint = require('unprint');
+const data = require('./data.json');
+
+const port = process.env.PORT || 3101;
+
+async function initTest() {
+	console.log(unprint);
+
+	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
+	// const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
+	// const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);
+
+	console.log('title', res.context.query.element('#title'), res.context.query.content('//*[contains(text(), "Test")]'));
+	console.log('date', res.context.query.date('#date', 'DD-MM-YYYY HH:mm'));
+	console.log('data', res.context.query.json('#json'));
+	console.log('items', res.context.query.contents('.item'));
+}
+
+async function initServer() {
+	const app = express();
+
+	app.use((req, res, next) => {
+		if (req.query.delay) {
+			setTimeout(() => {
+				next();
+			}, req.query.delay);
+
+			return;
+		}
+
+		next();
+	});
+
+	app.get('/html', (req, res) => {
+		res.sendFile(path.resolve(__dirname, 'index.html'));
+	});
+
+	app.get('/json', (req, res) => {
+		res.send(data);
+	});
+
+	app.get('/error/:code', (req, res) => {
+		res.status(Number(req.params.code)).send();
+	});
+
+	const server = app.listen(port, async () => {
+		const { address } = server.address();
+
+		console.log(`Test server listening on ${address}:${port}`);
+
+		await initTest();
+	});
+}
+
+initServer();