unprint 0.14.2 → 0.15.0

package/README.md

@@ -181,6 +181,19 @@ Options
 
 Returns the CSS style attributes of an element as an object.
 
+#### Query style URL
+`query.styleUrl([selector], [styleAttribute], [options])`
+
+Extracts the CSS `url()` link from a style attribute, such as a background.
+
+Arguments
+* `styleAttribute`: the CSS style attribute to extract the URL from
+
+#### Query style background
+`query.styleUrl([selector], [options])`
+
+Extracts the CSS `url()` background from a style attribute. Alias for `query.styleUrl([selector], 'background-image', [options])`.
+
 ### HTTP request
 * `unprint.get(url, [options])`
 * `unprint.post(url, body, [options])`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.14.2",
+  "version": "0.15.0",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {

package/src/app.js

@@ -523,6 +523,8 @@ function queryStyle(context, selector, customOptions) {
 			removeStyleFunctionSpaces(element, options);
 		}
 
+		console.log('OPTIONS', options.styleAttribute);
+
 		if (element.style) {
 			return options.styleAttribute
 				? element.style.getPropertyValue(options.styleAttribute)
@@ -554,6 +556,45 @@ function queryStyles(context, selector, customOptions) {
 	return elStyles.filter(Boolean);
 }
 
+function queryStyleUrl(context, selector, styleAttribute, customOptions) {
+	const options = {
+		styleAttribute,
+		...customOptions,
+	};
+
+	const style = queryStyle(context, selector, options);
+
+	if (!style) {
+		return null;
+	}
+
+	console.log('STYLE', styleAttribute, style);
+
+	const url = style.match(/url\(['"]?(.*)['"]?\)/)?.[1];
+
+	return url;
+}
+
+function queryStyleUrls(context, selector, styleAttribute, customOptions) {
+	const options = {
+		styleAttribute,
+		...customOptions,
+	};
+
+	const styles = queryStyles(context, selector, options);
+	const urls = styles.map((style) => style.match(/url\(['"]?(.*)['"]?\)/)?.[1])?.filter(Boolean);
+
+	return urls;
+}
+
+function queryBackground(context, selector, customOptions) {
+	return queryStyleUrl(context, selector, 'background-image', customOptions);
+}
+
+function queryBackgrounds(context, selector, customOptions) {
+	return queryStyleUrls(context, selector, 'background-image', customOptions);
+}
+
 function queryVideo(context, selector = 'video source', customOptions) {
 	const options = {
 		...context.options,
@@ -738,6 +779,8 @@ const queryFns = {
 	attributes: queryAttributes,
 	attr: queryAttribute,
 	attrs: queryAttributes,
+	background: queryBackground,
+	backgrounds: queryBackgrounds,
 	dataset: queryDataset,
 	datasets: queryDatasets,
 	data: queryDataset,
@@ -754,6 +797,8 @@ const queryFns = {
 	jsons: queryJsons,
 	style: queryStyle,
 	styles: queryStyles,
+	styleUrl: queryStyleUrl,
+	styleUrls: queryStyleUrls,
 	number: queryNumber,
 	num: queryNumber,
 	numbers: queryNumbers,
@@ -950,11 +995,13 @@ function setProxy(instance, options, url) {
 		instance.defaults.httpAgent = proxyAgent;
 		instance.defaults.httpsAgent = proxyAgent;
 
-		return;
+		return true;
 	}
 
 	instance.defaults.httpAgent = options.httpsAgent || new http.Agent({ ...options.agent });
 	instance.defaults.httpsAgent = options.httpsAgent || new https.Agent({ ...options.agent });
+
+	return false;
 }
 /* eslint-enable no-param-reassign */
 
@@ -967,16 +1014,6 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 
 	const { limiter, interval, concurrency } = getLimiter(url, options);
 
-	const feedbackBase = {
-		url,
-		method,
-		interval,
-		concurrency,
-		options,
-	};
-
-	events.emit('requestInit', feedbackBase);
-
 	const instance = axios.create({
 		data: body,
 		validateStatus: null,
@@ -987,7 +1024,18 @@ async function request(url, body, customOptions = {}, method = 'GET') {
 		// httpAgent: options.httpAgent || new http.Agent({ ...options.agent }),
 	});
 
-	setProxy(instance, options, url);
+	const isProxied = setProxy(instance, options, url);
+
+	const feedbackBase = {
+		url,
+		method,
+		interval,
+		concurrency,
+		isProxied,
+		options,
+	};
+
+	events.emit('requestInit', feedbackBase);
 
 	const res = await limiter.schedule(async () => instance.get(url));
 

package/tests/index.html

@@ -45,6 +45,7 @@
 		<!-- deliberate space in url( ) to test JSDOM's quirky handling of this -->
 		<div class="style" style="background-image: url( https://i.imgur.com/eDQmLys.jpg ); color: red;"></div>
 		<div class="background" style="background: url(https://i.imgur.com/xFHbuDL.jpeg) 0 0/contain no-repeat; color: green;"></div>
+		<div class="background" style="background: url('https://i.redd.it/e91oo4ueyeb71.jpg') 0 0/contain no-repeat; color: green;"></div>
 		<div class="style" style="margin: 1rem; color: blue;"></div>
 
 		<video id="video" poster="https://i.imgur.com/eDQmLys.jpg"><source src="https://i.imgur.com/eDQmLys.mp4"></video>

package/tests/init.js

@@ -28,7 +28,7 @@ async function initTest() {
 	unprint.on('requestInit', (initData) => console.log('init', initData));
 	unprint.on('requestError', (errorData) => console.error('error', errorData));
 	unprint.on('requestSuccess', (successData) => console.log('success', successData));
-	unprint.on('query', (queryData) => console.log('query', queryData));
+	// unprint.on('query', (queryData) => console.log('query', queryData));
 
 	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
 	// const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
@@ -60,6 +60,10 @@ async function initTest() {
 	console.log('style background', res.context.query.style('.background'));
 	console.log('styles', res.context.query.styles('.style'));
 	console.log('styles color', res.context.query.styles('.style', { styleAttribute: 'color' }));
+	console.log('style url', res.context.query.styleUrl('.background', 'background-image'));
+	console.log('style urls', res.context.query.styleUrls('.background', 'background-image'));
+	console.log('background', res.context.query.background('.background'));
+	console.log('backgrounds', res.context.query.backgrounds('.background'));
 	console.log('path', res.context.query.url('#path'));
 	console.log('relative path', res.context.query.url('#relativePath'));
 	console.log('exists', res.context.query.exists('#title'));