unprint 0.3.0 → 0.4.0

package/README.md

@@ -7,6 +7,15 @@ unprint is a web scraping utility built around JSDOM, providing convenience meth
 ## Usage
 `const unprint = require('unprint');`
 
+### Global options
+```
+unprint.options({
+	headers: {
+		'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
+	},
+})
+```
+
 ### Querying
 For optimal flexibility, unprint query methods can be used with or without initialization. If you already have access to DOM elements using another library or unprint instance, you can query it by using the uninitialized `query` methods provided directly from the library, and passing the element as the first argument, as such:
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unprint",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Simplify common web scraping tasks while staying in control of the data.",
   "main": "src/app.js",
   "scripts": {
@@ -24,6 +24,7 @@
   "dependencies": {
     "axios": "^0.27.2",
     "bottleneck": "^2.19.5",
+    "deepmerge": "^4.2.2",
     "eslint": "^8.17.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-base": "^15.0.0",

package/src/app.js

@@ -3,6 +3,7 @@
 const { JSDOM, VirtualConsole } = require('jsdom');
 const axios = require('axios').default;
 const moment = require('moment-timezone');
+const merge = require('deepmerge');
 
 const settings = {
 	throwErrors: false,
@@ -32,6 +33,12 @@ const defaultOptions = {
 	trim: true,
 };
 
+let globalOptions = {};
+
+function configure(newOptions) {
+	globalOptions = newOptions;
+}
+
 function trim(string) {
 	if (typeof string === 'string') {
 		return string.trim().replace(/\s+/g, ' ');
@@ -288,6 +295,30 @@ function queryImages(context, selector = 'img', customOptions) {
 	return imageUrls.map((imageUrl) => prefixUrl(imageUrl, options.origin, options));
 }
 
+function queryVideo(context, selector = 'source', customOptions) {
+	const options = {
+		...context.options,
+		attribute: 'src',
+		...customOptions,
+	};
+
+	const videoUrl = queryContent(context, selector, options);
+
+	return prefixUrl(videoUrl, options.origin, options);
+}
+
+function queryVideos(context, selector = 'source', customOptions) {
+	const options = {
+		...context.options,
+		attribute: 'src',
+		...customOptions,
+	};
+
+	const videoUrls = queryContents(context, selector, options);
+
+	return videoUrls.map((videoUrl) => prefixUrl(videoUrl, options.origin, options));
+}
+
 function extractJson(element) {
 	if (!element) {
 		return null;
@@ -385,6 +416,8 @@ const queryFns = {
 	date: queryDate,
 	dates: queryDates,
 	url: queryUrl,
+	video: queryVideo,
+	videos: queryVideos,
 };
 
 function isDomObject(element) {
@@ -485,12 +518,13 @@ function initAll(context, selector, options) {
 }
 
 async function request(url, body, customOptions = {}, method = 'GET') {
-	const options = {
+	const options = merge.all([{
 		timeout: 1000,
 		extract: true,
 		url,
-		...customOptions,
-	};
+	}, globalOptions, customOptions]);
+
+	console.log('options', options, globalOptions);
 
 	const res = await axios({
 		url,
@@ -551,6 +585,7 @@ async function post(url, body, options) {
 }
 
 module.exports = {
+	configure,
 	get,
 	post,
 	request,
@@ -559,5 +594,6 @@ module.exports = {
 	init,
 	initAll,
 	extractDate,
+	options: configure,
 	query: initQueryFns(queryFns),
 };

package/tests/init.js

@@ -10,6 +10,8 @@ const data = require('./data.json');
 const port = process.env.PORT || 3101;
 
 async function initTest() {
+	unprint.options({ headers: { 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36' } });
+
 	const res = await unprint.get(`http://127.0.0.1:${port}/html`, { select: 'body' });
 	// const jsonRes = await unprint.get(`http://127.0.0.1:${port}/json`);
 	// const errorRes = await unprint.get(`http://127.0.0.1:${port}/error/404`);