ph-api-scraper 3.4.0

package/src/helpers/utils_scrap.js

@@ -0,0 +1,65 @@
+const consts_global = require('./../constants/consts_global');
+const consts_page = require('./../constants/consts_page');
+const consts_scrap = require('./../constants/consts_scrap');
+const utils_global = require('./utils_global');
+const utils_sanitizer = require('./utils_sanitizer');
+
+module.exports = {
+	scraper_content_informations: (doc, keys, selectors, element_attributs) => {
+		const selectors_restricted = utils_global.selectors_restriction(keys, selectors);
+		return module.exports.scrap(doc, selectors_restricted, element_attributs);
+	},
+	is_data_available: (object, key) => {
+		return object.querySelector(key) !== null;
+	},
+	is_array_available: array => {
+		return array !== null && array.length > 0;
+	},
+	scrap_object_selected: (object, keys, key, attribut) => {
+		return module.exports.is_data_available(object, keys[key]) ? object.querySelector(keys[key])[attribut] : consts_global.NO_DATA;
+	},
+	scrap_inner_html: (object, keys, key) => {
+		return module.exports.scrap_object_selected(object, keys, key, 'innerHTML');
+	},
+	scrap_data_content: (object, keys, key) => {
+		return module.exports.is_data_available(object, keys[key]) ? object.querySelector(keys[key]).getAttribute('content') : consts_global.NO_DATA;
+	},
+	scrap_text_content: (object, keys, key) => {
+		return module.exports.scrap_object_selected(object, keys, key, 'textContent');
+	},
+	scrap_multi_text_content: (object, keys, key) => {
+		const elm = [...object.querySelectorAll(keys[key])];
+
+		return module.exports.is_array_available(elm) ? elm.map(node => node.textContent) : consts_global.NO_DATA;
+	},
+	scrap_javascript: (object, keys, key) => {
+		return JSON.parse(object.querySelector(keys[key]).textContent)[consts_page.javascript[key]];
+	},
+	scrap_null: (object, keys, key) => {
+		return module.exports.is_data_available(object, keys[key]);
+	},
+	scrap_default: (object, keys, key, attributs) => {
+		return object.querySelector(keys[key]).getAttribute(attributs[key]);
+	},
+	scrap: (object, keys, attributs) => {
+		return Object.fromEntries(Object.keys(keys).map(key => {
+			const scrap = consts_scrap.scrap_by_attribut_type[attributs[key]];
+
+			let scrap_data = null;
+			if (utils_global.is_parameter_missing(scrap)) {
+				scrap_data = module.exports.scrap_default(object, keys, key, attributs);
+			} else {
+				scrap_data = module.exports[scrap](object, keys, key);
+			}
+
+			return [key, scrap_data];
+		}));
+	},
+	scraper_array: (doc, global, selectors, attributs) => {
+		const elements = [...doc.querySelectorAll(global)];
+		return elements.map((element, index) => {
+			const temporary = module.exports.scrap(element, selectors, attributs);
+			return utils_sanitizer.sanitizer(temporary);
+		});
+	}
+};

package/src/index.js

@@ -0,0 +1,101 @@
+'use strict';
+
+const utils = require('./utils');
+const utils_sanitizer = require('./helpers/utils_sanitizer');
+const consts_global = require('./constants/consts_global');
+const page = require('./page');
+const page_search = require('./search');
+const page_model = require('./model');
+
+module.exports = {
+	/**
+	* Scrap the content of a video page of pornhub
+	*
+	* @params {string} url The url of the page you wanna scrap
+	* @params {array} key The array of key of value that you want to scrap
+	* @return {Object} The result of the scrap in an object containing only the key choosen
+	* @throws {Object} If an error happen
+	**/
+	page: async (url, key) => {
+		try {
+			const keys = utils.options_to_keys(key);
+			const source = await utils.url_to_source(url);
+			const datas = page.scraping_page(source, keys);
+			return utils_sanitizer.sanitizer(datas);
+		} catch (error) {
+			return utils.error_message(error);
+		}
+	},
+	/**
+	* Scrap the content of a model page of pornhub
+	*
+	* @params {string} name The exact name of the model
+	* @params {array} key The array of key of value that you want to scrap
+	* @params {string} type The type of the result (pornstar or model)
+	* @return {Object} The result of the scrap in an object containing only the key choosen
+	* @throws {Object} If an error happen
+	**/
+	model: async (name, key, type = consts_global.types.MODEL) => {
+		try {
+			const keys = utils.options_to_keys(key);
+			const url = utils.name_to_url(name, type);
+			const source = await utils.url_to_source(url);
+			const datas = page_model.scrap(source, keys);
+			const datas_filtered = page_model.filter_value(datas, keys);
+			return utils_sanitizer.sanitizer(datas_filtered);
+		} catch (error) {
+			return utils.error_message(error);
+		}
+	},
+	/**
+	* Scrap the content of the featured videos of pornhub
+	*
+	* @params {array} [key=null] The array of key of value that you want to scrap
+	* @params {object} [options=null] The options for display the video of a selected page
+	* @return {Object} The result of the scrap in an object containing only the key choosen
+	* @throws {Object} If an error happen
+	**/
+	video: async (key = null, options = null) => {
+		try {
+			const request_start_time = process.hrtime();
+			const usage_start = process.memoryUsage();
+
+			const keys = utils.options_to_keys(key);
+			if (!options || !options.page) {
+				options = options ? options : {};
+				options.page = 1;
+			}
+
+			const source = await utils.multi_url_to_source(options);
+			const datas = page_search.scraping_search(source, keys, options);
+			const datas_sanitize = utils_sanitizer.sanitizer(datas);
+			return {...datas_sanitize, ...utils.performance_calculation(request_start_time, usage_start)};
+		} catch (error) {
+			return utils.error_message(error);
+		}
+	},
+	/**
+	* Scrap the content of a search page of pornhub
+	*
+	* @params {string} search The exact terms of the search
+	* @params {array} key The array of key of value that you want to scrap
+	* @params {object} options The options for display the video of a selected page
+	* @return {object} The result of the scrap in an object containing only the key choosen
+	* @throws {object} If an error happen
+	**/
+	search: async (search, key, options) => {
+		try {
+			const keys = utils.options_to_keys(key);
+			if (!options || !options.page) {
+				options = options ? options : {};
+				options.page = 1;
+			}
+
+			const source = await utils.multi_url_to_source(options, search);
+			const datas = page_search.scraping_search(source, keys, options);
+			return utils_sanitizer.sanitizer(datas);
+		} catch (error) {
+			return utils.error_message(error);
+		}
+	}
+};

package/src/model.js

@@ -0,0 +1,44 @@
+const utils = require('./utils');
+const utils_sanitizer = require('./helpers/utils_sanitizer');
+const utils_scrap = require('./helpers/utils_scrap');
+const utils_global = require('./helpers/utils_global');
+const consts_global = require('./constants/consts_global');
+const consts_model = require('./constants/consts_model');
+
+module.exports = {
+	extract_informations_model: (doc, selector_elements) => {
+		const elements = [...doc.querySelectorAll(selector_elements)];
+
+		return Object.fromEntries(elements.map(element => {
+			const element_key = element.querySelector('span:nth-child(1)');
+			const element_key_text = utils_sanitizer.sanitizer_key(element_key.innerHTML);
+
+			const element_value = element.querySelector('span:nth-child(2)');
+			const element_value_text = element_value === null ? consts_global.NO_DATA : utils_sanitizer.sanitizer_string(element_value.innerHTML);
+
+			return [element_key_text, element_value_text];
+		}));
+	},
+	scrap_unfixed_content_informations: (doc, keys, selectors, attributs) => {
+		const informations = module.exports.extract_informations_model(doc, consts_model.PROFIL_INFOS_LIST);
+		const informations_restricted = utils_global.selectors_restriction(keys, informations);
+		return informations_restricted;
+	},
+	scrap: (source, keys) => {
+		const doc = utils.source_to_dom(source);
+
+		let datas = {};
+		datas = {...datas, ...utils_scrap.scraper_content_informations(doc, keys, consts_model.model_selectors, consts_model.model_element_attributs)};
+		datas = {...datas, ...module.exports.scrap_unfixed_content_informations(doc, keys, consts_model.model_selectors, consts_model.model_element_attributs)};
+
+		return datas;
+	},
+	filter_value: (datas, keys) => {
+		if (datas.VIDEO_NUMBER) {
+			const match = datas.VIDEO_NUMBER.match(/(?<=of )\d+/gi, '');
+			datas.VIDEO_NUMBER = match[0];
+		}
+
+		return datas;
+	}
+};

package/src/packages/postinstall.js

@@ -0,0 +1 @@
+console.log('Thanks for using my API! If you like it, please visit my github ;)');

package/src/page.js

@@ -0,0 +1,66 @@
+const utils = require('./utils');
+const utils_scrap = require('./helpers/utils_scrap');
+const consts_global = require('./constants/consts_global');
+const consts_page = require('./constants/consts_page');
+
+const scraper_comments_options = {
+	key: consts_global.keys.COMMENTS,
+	list: consts_page.COMMENTS_LIST,
+	selectors: consts_page.comment_selectors,
+	attributs: consts_page.page_element_attributs
+};
+
+const scraper_related_videos_options = {
+	key: consts_global.keys.RELATED_VIDEOS,
+	list: consts_page.RELATED_VIDEOS_LIST,
+	selectors: consts_page.related_videos_selectors,
+	attributs: consts_page.page_related_videos_element_attributs
+};
+
+module.exports = {
+	scraper_video_informations: (source, keys) => {
+		let rsl = {};
+
+		if (keys.includes(consts_global.keys.DOWNLOAD_URLS)) {
+			const matches = source.match(/(?<=\*\/)\w+/g);
+			const urls = [];
+			for (const match of matches) {
+				const regex = new RegExp('(?<=' + match + '=")[^;]+(?=")', 'g');
+				const value = source.match(regex)[0].replace(/[" +]/g, '');
+
+				if (value.startsWith('https')) {
+					if (urls.length === 4) {
+						break;
+					}
+
+					urls.push(value);
+				} else {
+					urls[urls.length - 1] += value;
+				}
+			}
+
+			rsl = urls.map(x => {
+				// Sometime PH does not provide a resolution, meaning the link is broken
+				const resolution = x.match(/(?<=_|\/)\d*P(?=_)/g);
+				return resolution !== null && resolution.length > 0 ? [x.match(/(?<=_|\/)\d*P(?=_)/g)[0], x] : null;
+			}).filter(x => x !== null);
+			rsl = Object.fromEntries(rsl);
+		}
+
+		return Object.keys(rsl).length > 0 ? rsl : null;
+	},
+	scraper_block_informations: (doc, keys, scrap_options) => {
+		return keys.includes(scrap_options.key) ? {[scrap_options.key]: utils_scrap.scraper_array(doc, scrap_options.list, scrap_options.selectors, scrap_options.attributs)} : {};
+	},
+	scraping_page: (source, keys) => {
+		const doc = utils.source_to_dom(source);
+		let datas = {};
+
+		datas = {...datas, ...utils_scrap.scraper_content_informations(doc, keys, consts_page.page_selectors, consts_page.page_element_attributs)};
+		datas = {...datas, DOWNLOAD_URLS: module.exports.scraper_video_informations(source, keys)};
+		datas = {...datas, ...module.exports.scraper_block_informations(doc, keys, scraper_comments_options)};
+		datas = {...datas, ...module.exports.scraper_block_informations(doc, keys, scraper_related_videos_options)};
+
+		return datas;
+	}
+};

package/src/search.js

@@ -0,0 +1,31 @@
+const utils = require('./utils');
+const utils_scrap = require('./helpers/utils_scrap');
+const consts_global = require('./constants/consts_global');
+const consts_search = require('./constants/consts_search');
+const consts_search_gifs = require('./constants/consts_search_gifs');
+const consts_search_pornstars = require('./constants/consts_search_pornstars');
+const consts_search_videos = require('./constants/consts_search_videos');
+const jsdom = require('jsdom');
+const {JSDOM} = jsdom;
+
+module.exports = {
+	scraping_search: (source, keys, options) => {
+		const doc = utils.source_to_dom(source);
+
+		let datas = {};
+		datas = {...datas, ...utils_scrap.scraper_content_informations(doc, keys, consts_search.primary_search_selectors, consts_search.page_search_element_attributs)};
+		if (!options.search || options.search === 'video') {
+			datas[consts_global.keys.RESULTS] = utils_scrap.scraper_array(doc, consts_search_videos.VIDEOS_LIST, consts_search_videos.videos_search_selectors, consts_search_videos.videos_element_attributs);
+		}
+
+		if (options.search === 'pornstars') {
+			datas[consts_global.keys.RESULTS] = utils_scrap.scraper_array(doc, consts_search_pornstars.PORNSTARS_LIST, consts_search_pornstars.pornstars_search_selectors, consts_search_pornstars.pornstars_element_attributs);
+		}
+
+		if (options.search === 'gifs') {
+			datas[consts_global.keys.RESULTS] = utils_scrap.scraper_array(doc, consts_search_gifs.GIFS_LIST, consts_search_gifs.gifs_search_selectors, consts_search_gifs.gifs_element_attributs);
+		}
+
+		return datas;
+	}
+};

package/src/utils.js

@@ -0,0 +1,197 @@
+const consts_global = require('./constants/consts_global');
+const consts_page = require('./constants/consts_page');
+const consts_queries = require('./constants/consts_queries');
+const utils_sanitizer = require('./helpers/utils_sanitizer');
+const Entities = require('html-entities').AllHtmlEntities;
+const entities = new Entities();
+const jsdom = require('jsdom');
+const {JSDOM} = jsdom;
+const got = require('got');
+const promise = require('promise');
+
+module.exports = {
+	/**
+	* Check if the parameter is defined and exist
+	*
+	* @params {string} parameter The parameter we want to test
+	* @return {boolean} True if the parameter exist or else false
+	**/
+	is_parameter_missing: parameter => {
+		return parameter === null || parameter === '' || parameter === undefined;
+	},
+	/**
+	* Read the keys passed and sanitize them in uppercase.
+	* If there is no key passed, we define a simple key `title`
+	* If the key passed is a string instead of an array, we put that key inside an array
+	*
+	* @params {(string|array)} key The single or multiples key passed with the call
+	* @return {array} The key in uppercase in an array
+	* @throws {Error} If the array passed in the call is empty
+	**/
+	options_to_keys: key => {
+		if (module.exports.is_parameter_missing(key)) {
+			return ['TITLE'];
+		}
+
+		const array_keys = Array.isArray(key) ? key : [key];
+		const array_keys_uppercase = array_keys.map(x => x.toUpperCase());
+
+		if (array_keys_uppercase.length === 0) {
+			throw new Error('A key need to be used with this call');
+		}
+
+		return array_keys_uppercase;
+	},
+	/**
+	* Transform the value of a filter from the package to his equivalent for pornhub
+	* By example is really not easy to remember that `mv` is the value for pornhub for `MOST_VIEWED`
+	* So the user of the api can just enter MOST_VIEWED and I will search for the pornhub value
+	*
+	* @params {string} value The query parameter that he wan to translate
+	* @return {string|null} The value translated if it exists or else null
+	**/
+	transform_filter: value => {
+		return module.exports.is_parameter_missing(value) ? null : consts_queries.filter[value];
+	},
+	/**
+	* Search in the array `values_allowed` if the `value` exist
+	* This function is used for checking the value available for a query parameter
+	*
+	* @params {string} value The value of the parameter to check
+	* @params {array} values_allowed The array that we will check into if the value is allowed
+	* @return {boolean} True if the value is allowed or else false
+	**/
+	is_value_allowed: (value, values_allowed) => {
+		return values_allowed.includes(value);
+	},
+	/**
+	* Create the url query for the parameter and value passed in argument
+	*
+	* @params {string} parameter The parameter that we want to config
+	* @params {string} value The value we want to give to the parameter
+	* @params {array} [values_allowed=null] The allowed value for this parameter
+	* @return {string} An url params with the form : & + parameter + = + value
+	**/
+	create_query: (parameter, value, values_allowed = null) => {
+		if (module.exports.is_parameter_missing(value)) {
+			return '';
+		}
+
+		if (values_allowed && !module.exports.is_value_allowed(value, values_allowed)) {
+			return '';
+		}
+
+		return '&' + parameter + '=' + value;
+	},
+	/**
+	* Create the url query of all the options
+	*
+	* @params {Object} options The object of the params and value
+	* @params {number} page_index The page of the search
+	* @params {string} [search=null] The term of the search
+	* @return {string} The url with all the params ready to be append to the url
+	**/
+	create_queries: (options, page_index, search = null) => {
+		let q = '';
+		q += module.exports.create_query('search', search);
+		q += module.exports.create_query('p', options.production, consts_queries.production);
+		q += module.exports.create_query('max_duration', options.max_duration, consts_queries.max_duration);
+		q += module.exports.create_query('min_duration', options.min_duration, consts_queries.min_duration);
+		q += module.exports.create_query('promo', options.promo, consts_queries.promo);
+		q += module.exports.create_query('o', module.exports.transform_filter(options.filter), Object.values(consts_queries.filter));
+
+		q += '&page=' + (page_index + 1);
+		q = q.replace('&', '?');
+		return q;
+	},
+	create_section_type: options => {
+		return options.search ? options.search : 'video';
+	},
+	create_link: (options, page_index, search = null) => {
+		let link = consts_global.links.BASE_URL + '/';
+		link += module.exports.create_section_type(options);
+		link += module.exports.is_parameter_missing(search) ? '' : '/' + consts_global.links.SEARCH;
+		link += module.exports.create_queries(options, page_index, search);
+		return link;
+	},
+	url_to_source: async url => {
+		url = module.exports.http_to_https(url);
+		const safe_url = url.toLowerCase();
+		const response = await got(safe_url);
+		return response.body;
+	},
+	/**
+	* Change the protocol http to https in an url
+	*
+	* @params {string} url The url that we want to replace the protocol
+	* @return {string} The url with https protocol
+	**/
+	http_to_https: url => {
+		return url.replace(/^http:/gi, 'https:');
+	},
+	multi_url_to_source: async (options, search = null) => {
+		return promise.all([...new Array(options.page)].map(async (page, page_index) => {
+			return module.exports.url_to_source(module.exports.create_link(options, page_index, search));
+		}));
+	},
+	name_to_url: (name, type) => {
+		if (module.exports.is_parameter_missing(name)) {
+			return null;
+		}
+
+		const slug = name.replace(/\s/gi, '-').toLowerCase();
+		const type_slug = type === consts_global.types.MODEL ? consts_global.types.MODEL : consts_global.types.PORNSTAR;
+		return consts_global.links.BASE_URL + '/' + type_slug + '/' + slug;
+	},
+	source_to_dom: source => {
+		const dom = new JSDOM(source);
+		return dom.window.document;
+	},
+	convert_to_second: time => {
+		if (module.exports.is_parameter_missing(time)) {
+			return consts_global.NO_DATA;
+		}
+
+		const time_splitted = time.split(':');
+		switch (time_splitted.length) {
+			case 3:
+				return ((+Number(time_splitted[0])) * 60 * 60) + ((+Number(time_splitted[1])) * 60) + (+Number(time_splitted[2]));
+			case 2:
+				return ((+Number(time_splitted[0])) * 60) + (+Number(time_splitted[1]));
+			default:
+				return Number(time);
+		}
+	},
+	/**
+	* Give additionnal informations about the performance of the request
+	*
+	* @params {Object} request_start_time The process.hrtime of the beginning of the request
+	* @params {Object} usage_start The process.memoryUsage at the beginning of the request
+	* @return {Object} The result of the performance of the request
+	**/
+	performance_calculation: (request_start_time, usage_start) => {
+		const request_duration = process.hrtime(request_start_time);
+		const request_performance_time = request_duration[0] + '.' + request_duration[1] + ' seconds';
+		const usage_end = process.memoryUsage();
+		const request_rss = usage_end.rss - usage_start.rss;
+		const request_heap_total = usage_end.heapTotal - usage_start.heapTotal;
+		const request_heap_used = usage_end.heapUsed - usage_start.heapUsed;
+		const performance = {
+			request_duration: request_performance_time,
+			request_diff_rss: request_rss,
+			request_diff_heap_total: request_heap_total,
+			request_diff_heap_used: request_heap_used
+		};
+		return {performance};
+	},
+	/**
+	* Handle the error message of the api
+	*
+	* @params {Object} error The error javascript object
+	* @return {Object} The object that notice there is an error
+	**/
+	error_message: error => {
+		console.log(error);
+		return {error: consts_global.errors.DEFAULT};
+	}
+};