wikiploy 1.0.0

package/.eslintrc.cjs

@@ -0,0 +1,41 @@
+module.exports = {
+	"env": {
+		"node": true,
+		"browser": true,
+		"es2021": true,
+	},
+	"globals": {
+		"$": true,
+		"mw": true,
+	},
+	"extends": "eslint:recommended",
+	"overrides": [{
+		"env": {
+			"node": true,
+			"browser": true,
+		},
+		"files": [
+			".eslintrc.{js,cjs}"
+		],
+		"parserOptions": {
+			"sourceType": "script"
+		}
+	}],
+	"parserOptions": {
+		"ecmaVersion": "latest",
+		"sourceType": "module"
+	},
+	"rules": {
+		"no-prototype-builtins": "off",
+		"indent": [
+			"error",
+			"tab",
+			{
+				"SwitchCase": 1,
+			},
+		],
+		//"array-bracket-newline": ["error", { "multiline": true, "minItems": 3 }],
+		//"array-element-newline": ["error", { "multiline": true }]
+		"array-element-newline": ["error", "consistent"]
+	}
+}

package/.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+	// Use IntelliSense to learn about possible attributes.
+	// Hover to view descriptions of existing attributes.
+	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+	"version": "0.2.0",
+	"configurations": [
+		{
+			"type": "node",
+			"request": "launch",
+			"name": "Run current JS",
+			"skipFiles": [
+				"<node_internals>/**"
+			],
+			"program": "${workspaceFolder}\\${file}"
+		},
+		{
+			"type": "node",
+			"request": "launch",
+			"name": "Deploy test.js",
+			"skipFiles": [
+				"<node_internals>/**"
+			],
+			"program": "${workspaceFolder}\\Wiki_bot_test.js"
+		}
+	]
+}

package/DeployConfig.js

@@ -0,0 +1,21 @@
+/**
+ * Deployment metadata.
+ */
+export default class DeployConfig {
+	/**
+	 * @param {DeployConfig} page Edit page (tab).
+	 */
+	constructor(options) {
+		this.src = options?.src;
+		this.dst = options?.dst;
+		if (!this.dst) {
+			this.dst = `~/${this.src}`;
+		}
+	}
+
+	/** info */
+	info() {
+		return `deploy "${this.src}" to "${this.dst}"`;
+	}
+
+}

package/PageCache.js

@@ -0,0 +1,134 @@
+
+/**
+ * Aggressive resource caching.
+ * 
+ * Multiple pages should be able to share this cache.
+ */
+export default class PageCache {
+	constructor() {
+		this.cache = {};
+		this.stats = {
+			fromCache: 0,
+			direct: 0,
+			saved: 0,
+		};
+		/** Fake max age [s] */
+		this.maxAge = 10 * 3600;
+	}
+
+	/** Usage info. */
+	info() {
+		console.log('[PageCache]'
+			, 'stats: ' + JSON.stringify(this.stats)
+			, 'urls: ' + Object.keys(this.cache).length
+		);
+	}
+
+	/**
+	 * Prepare page (tab) for caching.
+	 * 
+	 * Note! This should be done before opening a URL (`page.goto`).
+	 * 
+	 * @param {Page} page 
+	 */
+	async enable(page) {
+		const cache = this.cache;
+
+		// enable requests hijacking (~PWA)
+		await page.setRequestInterception(true);
+		
+		// serve from cache
+		page.on('request', async (request) => {
+			const url = request.url();
+			try {
+				if (url in cache) {
+					const cached = cache[url];
+					const method = request.method();
+					// if (cached.expires > Date.now()) {
+					if (method === 'GET') {
+						this.stats.fromCache++;
+						await request.respond(cached);
+						return;
+					}
+					else {
+						console.warn('[PageCache]', 'skipped', JSON.stringify({method, url}));
+					}
+				}
+			} catch (error) {
+				console.warn('[PageCache]', 'serving cache failed', url, '\n', error);
+			}
+			this.stats.direct++;
+			request.continue();
+		});
+
+		// save to cache
+		page.on('response', async (response) => {
+			const headers = response.headers();
+			const url = response.url();
+			let cacheit = this.shouldCache(url, headers);
+			if (cacheit) {
+				if (url in cache) {
+					return;
+				}
+
+				let buffer;
+				try {
+					buffer = await response.buffer();
+				} catch (error) {
+					return;
+				}
+
+				// cleanup headers
+				delete headers['nel'];
+				delete headers['report-to'];
+				delete headers['age'];
+				delete headers['cache-control'];
+				delete headers['set-cookie'];	// this brakes puppeteer
+				delete headers['server-timing'];
+				delete headers['x-cache'];
+				delete headers['x-cache-status'];
+
+				// save
+				this.stats.saved++;
+				cache[url] = {
+					status: response.status(),
+					headers,
+					body: buffer,
+					expires: Date.now() + (this.maxAge * 1000),
+				};
+			}
+		});
+	}
+
+	/** 
+	 * Check if we should cache the response.
+	 */
+	shouldCache(url, headers) {
+		const contentType = headers['content-type'] || '';
+		if (contentType.startsWith('text/html')) {
+			return false;
+		}
+		if (contentType.search(/^text\/(javascript|css)/i) >= 0) {
+			return true;
+		}
+
+		const cacheControl = headers['cache-control'] || '';
+		if (cacheControl.search(/max-age=[1-9]/) >= 0) {
+			return true;
+		}
+
+		// user scripts
+		if (url.search(/&ctype=text\/(javascript|css)/) >= 0) {
+			return true;
+		}
+		// MediaWiki modules
+		if (url.search(/load.+&modules/) >= 0) {
+			return true;
+		}
+
+		// console.log(JSON.stringify(headers, null, '\t'));
+		// console.log(url);
+
+		return null;
+	}
+}

package/README.md

@@ -0,0 +1,7 @@
+Wikiploy
+==========================
+
+User scripts and gadgets deployment for wikis (Wikipedia or more generally MediaWiki based wiki).
+Rollout your JS, CSS etc from your git repository to as many MW wikis as you need.
+
+This is using [Puppeteer](https://pptr.dev/) to control [Chrome Canary](https://www.google.com/chrome/canary/). You just open Chrome and run a script. The idea is that you are logged in in Chrome and so all edits are still your edits. You can keep the Canary running in the background when you are changing and deploying more stuff.

package/WikiOps.js

@@ -0,0 +1,139 @@
+import {
+	scrollIntoViewIfNeeded,
+	waitForSelector,
+	waitForSelectors,
+} from './chromeBase.js'
+
+import PageCache from './PageCache.js';
+
+/**
+ * Helper class for edits and navigation.
+ * 
+ * Note! Due to Chrome's large memory consupmtion on new tabs it is best to re-use tabs.
+ * So it is best to open tab before loop:
+		// open new tab
+		const page = await wikiBot.openTab(browser);
+		// some loop
+		for () {
+			await page.goto(url);
+			...do stuff...
+		}
+		page.close();
+ */
+export default class WikiOps {
+	/**
+	 * Init.
+	 * @param {PageCache} globalCache Cache for page resources.
+	 */
+	constructor(globalCache) {
+		this.cache = globalCache ? globalCache : new PageCache();
+	}
+
+	/**
+	 * Open new tab.
+	 * @returns {Browser} browser.
+	 */
+	async openTab(browser) {
+		const page = await browser.newPage();
+		await this.initViewport(page);
+		await this.cache.enable(page);
+		await this.disarmUnloadWarning(page);
+		const timeout = 5000;
+		page.setDefaultTimeout(timeout);
+		return page;
+	}
+
+	/**
+	 * Avoid leave-page warning.
+	 * @param {Page} page
+	 * @private
+	 */
+	async disarmUnloadWarning(page) {
+		// force to accept the warning
+		page.on("dialog", (dialog) => {
+			if (dialog.type() === "beforeunload") {
+				dialog.accept();
+			}
+		});
+		await page.evaluate(() => {
+			window.onbeforeunload = null;
+			window.addEventListener("load", () => {
+				$(window).off('beforeunload');
+			});
+		});
+	}
+
+	/**
+	 * Init view.
+	 * @param {Page} targetPage 
+	 * @private
+	 */
+	async initViewport(targetPage) {
+		await targetPage.setViewport({
+			width: 1200,
+			height: 900
+		})
+	}
+
+	/** Save. */
+	async saveEdit(targetPage) {
+		const timeout = 200;
+
+		await scrollIntoViewIfNeeded([
+			'#wpSave'
+		], targetPage, timeout);
+		const element = await waitForSelectors([
+			'#wpSave'
+		], targetPage, {
+			timeout,
+			visible: true
+		});
+		let nav = targetPage.waitForNavigation(); // init wait
+		await element.click({
+			offset: {
+				x: 10,
+				y: 4,
+			},
+		});
+		await nav; // wait for form submit
+	}
+
+	/** Go to url (and wait for it). */
+	async goto(page, url) {
+		let nav = page.waitForNavigation(); // init wait
+		await page.goto(url);
+		await nav; // wait for url
+		await this.disarmUnloadWarning(page);
+	}
+
+	/** Change intput's value. */
+	async fillEdit(page, value) {
+		const timeout = 5000;
+		await waitForSelector('#editform', page, {
+			timeout,
+		});
+		// await changeElementValue(element, value);
+		await page.evaluate((value) => {
+			// remove editors (plain, WYSIWYG)
+			document.querySelectorAll('#editform textarea, .ace_editor').forEach(el=>el.remove());
+			// add plain textarea
+			document.querySelector('#editform').insertAdjacentHTML('afterbegin', `
+				<textarea id="wpTextbox1" name="wpTextbox1"
+				></textarea>
+			`);
+			// insert value
+			let input = document.querySelector('#wpTextbox1');
+			input.value = value;
+		}, value);
+	}
+
+	/** Insert summary. */
+	async fillSummary(page, summary) {
+		await page.evaluate((summary) => {
+			let wpSummary = document.querySelector('#wpSummary');
+			wpSummary.value = summary;
+		}, summary);
+	}
+
+	
+}

package/Wiki_bot_test.js

@@ -0,0 +1,23 @@
+import DeployConfig from './DeployConfig.js';
+import Wikiploy from './Wikiploy.js';
+
+const ployBot = new Wikiploy();
+// mock
+// ployBot.mock = true;
+// ployBot.mockSleep = 5_000;
+
+(async () => {
+	const configs = [];
+	configs.push(new DeployConfig({
+		src: 'test.js',
+		dst: 'User:Nux/test-jsbot--test.js',
+	}));
+	configs.push(new DeployConfig({
+		src: 'test.css',
+		dst: 'User:Nux/test-jsbot--test.css',
+	}));
+	await ployBot.deploy(configs);
+})().catch(err => {
+	console.error(err);
+	process.exit(1);
+});

package/Wikiploy.js

@@ -0,0 +1,133 @@
+import puppeteer, { Browser } from 'puppeteer'; // v13+
+
+import WikiOps from './WikiOps.js';
+import PageCache from './PageCache.js';
+
+import { promises as fs } from "fs";	// node v11+
+
+import {
+	wsBrowserPort,
+} from './chrome.config.js'
+
+// eslint-disable-next-line no-unused-vars
+function sleep(sleepMs) {
+	return new Promise((resolve)=>{setTimeout(()=>resolve(), sleepMs)});
+}
+
+/**
+ * MediaWiki deployment automation.
+ * 
+ * @property _browser {Browser} The user's email
+ */
+export default class Wikiploy {
+	constructor() {
+		this.cache = new PageCache();
+		/** Disable save. */
+		this.mock = false;
+		/** Wait before close [ms] (or you can set a breakpoint to check stuff). */
+		this.mockSleep = 0;
+
+		/** Browser connection. */
+		this._browser = false;
+
+		/** Bot helper. */
+		this._bot = new WikiOps(this.cache);
+	}
+
+	/**
+	 * Deploy scripts.
+	 * @param {DeployConfig[]} configs 
+	 */
+	async deploy(configs) {
+		const bot = this._bot;
+		const browser = await this.init();
+		const page = await bot.openTab(browser);
+		console.log(JSON.stringify(configs));
+		// main loop
+		for (const config of configs) {
+			await this.save(config, page);
+		}
+		page.close();
+		console.log(`done`);
+		process.exit(0);
+	}
+
+	/**
+	 * Deploy script.
+	 * @param {DeployConfig} config Config.
+	 * @param {Page} page
+	 */
+	async save(config, page) {
+		console.log('[Wikiploy]', config.info());
+		const bot = this._bot;
+		// navigate
+		let url = this.editUrl(config.dst);
+		await bot.goto(page, url);
+		// insert the content of the file into the edit field
+		const contents = await fs.readFile(config.src, 'utf8');
+		await bot.fillEdit(page, contents);
+		// edit description
+		const summary = this.preapreSummary(config);
+		await bot.fillSummary(page, summary);
+
+		// save
+		if (!this.mock) {
+			await bot.saveEdit(page);
+			console.log('saved');
+		} else {
+			await sleep(this.mockSleep);
+		}
+	}
+
+
+	/**
+	 * Prepare edit summary.
+	 * 
+	 * @param {String} pageTitle Title with namespace.
+	 * @returns {String} Full edit URL.
+	 * @private
+	 */
+	preapreSummary(config) {
+		return '[Wikiploy]' + ` ${config.src}`;
+	}
+
+	/**
+	 * Prepare edit URL.
+	 * 
+	 * @param {String} pageTitle Title with namespace.
+	 * @returns {String} Full edit URL.
+	 * @private
+	 */
+	editUrl(pageTitle) {
+		const baseUrl = `https://pl.wikipedia.org/w/index.php`;	// TODO: config/options
+
+		// common params
+		// note that submit action is not affected by new wikicode editor
+		let params = `
+			&useskin=monobook
+			&action=submit
+		`.replace(/\s+/g, '');
+
+		return baseUrl + '?title=' + encodeURIComponent(pageTitle) + params;
+	}
+	
+	/**
+	 * Init browser connection.
+	 * 
+	 * @returns {Browser} the browser API.
+	 */
+	async init() {
+		if (this._browser instanceof Browser) {
+			return this._browser;
+		}
+
+		// connect to current (open) Chrome window
+		const browserURL = `http://127.0.0.1:${wsBrowserPort}`;
+		const browser = await puppeteer.connect({
+			// browserWSEndpoint: wsUrl,
+			browserURL,
+		});
+
+		return browser;
+	}
+}

package/chrome.config.js

@@ -0,0 +1,10 @@
+/**
+ * Chrome configuration.
+ * 
+ * You should have a Canary edition of Chrome. It seems easiest to setup doesn't involve your main Chrome.
+ * 
+ * Then just run Chrome Canary with param: "...\chrome.exe" --remote-debugging-port=9222
+ */
+
+/** Remote debugging port. */
+export const wsBrowserPort = '9222';

package/chromeBase.js

@@ -0,0 +1,252 @@
+/* eslint-disable no-undef */
+/* eslint-disable indent */
+
+/**
+ * Functions provided by Chrome Recorder.
+ */
+
+export async function waitForSelectors(selectors, frame, options) {
+	for (const selector of selectors) {
+		try {
+			return await waitForSelector(selector, frame, options);
+		} catch (err) {
+			console.error(err);
+		}
+	}
+	throw new Error('Could not find element for selectors: ' + JSON.stringify(selectors));
+}
+
+export async function scrollIntoViewIfNeeded(selectors, frame, timeout) {
+	const element = await waitForSelectors(selectors, frame, {
+		visible: false,
+		timeout
+	});
+	if (!element) {
+		throw new Error(
+			'The element could not be found.'
+		);
+	}
+	await waitForConnected(element, timeout);
+	const isInViewport = await element.isIntersectingViewport({
+		threshold: 0
+	});
+	if (isInViewport) {
+		return;
+	}
+	await element.evaluate(element => {
+		element.scrollIntoView({
+			block: 'center',
+			inline: 'center',
+			behavior: 'auto',
+		});
+	});
+	await waitForInViewport(element, timeout);
+}
+
+export async function waitForConnected(element, timeout) {
+	await waitForFunction(async () => {
+		return await element.getProperty('isConnected');
+	}, timeout);
+}
+
+export async function waitForInViewport(element, timeout) {
+	await waitForFunction(async () => {
+		return await element.isIntersectingViewport({
+			threshold: 0
+		});
+	}, timeout);
+}
+
+export async function waitForSelector(selector, frame, options) {
+	if (!Array.isArray(selector)) {
+		selector = [selector];
+	}
+	if (!selector.length) {
+		throw new Error('Empty selector provided to waitForSelector');
+	}
+	let element = null;
+	for (let i = 0; i < selector.length; i++) {
+		const part = selector[i];
+		if (element) {
+			element = await element.waitForSelector(part, options);
+		} else {
+			element = await frame.waitForSelector(part, options);
+		}
+		if (!element) {
+			throw new Error('Could not find element: ' + selector.join('>>'));
+		}
+		if (i < selector.length - 1) {
+			element = (await element.evaluateHandle(el => el.shadowRoot ? el.shadowRoot : el)).asElement();
+		}
+	}
+	if (!element) {
+		throw new Error('Could not find element: ' + selector.join('|'));
+	}
+	return element;
+}
+
+export async function waitForElement(step, frame, timeout) {
+	const {
+		count = 1,
+			operator = '>=',
+			visible = true,
+			properties,
+			attributes,
+	} = step;
+	const compFn = {
+		'==': (a, b) => a === b,
+		'>=': (a, b) => a >= b,
+		'<=': (a, b) => a <= b,
+	} [operator];
+	await waitForFunction(async () => {
+		const elements = await querySelectorsAll(step.selectors, frame);
+		let result = compFn(elements.length, count);
+		const elementsHandle = await frame.evaluateHandle((...elements) => {
+			return elements;
+		}, ...elements);
+		await Promise.all(elements.map((element) => element.dispose()));
+		if (result && (properties || attributes)) {
+			result = await elementsHandle.evaluate(
+				(elements, properties, attributes) => {
+					for (const element of elements) {
+						if (attributes) {
+							for (const [name, value] of Object.entries(attributes)) {
+								if (element.getAttribute(name) !== value) {
+									return false;
+								}
+							}
+						}
+						if (properties) {
+							if (!isDeepMatch(properties, element)) {
+								return false;
+							}
+						}
+					}
+					return true;
+
+					function isDeepMatch(a, b) {
+						if (a === b) {
+							return true;
+						}
+						if ((a && !b) || (!a && b)) {
+							return false;
+						}
+						if (!(a instanceof Object) || !(b instanceof Object)) {
+							return false;
+						}
+						for (const [key, value] of Object.entries(a)) {
+							if (!isDeepMatch(value, b[key])) {
+								return false;
+							}
+						}
+						return true;
+					}
+				},
+				properties,
+				attributes
+			);
+		}
+		await elementsHandle.dispose();
+		return result === visible;
+	}, timeout);
+}
+
+export async function querySelectorsAll(selectors, frame) {
+	for (const selector of selectors) {
+		const result = await querySelectorAll(selector, frame);
+		if (result.length) {
+			return result;
+		}
+	}
+	return [];
+}
+
+export async function querySelectorAll(selector, frame) {
+	if (!Array.isArray(selector)) {
+		selector = [selector];
+	}
+	if (!selector.length) {
+		throw new Error('Empty selector provided to querySelectorAll');
+	}
+	let elements = [];
+	for (let i = 0; i < selector.length; i++) {
+		const part = selector[i];
+		if (i === 0) {
+			elements = await frame.$$(part);
+		} else {
+			const tmpElements = elements;
+			elements = [];
+			for (const el of tmpElements) {
+				elements.push(...(await el.$$(part)));
+			}
+		}
+		if (elements.length === 0) {
+			return [];
+		}
+		if (i < selector.length - 1) {
+			const tmpElements = [];
+			for (const el of elements) {
+				const newEl = (await el.evaluateHandle(el => el.shadowRoot ? el.shadowRoot : el)).asElement();
+				if (newEl) {
+					tmpElements.push(newEl);
+				}
+			}
+			elements = tmpElements;
+		}
+	}
+	return elements;
+}
+
+export async function waitForFunction(fn, timeout) {
+	let isActive = true;
+	const timeoutId = setTimeout(() => {
+		isActive = false;
+	}, timeout);
+	while (isActive) {
+		const result = await fn();
+		if (result) {
+			clearTimeout(timeoutId);
+			return;
+		}
+		await new Promise(resolve => setTimeout(resolve, 100));
+	}
+	throw new Error('Timed out');
+}
+
+export async function changeSelectElement(element, value) {
+	await element.select(value);
+	await element.evaluateHandle((e) => {
+		e.blur();
+		e.focus();
+	});
+}
+
+export async function changeElementValue(element, value) {
+	await element.focus();
+	await element.evaluate((input, value) => {
+		input.value = value;
+		input.dispatchEvent(new Event('input', {
+			bubbles: true
+		}));
+		input.dispatchEvent(new Event('change', {
+			bubbles: true
+		}));
+	}, value);
+}
+
+export async function typeIntoElement(element, value) {
+	const textToType = await element.evaluate((input, newValue) => {
+		if (
+			newValue.length <= input.value.length ||
+			!newValue.startsWith(input.value)
+		) {
+			input.value = '';
+			return newValue;
+		}
+		const originalValue = input.value;
+		input.value = '';
+		input.value = originalValue;
+		return newValue.substring(originalValue.length);
+	}, value);
+	await element.type(textToType);
+}

package/package.json

@@ -0,0 +1,35 @@
+{
+	"name": "wikiploy",
+	"version": "1.0.0",
+	"description": "User scripts and gadgets deployment for MediaWiki (Wikipedia).",
+	"main": "Wiki_bot_test.js",
+	"type": "module",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/Eccenux/Wikiploy.git"
+	},
+	"keywords": [
+		"Mediawiki",
+		"Wikipedia",
+		"deploy",
+		"rollout",
+		"gadgets"
+	],
+	"author": "Maciej Nux Jaros",
+	"license": "MIT",
+	"bugs": {
+		"url": "https://github.com/Eccenux/Wikiploy/issues"
+	},
+	"homepage": "https://github.com/Eccenux/Wikiploy#readme",
+	"dependencies": {
+		"puppeteer": "^20.7.2"
+	},
+	"devDependencies": {
+		"chai": "^4.3.7",
+		"mocha": "^10.2.0",		
+		"eslint": "^8.43.0"
+	}
+}

package/test/DeployConfig.test.js

@@ -0,0 +1,31 @@
+/* global describe, it */
+import { assert } from 'chai';
+import DeployConfig from '../DeployConfig.js';
+
+describe('DeployConfig', function () {
+	
+	describe('init', function () {
+		it('should set src, dst', function () {
+			let config = {
+				src: 'test.src.js',
+				dst: 'User:Nux/test.dst.js',
+			};
+			let result = new DeployConfig(config);
+			console.log(result);
+			assert.equal(result.src, config.src);
+			assert.equal(result.dst, config.dst);
+			// console.log(new DeployConfig({
+			// 	src: 'test.js',
+			// }));
+		});
+		it('should default to home', function () {
+			let config = {
+				src: 'test.js',
+			};
+			let expected = '~/test.js';
+			let result = new DeployConfig(config);
+			console.log(result);
+			assert.equal(result.dst, expected);
+		});
+	});
+});

package/test.css

@@ -0,0 +1,5 @@
+/* test.css */
+.yadda.yadda.yadda {
+	content: 'bla, bla, bla';
+}
+.something {}

package/test.js

@@ -0,0 +1,3 @@
+// test.js
+console.log('test');
+// edit