wikiparser-node 0.8.1-m → 0.9.0

package/README.md

@@ -0,0 +1,39 @@
+[![npm version](https://badge.fury.io/js/wikiparser-node.svg)](https://www.npmjs.com/package/wikiparser-node)
+
+# 简介
+wikiparser-node 是一款由 Bhsd 开发的基于 [Node.js](https://nodejs.org/en/) 环境的离线[维基文本](https://www.mediawiki.org/wiki/Wikitext)语法解析器，可以解析绝大部分的维基语法并生成[语法树](https://en.wikipedia.org/wiki/Abstract_syntax_tree)，还可以很方便地对语法树进行查询和修改，最后返回修改后的维基文本。语法树的每个节点对应一个仿照 [HTMLElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement) 类设计的类 [Token](https://github.com/bhsd-harry/wikiparser-node/wiki/01.-Token)。
+
+# 使用方法
+
+```js
+var Parser = require('wikiparser-node');
+```
+
+更多文档请查阅 [Wiki](https://github.com/bhsd-harry/wikiparser-node/wiki)。
+
+# 目录
+
+1. [Parser](https://github.com/bhsd-harry/wikiparser-node/wiki/Home#parser)
+2. [AstElement](https://github.com/bhsd-harry/wikiparser-node/wiki/01.-Token#astelement)
+3. [Token](https://github.com/bhsd-harry/wikiparser-node/wiki/01.-Token#token)
+4. [CommentToken](https://github.com/bhsd-harry/wikiparser-node/wiki/02.-CommentToken等#commenttoken)
+5. [ExtToken](https://github.com/bhsd-harry/wikiparser-node/wiki/03.-ExtToken)
+6. [AttributeToken](https://github.com/bhsd-harry/wikiparser-node/wiki/04.-AttributeToken)
+7. [HeadingToken](https://github.com/bhsd-harry/wikiparser-node/wiki/05.-HeadingToken)
+8. [ArgToken](https://github.com/bhsd-harry/wikiparser-node/wiki/06.-ArgToken)
+9. [TranscludeToken](https://github.com/bhsd-harry/wikiparser-node/wiki/07.-TranscludeToken)
+10. [ParameterToken](https://github.com/bhsd-harry/wikiparser-node/wiki/08.-ParameterToken)
+11. [HtmlToken](https://github.com/bhsd-harry/wikiparser-node/wiki/09.-HtmlToken)
+12. [TableToken](https://github.com/bhsd-harry/wikiparser-node/wiki/10.-TableToken)
+13. [TdToken](https://github.com/bhsd-harry/wikiparser-node/wiki/11.-TdToken)
+14. [DoubleUnderscoreToken](https://github.com/bhsd-harry/wikiparser-node/wiki/12.-DoubleUnderscoreToken)
+15. [LinkToken](https://github.com/bhsd-harry/wikiparser-node/wiki/13.-LinkToken)
+16. [CategoryToken](https://github.com/bhsd-harry/wikiparser-node/wiki/14.-CategoryToken)
+17. [FileToken](https://github.com/bhsd-harry/wikiparser-node/wiki/15.-FileToken和GalleryImageToken#filetoken)
+18. [ImageParameterToken](https://github.com/bhsd-harry/wikiparser-node/wiki/16.-ImageParameterToken)
+19. [ExtLinkToken](https://github.com/bhsd-harry/wikiparser-node/wiki/17.-ExtLinkToken和MagicLinkToken#extlinktoken)
+20. [MagicLinkToken](https://github.com/bhsd-harry/wikiparser-node/wiki/17.-ExtLinkToken和MagicLinkToken#magiclinktoken)
+21. [ConverterToken](https://github.com/bhsd-harry/wikiparser-node/wiki/18.-ConverterToken)
+22. [ConverterRuleToken](https://github.com/bhsd-harry/wikiparser-node/wiki/19.-ConverterRuleToken)
+23. [选择器](https://github.com/bhsd-harry/wikiparser-node/wiki/20.-选择器)
+24. [$ (TokenCollection)](https://github.com/bhsd-harry/wikiparser-node/wiki/21.-$-(TokenCollection))

package/config/moegirl.json

@@ -367,6 +367,7 @@
 			"样式路径": "stylepath",
 			"stylepath": "stylepath",
 			"msgnw": "msgnw",
+			"#regex": "regex",
 			"#related": "related",
 			"#cscore": "cscore"
 		},

package/i18n/zh-hans.json

@@ -0,0 +1,44 @@
+{
+	"<imagemap> without an image": "缺少图片的<imagemap>",
+	"$1 in URL": "URL中的$1",
+	"additional \"|\" in a table cell": "表格单元格中多余的\"|\"",
+	"additional \"|\" in the link text": "链接文本中多余的\"|\"",
+	"attributes of a closing tag": "位于闭合标签的属性",
+	"conflicting image $1 parameter": "冲突的图片$1参数",
+	"containing invalid attribute": "包含无效属性",
+	"content to be moved out from the table": "将被移出表格的内容",
+	"duplicated $1 attribute": "重复的$1属性",
+	"duplicated image $1 parameter": "重复的图片$1参数",
+	"duplicated parameter": "重复参数",
+	"extension tag in HTML tag attributes": "HTML标签属性中的扩展标签",
+	"frame": "框架",
+	"full-width punctuation": "全角标点",
+	"horizontal-alignment": "水平对齐",
+	"HTML tag in table attributes": "表格属性中的HTML标签",
+	"illegal attribute name": "非法的属性名",
+	"illegal module name": "非法的模块名称",
+	"insecure style": "不安全的样式",
+	"invalid content in <$1>": "<$1>内的无效内容",
+	"invalid conversion flag": "无效的转换标记",
+	"invalid gallery image": "无效的图库图片",
+	"invalid gallery image parameter": "无效的图库图片参数",
+	"invalid link in <imagemap>": "无效的<imagemap>链接",
+	"invalid parameter of $1": "$1的无效参数",
+	"invalid self-closing tag": "无效自封闭标签",
+	"invisible content inside triple brackets": "三重括号内的不可见部分",
+	"lonely \"$1\"": "孤立的\"$1\"",
+	"nothing should be in <$1>": "<$1>标签内不应有任何内容",
+	"section header in a HTML tag": "HTML标签属性中的段落标题",
+	"tag that is both closing and self-closing": "同时闭合和自封闭的标签",
+	"unbalanced \"=\" in a section header": "段落标题中不平衡的\"=\"",
+	"unclosed HTML comment": "未闭合的HTML注释",
+	"unclosed quotes": "未闭合的引号",
+	"unclosed table": "未闭合的表格",
+	"unclosed tag": "未闭合的标签",
+	"unescaped query string in an anonymous parameter": "匿名参数中未转义的查询参数",
+	"unexpected template argument": "未预期的模板参数",
+	"unmatched closing tag": "未匹配的闭合标签",
+	"unnecessary URL encoding in an internal link": "内链中不必要的URL编码",
+	"useless fragment": "多余的fragment",
+	"vertical-alignment": "垂直对齐"
+}

package/i18n/zh-hant.json

@@ -0,0 +1,44 @@
+{
+	"<imagemap> without an image": "缺少圖片的<imagemap>",
+	"$1 in URL": "URL中的$1",
+	"additional \"|\" in a table cell": "表哥單元格中多餘的\"|\"",
+	"additional \"|\" in the link text": "連結文本中多餘的\"|\"",
+	"attributes of a closing tag": "位於閉合標籤的屬性",
+	"conflicting image $1 parameter": "衝突的圖片$1參數",
+	"containing invalid attribute": "包含無效屬性",
+	"content to be moved out from the table": "將被移出表格的內容",
+	"duplicated $1 attribute": "重複的$1屬性",
+	"duplicated image $1 parameter": "重複的圖片$1參數",
+	"duplicated parameter": "重複參數",
+	"extension tag in HTML tag attributes": "HTML標籤屬性中的擴展標籤",
+	"frame": "框架",
+	"full-width punctuation": "全形標點",
+	"horizontal-alignment": "水瓶對齊",
+	"HTML tag in table attributes": "表格屬性中的HTML標籤",
+	"illegal attribute name": "非法的屬性名",
+	"illegal module name": "非法的模組名稱",
+	"insecure style": "不安全的樣式",
+	"invalid content in <$1>": "<$1>內的無效內容",
+	"invalid conversion flag": "無效的轉換標記",
+	"invalid gallery image": "無效的圖庫圖片",
+	"invalid gallery image parameter": "無效的圖庫圖片參數",
+	"invalid link in <imagemap>": "無效的<imagemap>連結",
+	"invalid parameter of $1": "$1的無效參數",
+	"invalid self-closing tag": "無效自封閉標籤",
+	"invisible content inside triple brackets": "三重括號內的不可見部分",
+	"lonely \"$1\"": "孤立的\"$1\"",
+	"nothing should be in <$1>": "<$1>標籤內不應有任何內容",
+	"section header in a HTML tag": "HTML標籤屬性中的段落標題",
+	"tag that is both closing and self-closing": "同時閉合和自封閉的標籤",
+	"unbalanced \"=\" in a section header": "段落標題中不平衡的\"=\"",
+	"unclosed HTML comment": "未閉合的HTML註釋",
+	"unclosed quotes": "未閉合的引號",
+	"unclosed table": "未閉合的表格",
+	"unclosed tag": "未閉合的標籤",
+	"unescaped query string in an anonymous parameter": "匿名參數中未轉義的查詢參數",
+	"unexpected template argument": "未預期的模板參數",
+	"unmatched closing tag": "未匹配的閉合標籤",
+	"unnecessary URL encoding in an internal link": "內部連結中不必要的URL編碼",
+	"useless fragment": "多餘的fragment",
+	"vertical-alignment": "垂直對齊"
+}

package/index.js

@@ -1,16 +1,140 @@
 'use strict';
 
+const fs = require('fs'),
+	path = require('path');
+
 const /** @type {Parser} */ Parser = {
-	config: undefined,
-	minConfig: require('./config/minimum'),
+	config: './config/default',
+	i18n: undefined,
 
 	MAX_STAGE: 11,
 
-	getConfig(path) {
-		if (path) {
-			this.config = require(path);
+	warning: true,
+	debugging: false,
+	running: false,
+
+	classes: {},
+	mixins: {},
+	parsers: {},
+	tool: {},
+
+	aliases: [
+		['AstText'],
+		['CommentToken', 'ExtToken', 'IncludeToken', 'NoincludeToken'],
+		['ArgToken', 'TranscludeToken', 'HeadingToken'],
+		['HtmlToken'],
+		['TableToken'],
+		['HrToken', 'DoubleUnderscoreToken'],
+		['LinkToken', 'FileToken', 'CategoryToken'],
+		['QuoteToken'],
+		['ExtLinkToken'],
+		['MagicLinkToken'],
+		['ListToken', 'DdToken'],
+		['ConverterToken'],
+	],
+	typeAliases: {
+		text: ['string', 'str'],
+		plain: ['regular', 'normal'],
+		// comment and extension
+		onlyinclude: ['only-include'],
+		noinclude: ['no-include'],
+		include: ['includeonly', 'include-only'],
+		comment: undefined,
+		ext: ['extension'],
+		'ext-attrs': ['extension-attrs', 'ext-attributes', 'extension-attributes'],
+		'ext-attr-dirty': ['extension-attr-dirty', 'ext-attribute-dirty', 'extension-attribute-dirty'],
+		'ext-attr': ['extension-attr', 'ext-attribute', 'extension-attribute'],
+		'attr-key': ['attribute-key'],
+		'attr-value': ['attribute-value', 'attr-val', 'attribute-val'],
+		'ext-inner': ['extension-inner'],
+		// triple brackets
+		arg: ['argument'],
+		'arg-name': ['argument-name'],
+		'arg-default': ['argument-default'],
+		hidden: ['arg-redundant'],
+		// double brackets
+		'magic-word': ['parser-function', 'parser-func'],
+		'magic-word-name': ['parser-function-name', 'parser-func-name'],
+		'invoke-function': ['invoke-func', 'lua-function', 'lua-func', 'module-function', 'module-func'],
+		'invoke-module': ['lua-module'],
+		template: undefined,
+		'template-name': undefined,
+		parameter: ['param'],
+		'parameter-key': ['param-key'],
+		'parameter-value': ['parameter-val', 'param-value', 'param-val'],
+		// heading
+		heading: ['header'],
+		'heading-title': ['header-title'],
+		'heading-trail': ['header-trail'],
+		// html
+		html: undefined,
+		'html-attrs': ['html-attributes'],
+		'html-attr-dirty': ['html-attribute-dirty'],
+		'html-attr': ['html-attribute'],
+		// table
+		table: undefined,
+		tr: ['table-row'],
+		td: ['table-cell', 'table-data'],
+		'table-syntax': undefined,
+		'table-attrs': ['tr-attrs', 'td-attrs', 'table-attributes', 'tr-attributes', 'td-attributes'],
+		'table-attr-dirty':
+			['tr-attr-dirty', 'td-attr-dirty', 'table-attribute-dirty', 'tr-attribute-dirty', 'td-attribute-dirty'],
+		'table-attr': ['tr-attr', 'td-attr', 'table-attribute', 'tr-attribute', 'td-attribute'],
+		'table-inter': undefined,
+		'td-inner': ['table-cell-inner', 'table-data-inner'],
+		// hr and double-underscore
+		hr: ['horizontal'],
+		'double-underscore': ['underscore', 'behavior-switch', 'behaviour-switch'],
+		// link
+		link: ['wikilink'],
+		'link-target': ['wikilink-target'],
+		'link-text': ['wikilink-text'],
+		category: ['category-link', 'cat', 'cat-link'],
+		file: ['file-link', 'image', 'image-link', 'img', 'img-link'],
+		'gallery-image': ['gallery-file', 'gallery-img'],
+		'imagemap-image': ['imagemap-file', 'imagemap-img', 'image-map-image', 'image-map-file', 'image-map-img'],
+		'image-parameter': ['img-parameter', 'image-param', 'img-param'],
+		// quotes
+		quote: ['quotes', 'quot', 'apostrophe', 'apostrophes', 'apos'],
+		// external link
+		'ext-link': ['external-link'],
+		'ext-link-text': ['external-link-text'],
+		'ext-link-url': ['external-link-url'],
+		// magic link
+		'free-ext-link': ['free-external-link', 'magic-link'],
+		// list
+		list: ['ol', 'ordered-list', 'ul', 'unordered-list', 'dl', 'description-list'],
+		dd: ['indent', 'indentation'],
+		// converter
+		converter: ['convert', 'conversion'],
+		'converter-flags': ['convert-flags', 'conversion-flags'],
+		'converter-flag': ['convert-flag', 'conversion-flag'],
+		'converter-rule': ['convert-rule', 'conversion-rule'],
+		'converter-rule-noconvert': ['convert-rule-noconvert', 'conversion-rule-noconvert'],
+		'converter-rule-variant': ['convert-rule-variant', 'conversion-rule-variant'],
+		'converter-rule-to': ['convert-rule-to', 'conversion-rule-to'],
+		'converter-rule-from': ['convert-rule-from', 'conversion-rule-from'],
+		// specific extensions
+		'param-line': ['parameter-line'],
+		'charinsert-line': undefined,
+		'imagemap-link': ['image-map-link'],
+	},
+
+	promises: [Promise.resolve()],
+
+	getConfig() {
+		if (typeof this.config === 'string') {
+			this.config = require(this.config);
+		}
+		return {...this.config, excludes: []};
+	},
+
+	msg(msg, arg) {
+		if (typeof this.i18n === 'string') {
+			this.i18n = require(this.i18n);
 		}
-		return {...this.minConfig, ...this.config, excludes: []};
+		msg = this.i18n?.[msg] ?? msg;
+		return msg.replace('$1', arg);
 	},
 
 	normalizeTitle(
@@ -35,6 +159,22 @@ const /** @type {Parser} */ Parser = {
 		}
 		const Title = require('./lib/title');
 		const titleObj = new Title(String(title), defaultNs, config, decode, selfLink);
+		if (token) {
+			/**
+			 * 重建部分属性值
+			 * @param {string[]} keys 属性键
+			 */
+			const build = keys => {
+				for (const key of keys) {
+					if (titleObj[key]?.includes('\0')) {
+						titleObj[key] = token.getAttribute('buildFromStr')(titleObj[key], 'text');
+					}
+				}
+			};
+			this.run(() => {
+				build(['title', 'main', 'fragment']);
+			});
+		}
 		return titleObj;
 	},
 
@@ -48,13 +188,96 @@ const /** @type {Parser} */ Parser = {
 			token = new Token(wikitext, config);
 			try {
 				token.parse(maxStage, include);
-			} catch {}
+			} catch (e) {
+				if (e instanceof Error) {
+					const file = path.join(__dirname, 'errors', new Date().toISOString()),
+						stage = token.getAttribute('stage');
+					fs.writeFileSync(file, stage === this.MAX_STAGE ? wikitext : String(token));
+					fs.writeFileSync(`${file}.err`, e.stack);
+					fs.writeFileSync(`${file}.json`, JSON.stringify({
+						stage, include: token.getAttribute('include'), config: this.config,
+					}, null, '\t'));
+				}
+				throw e;
+			}
 		});
+		if (this.debugging) {
+			let restored = String(token),
+				process = '解析';
+			if (restored === wikitext) {
+				const entities = {lt: '<', gt: '>', amp: '&'};
+				restored = token.print().replace(
+					/<[^<]+?>|&([lg]t|amp);/gu,
+					/** @param {string} s */ (_, s) => s ? entities[s] : '',
+				);
+				process = '渲染HTML';
+			}
+			if (restored !== wikitext) {
+				const diff = require('./util/diff');
+				const {promises: {0: cur, length}} = this;
+				this.promises.unshift((async () => {
+					await cur;
+					this.error(`${process}过程中不可逆地修改了原始文本！`);
+					return diff(wikitext, restored, length);
+				})());
+			}
+		}
 		return token;
 	},
 
 	run(callback) {
-		return callback();
+		const {running} = this;
+		this.running = true;
+		try {
+			const result = callback();
+			this.running = running;
+			return result;
+		} catch (e) {
+			this.running = running;
+			throw e;
+		}
+	},
+
+	warn(msg, ...args) {
+		if (this.warning) {
+			console.warn('\x1B[33m%s\x1B[0m', msg, ...args);
+		}
+	},
+	debug(msg, ...args) {
+		if (this.debugging) {
+			console.debug('\x1B[34m%s\x1B[0m', msg, ...args);
+		}
+	},
+	error(msg, ...args) {
+		console.error('\x1B[31m%s\x1B[0m', msg, ...args);
+	},
+	info(msg, ...args) {
+		console.info('\x1B[32m%s\x1B[0m', msg, ...args);
+	},
+
+	log(f) {
+		if (typeof f === 'function') {
+			console.log(String(f));
+		}
+	},
+
+	clearCache() {
+		const entries = [
+			...Object.entries(this.classes),
+			...Object.entries(this.mixins),
+			...Object.entries(this.parsers),
+			...Object.entries(this.tool),
+		];
+		for (const [, filePath] of entries) {
+			try {
+				delete require.cache[require.resolve(filePath)];
+			} catch {}
+		}
+		for (const [name, filePath] of entries) {
+			if (name in global) {
+				global[name] = require(filePath);
+			}
+		}
 	},
 
 	isInterwiki(title, {interwiki} = Parser.getConfig()) {
@@ -62,11 +285,42 @@ const /** @type {Parser} */ Parser = {
 		return new RegExp(`^(${interwiki.join('|')})\\s*:`, 'iu')
 			.exec(title.replaceAll('_', ' ').replace(/^\s*:?\s*/u, ''));
 	},
+
+	reparse(date) {
+		const main = fs.readdirSync(path.join(__dirname, 'errors'))
+			.find(name => name.startsWith(date) && name.endsWith('Z'));
+		if (!main) {
+			throw new RangeError(`找不到对应时间戳的错误记录：${date}`);
+		}
+		const file = path.join(__dirname, 'errors', main),
+			wikitext = fs.readFileSync(file, 'utf8');
+		const {stage, include, config} = require(`${file}.json`),
+			Token = require('./src');
+		this.config = config;
+		return this.run(() => {
+			const halfParsed = stage < this.MAX_STAGE,
+				token = new Token(wikitext, this.getConfig(), halfParsed);
+			if (halfParsed) {
+				token.setAttribute('stage', stage).getAttribute('parseOnce')(stage, include);
+			} else {
+				token.parse(undefined, include);
+			}
+			fs.unlinkSync(file);
+			fs.unlinkSync(`${file}.err`);
+			fs.unlinkSync(`${file}.json`);
+			return token;
+		});
+	},
+
+	getTool() {
+		delete require.cache[require.resolve('./tool')];
+		return require('./tool');
+	},
 };
 
 const /** @type {PropertyDescriptorMap} */ def = {},
-	immutable = new Set(['MAX_STAGE', 'minConfig']),
-	enumerable = new Set(['config', 'normalizeTitle', 'parse', 'isInterwiki']);
+	immutable = new Set(['MAX_STAGE', 'aliases', 'typeAliases', 'promises']),
+	enumerable = new Set(['config', 'normalizeTitle', 'parse', 'isInterwiki', 'getTool']);
 for (const key in Parser) {
 	if (immutable.has(key)) {
 		def[key] = {enumerable: false, writable: false};