wikiparser-node 0.8.0-b → 0.8.0-m

package/parser/brackets.js

@@ -0,0 +1,119 @@
+'use strict';
+
+const {removeComment} = require('../util/string'),
+	Parser = require('..'),
+	HeadingToken = require('../src/heading'),
+	TranscludeToken = require('../src/transclude'),
+	ArgToken = require('../src/arg');
+
+/**
+ * 解析花括号
+ * @param {string} text wikitext
+ * @param {accum} accum
+ * @throws TranscludeToken.constructor()
+ */
+const parseBrackets = (text, config = Parser.getConfig(), accum = []) => {
+	const source = `${config.excludes.includes('heading') ? '' : '^(\0\\d+c\x7F)*={1,6}|'}\\[\\[|\\{{2,}|-\\{(?!\\{)`,
+		/** @type {BracketExecArray[]} */ stack = [],
+		closes = {'=': '\n', '{': '\\}{2,}|\\|', '-': '\\}-', '[': '\\]\\]'},
+		/** @type {Record<string, string>} */ marks = {'!': '!', '!!': '+', '(!': '{', '!)': '}', '!-': '-', '=': '~'};
+	let regex = new RegExp(source, 'gmu'),
+		/** @type {BracketExecArray} */ mt = regex.exec(text),
+		moreBraces = text.includes('}}'),
+		lastIndex;
+	while (mt || lastIndex <= text.length && stack.at(-1)?.[0]?.[0] === '=') {
+		if (mt?.[1]) {
+			const [, {length}] = mt;
+			mt[0] = mt[0].slice(length);
+			mt.index += length;
+		}
+		const {0: syntax, index: curIndex} = mt ?? {0: '\n', index: text.length},
+			/** @type {BracketExecArray} */ top = stack.pop() ?? {},
+			{0: open, index, parts, findEqual: topFindEqual, pos: topPos} = top,
+			innerEqual = syntax === '=' && topFindEqual;
+		if (syntax === ']]' || syntax === '}-') { // 情形1：闭合内链或转换
+			lastIndex = curIndex + 2;
+		} else if (syntax === '\n') { // 情形2：闭合标题
+			lastIndex = curIndex + 1;
+			const {pos, findEqual} = stack.at(-1) ?? {};
+			if (!pos || findEqual || removeComment(text.slice(pos, index)) !== '') {
+				const rmt = /^(={1,6})(.+)\1((?:\s|\0\d+c\x7F)*)$/u.exec(text.slice(index, curIndex));
+				if (rmt) {
+					text = `${text.slice(0, index)}\0${accum.length}h\x7F${text.slice(curIndex)}`;
+					lastIndex = index + 4 + String(accum.length).length;
+					new HeadingToken(rmt[1].length, rmt.slice(2), config, accum);
+				}
+			}
+		} else if (syntax === '|' || innerEqual) { // 情形3：模板内部，含行首单个'='
+			lastIndex = curIndex + 1;
+			parts.at(-1).push(text.slice(topPos, curIndex));
+			if (syntax === '|') {
+				parts.push([]);
+			}
+			top.pos = lastIndex;
+			top.findEqual = syntax === '|';
+			stack.push(top);
+		} else if (syntax.startsWith('}}')) { // 情形4：闭合模板
+			const close = syntax.slice(0, Math.min(open.length, 3)),
+				rest = open.length - close.length,
+				{length} = accum;
+			lastIndex = curIndex + close.length; // 这不是最终的lastIndex
+			parts.at(-1).push(text.slice(topPos, curIndex));
+			let skip = false,
+				ch = 't';
+			if (close.length === 3) {
+				new ArgToken(parts.map(part => part.join('=')), config, accum);
+			} else {
+				const name = removeComment(parts[0][0]).trim();
+				if (name in marks) {
+					ch = marks[name]; // 标记{{!}}等
+				} else if (/^(?:fullurl|canonicalurl|filepath):.|^server$/iu.test(name)) {
+					ch = 'm';
+				} else if (/^#vardefine:./iu.test(name)) {
+					ch = 'c';
+				}
+				try {
+					new TranscludeToken(parts[0][0], parts.slice(1), config, accum);
+				} catch (e) {
+					if (e instanceof Error && e.message.startsWith('非法的模板名称：')) {
+						lastIndex = index + open.length;
+						skip = true;
+					} else {
+						throw e;
+					}
+				}
+			}
+			if (!skip) {
+				text = `${text.slice(0, index + rest)}\0${length}${ch}\x7F${text.slice(lastIndex)}`;
+				lastIndex = index + rest + 3 + String(length).length;
+				if (rest > 1) {
+					stack.push({0: open.slice(0, rest), index, pos: index + rest, parts: [[]]});
+				} else if (rest === 1 && text[index - 1] === '-') {
+					stack.push({0: '-{', index: index - 1, pos: index + 1, parts: [[]]});
+				}
+			}
+		} else { // 情形5：开启
+			lastIndex = curIndex + syntax.length;
+			if (syntax[0] === '{') {
+				mt.pos = lastIndex;
+				mt.parts = [[]];
+			}
+			stack.push(...'0' in top ? [top] : [], mt);
+		}
+		moreBraces &&= text.slice(lastIndex).includes('}}');
+		let curTop = stack.at(-1);
+		if (!moreBraces && curTop?.[0]?.[0] === '{') {
+			stack.pop();
+			curTop = stack.at(-1);
+		}
+		regex = new RegExp(source + (curTop
+			? `|${closes[curTop[0][0]]}${curTop.findEqual ? '|=' : ''}`
+			: ''
+		), 'gmu');
+		regex.lastIndex = lastIndex;
+		mt = regex.exec(text);
+	}
+	return text;
+};
+
+module.exports = parseBrackets;

package/parser/commentAndExt.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const Parser = require('..'),
+	OnlyincludeToken = require('../src/onlyinclude'),
+	NoincludeToken = require('../src/nowiki/noinclude'),
+	IncludeToken = require('../src/tagPair/include'),
+	ExtToken = require('../src/tagPair/ext'),
+	CommentToken = require('../src/nowiki/comment');
+
+/**
+ * 解析HTML注释和扩展标签
+ * @param {string} text wikitext
+ * @param {accum} accum
+ * @param {boolean} includeOnly 是否嵌入
+ */
+const parseCommentAndExt = (text, config = Parser.getConfig(), accum = [], includeOnly = false) => {
+	const onlyinclude = /<onlyinclude>(.*?)<\/onlyinclude>/gsu;
+	if (includeOnly && text.search(onlyinclude) !== -1) { // `<onlyinclude>`拥有最高优先级
+		return text.replace(onlyinclude, /** @param {string} inner */ (_, inner) => {
+			const str = `\0${accum.length}e\x7F`;
+			new OnlyincludeToken(inner, config, accum);
+			return str;
+		}).replace(/(^|\0\d+e\x7F)(.*?)(?=$|\0\d+e\x7F)/gsu, (_, lead, substr) => {
+			if (substr === '') {
+				return lead;
+			}
+			new NoincludeToken(substr, config, accum);
+			return `${lead}\0${accum.length - 1}c\x7F`;
+		});
+	}
+	const ext = config.ext.join('|'),
+		includeRegex = includeOnly ? 'includeonly' : '(?:no|only)include',
+		noincludeRegex = includeOnly ? 'noinclude' : 'includeonly',
+		regex = new RegExp(
+			'<!--.*?(?:-->|$)|' // comment
+			+ `<${includeRegex}(?:\\s[^>]*?)?>|</${includeRegex}\\s*>|` // <includeonly>
+			+ `<(${ext})(\\s[^>]*?)?(?:/>|>(.*?)</(\\1\\s*)>)|` // 扩展标签
+			+ `<(${noincludeRegex})(\\s[^>]*?)?(?:/>|>(.*?)(?:</(\\5\\s*)>|$))`, // <noinclude>
+			'gisu',
+		);
+	return text.replace(
+		regex,
+		/** @type {function(...string): string} */
+		(substr, name, attr, inner, closing, include, includeAttr, includeInner, includeClosing) => {
+			const str = `\0${accum.length}${name ? 'e' : 'c'}\x7F`;
+			if (name) {
+				new ExtToken(name, attr, inner, closing, config, accum);
+			} else if (substr.startsWith('<!--')) {
+				const closed = substr.endsWith('-->');
+				new CommentToken(substr.slice(4, closed ? -3 : undefined), closed, config, accum);
+			} else if (include) {
+				new IncludeToken(include, includeAttr, includeInner, includeClosing, config, accum);
+			} else {
+				new NoincludeToken(substr, config, accum);
+			}
+			return str;
+		},
+	);
+};
+
+module.exports = parseCommentAndExt;

package/parser/converter.js

@@ -0,0 +1,45 @@
+'use strict';
+
+const Parser = require('..'),
+	ConverterToken = require('../src/converter');
+
+/**
+ * 解析语言变体转换
+ * @param {string} wikitext wikitext
+ * @param {accum} accum
+ */
+const parseConverter = (wikitext, config = Parser.getConfig(), accum = []) => {
+	const regex1 = /-\{/gu,
+		regex2 = /-\{|\}-/gu,
+		/** @type {RegExpExecArray[]} */ stack = [];
+	let regex = regex1,
+		mt = regex.exec(wikitext);
+	while (mt) {
+		const {0: syntax, index} = mt;
+		if (syntax === '}-') {
+			const top = stack.pop(),
+				{length} = accum,
+				str = wikitext.slice(top.index + 2, index),
+				i = str.indexOf('|'),
+				[flags, text] = i === -1 ? [[], str] : [str.slice(0, i).split(';'), str.slice(i + 1)],
+				temp = text.replace(/(&[#a-z\d]+);/giu, '$1\x01'),
+				variants = `(?:${config.variants.join('|')})`,
+				rules = temp.split(new RegExp(`;(?=\\s*(?:${variants}|[^;]*?=>\\s*${variants})\\s*:)`, 'u'))
+					.map(rule => rule.replaceAll('\x01', ';'));
+			new ConverterToken(flags, rules, config, accum);
+			wikitext = `${wikitext.slice(0, top.index)}\0${length}v\x7F${wikitext.slice(index + 2)}`;
+			if (stack.length === 0) {
+				regex = regex1;
+			}
+			regex.lastIndex = top.index + 3 + String(length).length;
+		} else {
+			stack.push(mt);
+			regex = regex2;
+			regex.lastIndex = index + 2;
+		}
+		mt = regex.exec(wikitext);
+	}
+	return wikitext;
+};
+
+module.exports = parseConverter;

package/parser/externalLinks.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const {extUrlChar, extUrlCharFirst} = require('../util/string'),
+	Parser = require('..'),
+	ExtLinkToken = require('../src/extLink');
+
+/**
+ * 解析外部链接
+ * @param {string} wikitext wikitext
+ * @param {accum} accum
+ */
+const parseExternalLinks = (wikitext, config = Parser.getConfig(), accum = []) => {
+	const regex = new RegExp(
+		`\\[((?:(?:${config.protocol}|//)${extUrlCharFirst}|\0\\d+m\x7F)${
+			extUrlChar
+		})(\\p{Zs}*)([^\\]\x01-\x08\x0A-\x1F\uFFFD]*)\\]`,
+		'giu',
+	);
+	return wikitext.replace(regex, /** @type {function(...string): string} */ (_, url, space, text) => {
+		const {length} = accum,
+			mt = /&[lg]t;/u.exec(url);
+		if (mt) {
+			url = url.slice(0, mt.index);
+			space = '';
+			text = `${url.slice(mt.index)}${space}${text}`;
+		}
+		new ExtLinkToken(url, space, text, config, accum);
+		return `\0${length}w\x7F`;
+	});
+};
+
+module.exports = parseExternalLinks;

package/parser/hrAndDoubleUnderscore.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const Parser = require('..'),
+	AstText = require('../lib/text'),
+	Token = require('../src'),
+	HrToken = require('../src/nowiki/hr'),
+	DoubleUnderscoreToken = require('../src/nowiki/doubleUnderscore');
+
+/**
+ * 解析\<hr\>和状态开关
+ * @param {Token & {firstChild: AstText}} root 根节点
+ * @param {accum} accum
+ */
+const parseHrAndDoubleUnderscore = ({firstChild: {data}, type, name}, config = Parser.getConfig(), accum = []) => {
+	const {doubleUnderscore} = config,
+		insensitive = new Set(doubleUnderscore[0]),
+		sensitive = new Set(doubleUnderscore[1]);
+	if (type !== 'root' && (type !== 'ext-inner' || name !== 'poem')) {
+		data = `\0${data}`;
+	}
+	data = data.replace(/^((?:\0\d+c\x7F)*)(-{4,})/gmu, (_, lead, m) => {
+		new HrToken(m.length, config, accum);
+		return `${lead}\0${accum.length - 1}r\x7F`;
+	}).replace(
+		new RegExp(`__(${doubleUnderscore.flat().join('|')})__`, 'giu'),
+		/** @param {string} p1 */ (m, p1) => {
+			if (insensitive.has(p1.toLowerCase()) || sensitive.has(p1)) {
+				new DoubleUnderscoreToken(p1, config, accum);
+				return `\0${accum.length - 1}u\x7F`;
+			}
+			return m;
+		},
+	);
+	return type === 'root' || type === 'ext-inner' && name === 'poem' ? data : data.slice(1);
+};
+
+module.exports = parseHrAndDoubleUnderscore;

package/parser/html.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const Parser = require('..'),
+	AttributesToken = require('../src/attributes'),
+	HtmlToken = require('../src/html');
+
+/**
+ * 解析HTML标签
+ * @param {string} wikitext wikitext
+ * @param {accum} accum
+ */
+const parseHtml = (wikitext, config = Parser.getConfig(), accum = []) => {
+	const regex = /^(\/?)([a-z][^\s/>]*)((?:\s|\/(?!>))[^>]*?)?(\/?>)([^<]*)$/iu,
+		elements = new Set(config.html.flat()),
+		bits = wikitext.split('<');
+	let text = bits.shift();
+	for (const x of bits) {
+		const mt = regex.exec(x),
+			t = mt?.[2],
+			name = t?.toLowerCase();
+		if (!mt || !elements.has(name)) {
+			text += `<${x}`;
+			continue;
+		}
+		const [, slash,, params = '', brace, rest] = mt,
+			attr = new AttributesToken(params, 'html-attrs', name, config, accum),
+			itemprop = attr.getAttr('itemprop');
+		if (name === 'meta' && (itemprop === undefined || attr.getAttr('content') === undefined)
+			|| name === 'link' && (itemprop === undefined || attr.getAttr('href') === undefined)
+		) {
+			text += `<${x}`;
+			accum.pop();
+			continue;
+		}
+		text += `\0${accum.length}x\x7F${rest}`;
+		new HtmlToken(t, attr, slash === '/', brace === '/>', config, accum);
+	}
+	return text;
+};
+
+module.exports = parseHtml;

package/parser/links.js

@@ -0,0 +1,93 @@
+'use strict';
+
+const Parser = require('..'),
+	LinkToken = require('../src/link'),
+	FileToken = require('../src/link/file'),
+	CategoryToken = require('../src/link/category');
+
+/**
+ * 解析内部链接
+ * @param {string} wikitext wikitext
+ * @param {accum} accum
+ */
+const parseLinks = (wikitext, config = Parser.getConfig(), accum = []) => {
+	const parseQuotes = require('./quotes.js');
+	const regex = /^((?:(?!\0\d+!\x7F)[^\n<>[\]{}|])+)(?:(\||\0\d+!\x7F)(.*?[^\]]))?\]\](.*)$/su,
+		regexImg = /^((?:(?!\0\d+!\x7F)[^\n<>[\]{}|])+)(\||\0\d+!\x7F)(.*)$/su,
+		regexExt = new RegExp(`^\\s*(?:${config.protocol})`, 'iu'),
+		bits = wikitext.split('[[');
+	let s = bits.shift();
+	for (let i = 0; i < bits.length; i++) {
+		let mightBeImg, link, delimiter, text, after;
+		const x = bits[i],
+			m = regex.exec(x);
+		if (m) {
+			[, link, delimiter, text, after] = m;
+			if (after[0] === ']' && text?.includes('[')) {
+				text += ']';
+				after = after.slice(1);
+			}
+		} else {
+			const m2 = regexImg.exec(x);
+			if (m2) {
+				mightBeImg = true;
+				[, link, delimiter, text] = m2;
+			}
+		}
+		if (link === undefined || regexExt.test(link) || /\0\d+[exhbru]\x7F/u.test(link)) {
+			s += `[[${x}`;
+			continue;
+		}
+		const force = link.trim()[0] === ':';
+		if (force && mightBeImg) {
+			s += `[[${x}`;
+			continue;
+		}
+		const title = Parser.normalizeTitle(link, 0, false, config, true, true, true),
+			{ns, valid} = title;
+		if (!valid) {
+			s += `[[${x}`;
+			continue;
+		} else if (mightBeImg) {
+			if (ns !== 6) {
+				s += `[[${x}`;
+				continue;
+			}
+			let found;
+			for (i++; i < bits.length; i++) {
+				const next = bits[i],
+					p = next.split(']]');
+				if (p.length > 2) {
+					found = true;
+					text += `[[${p[0]}]]${p[1]}`;
+					after = p.slice(2).join(']]');
+					break;
+				} else if (p.length === 2) {
+					text += `[[${p[0]}]]${p[1]}`;
+				} else {
+					text += `[[${next}`;
+					break;
+				}
+			}
+			text = parseLinks(text, config, accum);
+			if (!found) {
+				s += `[[${link}${delimiter}${text}`;
+				continue;
+			}
+		}
+		text &&= parseQuotes(text, config, accum);
+		s += `\0${accum.length}l\x7F${after}`;
+		let SomeLinkToken = LinkToken;
+		if (!force) {
+			if (ns === 6) {
+				SomeLinkToken = FileToken;
+			} else if (ns === 14) {
+				SomeLinkToken = CategoryToken;
+			}
+		}
+		new SomeLinkToken(link, text, title, config, accum, delimiter);
+	}
+	return s;
+};
+
+module.exports = parseLinks;

package/parser/list.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const Parser = require('..'),
+	ListToken = require('../src/nowiki/list'),
+	DdToken = require('../src/nowiki/dd');
+
+/**
+ * 解析列表
+ * @param {string} text wikitext
+ * @param {accum} accum
+ */
+const parseList = (text, config = Parser.getConfig(), accum = []) => {
+	const mt = /^((?:\0\d+c\x7F)*)([;:*#]+)/u.exec(text);
+	if (!mt) {
+		return text;
+	}
+	const [total, comment, prefix] = mt;
+	text = `${comment}\0${accum.length}d\x7F${text.slice(total.length)}`;
+	new ListToken(prefix, config, accum);
+	let dt = prefix.split(';').length - 1;
+	if (!dt) {
+		return text;
+	}
+	let regex = /:+|-\{/gu,
+		ex = regex.exec(text),
+		lc = 0;
+	while (ex && dt) {
+		const {0: syntax, index} = ex;
+		if (syntax[0] === ':') {
+			if (syntax.length >= dt) {
+				new DdToken(':'.repeat(dt), config, accum);
+				return `${text.slice(0, index)}\0${accum.length - 1}d\x7F${text.slice(index + dt)}`;
+			}
+			text = `${text.slice(0, index)}\0${accum.length}d\x7F${text.slice(regex.lastIndex)}`;
+			dt -= syntax.length;
+			regex.lastIndex = index + 4 + String(accum.length).length;
+			new DdToken(syntax, config, accum);
+		} else if (syntax === '-{') {
+			if (!lc) {
+				const {lastIndex} = regex;
+				regex = /-\{|\}-/gu;
+				regex.lastIndex = lastIndex;
+			}
+			lc++;
+		} else {
+			lc--;
+			if (!lc) {
+				const {lastIndex} = regex;
+				regex = /:+|-\{/gu;
+				regex.lastIndex = lastIndex;
+			}
+		}
+		ex = regex.exec(text);
+	}
+	return text;
+};
+
+module.exports = parseList;

package/parser/magicLinks.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const {extUrlChar, extUrlCharFirst} = require('../util/string'),
+	Parser = require('..'),
+	MagicLinkToken = require('../src/magicLink');
+
+/**
+ * 解析自由外链
+ * @param {string} wikitext wikitext
+ * @param {accum} accum
+ */
+const parseMagicLinks = (wikitext, config = Parser.getConfig(), accum = []) => {
+	const regex = new RegExp(`(^|[^\\p{L}\\d_])(?:${config.protocol})(${extUrlCharFirst}${extUrlChar})`, 'giu');
+	return wikitext.replace(regex, /** @param {string} p1 */ (m, lead, p1) => {
+		let trail = '',
+			url = lead ? m.slice(1) : m;
+		const m2 = /&(?:lt|gt|nbsp|#x0*(?:3[ce]|a0)|#0*(?:6[02]|160));/iu.exec(url);
+		if (m2) {
+			trail = url.slice(m2.index);
+			url = url.slice(0, m2.index);
+		}
+		const sep = new RegExp(`[,;.:!?${url.includes('(') ? '' : ')'}]+$`, 'u'),
+			sepChars = sep.exec(url);
+		if (sepChars) {
+			let correction = 0;
+			if (sepChars[0][0] === ';' && /&(?:[a-z]+|#x[\da-f]+|#\d+)$/iu.test(url.slice(0, sepChars.index))) {
+				correction = 1;
+			}
+			trail = `${url.slice(sepChars.index + correction)}${trail}`;
+			url = url.slice(0, sepChars.index + correction);
+		}
+		if (trail.length >= p1.length) {
+			return m;
+		}
+		new MagicLinkToken(url, false, config, accum);
+		return `${lead}\0${accum.length - 1}w\x7F${trail}`;
+	});
+};
+
+module.exports = parseMagicLinks;

package/parser/quotes.js

@@ -0,0 +1,63 @@
+'use strict';
+
+const Parser = require('..'),
+	QuoteToken = require('../src/nowiki/quote');
+
+/**
+ * 解析单引号
+ * @param {string} text wikitext
+ * @param {accum} accum
+ */
+const parseQuotes = (text, config = Parser.getConfig(), accum = []) => {
+	const arr = text.split(/('{2,})/u),
+		{length} = arr;
+	if (length === 1) {
+		return text;
+	}
+	let nBold = 0,
+		nItalic = 0,
+		firstSingle, firstMulti, firstSpace;
+	for (let i = 1; i < length; i += 2) {
+		const {length: len} = arr[i];
+		switch (len) {
+			case 2:
+				nItalic++;
+				break;
+			case 4:
+				arr[i - 1] += `'`;
+				arr[i] = `'''`;
+				// fall through
+			case 3:
+				nBold++;
+				if (firstSingle) {
+					break;
+				} else if (arr[i - 1].endsWith(' ')) {
+					if (!firstMulti && !firstSpace) {
+						firstSpace = i;
+					}
+				} else if (arr[i - 1].at(-2) === ' ') {
+					firstSingle = i;
+				} else {
+					firstMulti ||= i;
+				}
+				break;
+			default:
+				arr[i - 1] += `'`.repeat(len - 5);
+				arr[i] = `'''''`;
+				nItalic++;
+				nBold++;
+		}
+	}
+	if (nItalic % 2 === 1 && nBold % 2 === 1) {
+		const i = firstSingle ?? firstMulti ?? firstSpace;
+		arr[i] = `''`;
+		arr[i - 1] += `'`;
+	}
+	for (let i = 1; i < length; i += 2) {
+		new QuoteToken(arr[i].length, config, accum);
+		arr[i] = `\0${accum.length - 1}q\x7F`;
+	}
+	return arr.join('');
+};
+
+module.exports = parseQuotes;