umberto 7.0.2 → 8.0.0

package/scripts/utils/pug-to-xml-binding/parser/xml-components-parser.js

@@ -0,0 +1,414 @@
+/**
+ * @license Copyright (c) 2017-2025, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+'use strict';
+
+const { inspect } = require( 'node:util' );
+
+const removeIndentation = require( '../../remove-indentation.js' );
+const ParserCursor = require( '../../parser-cursor.js' );
+
+/**
+ * A parser for converting text containing HTML-like markup into a tree structure.
+ * Uses a cursor-based approach with backtracking capabilities for robust parsing.
+ */
+module.exports = class XMLComponentsParser {
+	#config;
+	#cursor;
+	#tree;
+
+	/**
+	 * Creates a new XMLComponentsParser instance.
+	 *
+	 * @param cursor - The cursor instance to use for parsing.
+	 * @param config - Optional configuration for the parser.
+	 * @param [config.patternElements=[]] - An array of allowed element patterns.
+	 * @param [config.trimRootText=false] - Whether to trim whitespace from the root text content.
+	 * @throws If the cursor is not provided or is invalid.
+	 */
+	constructor( cursor, { patternElements = [], trimRootText = false } = {} ) {
+		this.#cursor = cursor;
+		this.#config = {
+			patternElements
+		};
+
+		this.#tree = this.#eatChildren( {
+			trimText: trimRootText
+		} );
+	}
+
+	/**
+	 * Creates a new parser instance from text input.
+	 *
+	 * @param text - The text to parse.
+	 * @returns A new XMLComponentsParser instance.
+	 */
+	static parseText( text, config ) {
+		return new XMLComponentsParser( new ParserCursor( text ), config );
+	}
+
+	/**
+	 * Gets the parsed tree structure.
+	 *
+	 * @returns The parsed tree as an array of nodes.
+	 */
+	get tree() {
+		return this.#tree;
+	}
+
+	/**
+	 * Returns a string representation of the parsed tree for debugging purposes.
+	 *
+	 * @returns A formatted string representation of the tree.
+	 */
+	toString() {
+		return inspect(
+			this.#tree,
+			{ compact: true, depth: null, breakLength: 80 }
+		);
+	};
+
+	/**
+	 * Creates a wrapper function that provides backtracking capabilities to any matcher function.
+	 * If the matcher returns undefined, the cursor position is automatically restored.
+	 *
+	 * @param matcher - The matcher function to wrap with backtracking.
+	 * @returns A wrapped function with automatic cursor restoration.
+	 */
+	#tryOrRestoreCursor = matcher => ( ...args ) => this.#cursor.try( () => matcher( ...args ) );
+
+	/**
+	 * Creates a context string with error position marker for debugging.
+	 *
+	 * @returns A formatted context string with position marker.
+	 */
+	#getErrorContext() {
+		const region = {
+			start: Math.max( 0, this.#cursor.index - 20 ),
+			end: Math.min( this.#cursor.text.length, this.#cursor.index + 20 )
+		};
+
+		const context = this.#cursor.text.slice( region.start, region.end );
+		const contextMarker = ' '.repeat( this.#cursor.index - region.start ) + '^';
+
+		return `\nContext: ${ context }\n         ${ contextMarker }`;
+	}
+
+	/**
+	 * Parses child nodes until a terminal tag is encountered or end of input is reached.
+	 * Handles text content, backtick-quoted sections, and nested elements.
+	 *
+	 * @param options - Options for parsing children.
+	 * @param [options.trimText=false] - Whether to trim whitespace from text content.
+	 * @param [options.terminalTag] - If provided, parsing stops when this tag is encountered.
+	 * @returns An array of parsed child nodes.
+	 */
+	#eatChildren = this.#tryOrRestoreCursor( ( { trimText, terminalTag } = {} ) => {
+		const children = [];
+		let acc = '';
+
+		for ( ; this.#cursor.index < this.#cursor.text.length; ) {
+			const ch = this.#cursor.text[ this.#cursor.index ];
+
+			if ( ch === '`' ) {
+				acc += this.#eatBacktick();
+				continue;
+			}
+
+			if ( ch === '<' ) {
+				if ( terminalTag ) {
+					const closingTag = this.#eatCloseTag( terminalTag.name );
+
+					if ( closingTag ) {
+						break;
+					}
+				}
+
+				const child = this.#eatElement();
+
+				if ( child ) {
+					flushAccumulator();
+					children.push( child );
+					continue;
+				}
+			}
+
+			acc += ch;
+			this.#cursor.index++;
+		}
+
+		acc.trim();
+		flushAccumulator();
+
+		return children;
+
+		/**
+		 * Flushes the accumulated text content into a text node.
+		 * If the accumulated text is empty, it does nothing.
+		 * Otherwise, it creates a text node with the trimmed content and resets the accumulator.
+		 */
+		function flushAccumulator() {
+			const parsedText = trimText ? removeIndentation( acc ) : acc;
+
+			acc = '';
+
+			if ( parsedText ) {
+				children.push( {
+					type: 'text',
+					content: parsedText
+				} );
+			}
+		}
+	} );
+
+	/**
+	 * Parses a complete HTML-like element including its opening tag, children, and closing tag.
+	 * Handles both self-closing and regular elements.
+	 *
+	 * @returns A parsed element node or undefined if no valid element is found.
+	 */
+	#eatElement = this.#tryOrRestoreCursor( () => {
+		const openTag = this.#eatOpenTag();
+
+		if ( !openTag ) {
+			return;
+		}
+
+		let children;
+
+		if ( openTag.selfClosing ) {
+			children = [];
+		} else {
+			children = this.#eatChildren( {
+				terminalTag: openTag,
+				trimText: true
+			} );
+		}
+
+		return {
+			...openTag,
+			type: 'element',
+			children
+		};
+	} );
+
+	/**
+	 * Parses an opening HTML tag including tag name, attributes, and self-closing indicator.
+	 * Validates tag syntax and throws errors for malformed tags.
+	 *
+	 * @returns A parsed opening tag object or undefined if no valid tag is found.
+	 */
+	#eatOpenTag = this.#tryOrRestoreCursor( () => {
+		if ( !this.#cursor.eatTerminal( '<' ) ) {
+			return;
+		}
+
+		// It's closing tag, e.g. </div>
+		if ( this.#cursor.eatTerminal( '/' ) ) {
+			return;
+		}
+
+		const tag = this.#cursor.eatWord();
+
+		if ( !tag ) {
+			return;
+		}
+
+		const patternElement = findMatchingAllowedTagPattern( this.#config.patternElements, tag );
+
+		if ( !patternElement ) {
+			return;
+		}
+
+		this.#cursor.eatAllNextTerminals( ' \t\n' );
+
+		const attributes = this.#eatAttributes();
+		const isSelfClosing = this.#cursor.eatTerminals( '/>' );
+
+		if ( !isSelfClosing && !this.#cursor.eatTerminal( '>' ) ) {
+			throw new Error(
+				`Invalid tag syntax for <${ tag }> at index ${ this.#cursor.index }. ` +
+				`Expected '>' but found '${ this.#cursor.text[ this.#cursor.index ] }'.` +
+				this.#getErrorContext()
+			);
+		}
+
+		const node = {
+			name: tag,
+			patternElement,
+			attributes,
+			selfClosing: isSelfClosing
+		};
+
+		if ( patternElement.map ) {
+			return patternElement.map( node );
+		}
+
+		return node;
+	} );
+
+	/**
+	 * Parses a closing HTML tag and validates its syntax.
+	 * Used to match closing tags with their corresponding opening tags.
+	 *
+	 * @returns A parsed closing tag object or undefined if no valid closing tag is found.
+	 */
+	#eatCloseTag = this.#tryOrRestoreCursor( tagName => {
+		if ( !this.#cursor.eatTerminals( '</' ) ) {
+			return;
+		}
+
+		const tag = this.#cursor.eatWord();
+
+		if ( !tag ) {
+			return;
+		}
+
+		this.#cursor.eatAllNextTerminals( ' \t\n' );
+
+		if ( !this.#cursor.eatTerminal( '>' ) ) {
+			throw new Error(
+				`Invalid close tag syntax for </${ tag }> at index ${ this.#cursor.index }. ` +
+				`Expected '>' but found '${ this.#cursor.text[ this.#cursor.index ] }'.` +
+				this.#getErrorContext()
+			);
+		}
+
+		if ( tagName && tag !== tagName ) {
+			return;
+		}
+
+		return {
+			name: tag
+		};
+	} );
+
+	/**
+	 * Parses content between backticks, handling escaped characters.
+	 * Backtick sections are treated as literal content that should not be parsed as markup.
+	 *
+	 * @returns The content between backticks including the backticks themselves.
+	 */
+	#eatBacktick = this.#tryOrRestoreCursor( () => {
+		let acc = '`';
+		this.#cursor.index++;
+
+		while ( this.#cursor.index < this.#cursor.text.length ) {
+			const ch = this.#cursor.text[ this.#cursor.index ];
+
+			if ( ch === '`' ) {
+				acc += '`';
+				this.#cursor.index++;
+				break;
+			}
+
+			if ( ch === '\\' ) {
+				acc += '\\';
+				this.#cursor.index++;
+				if ( this.#cursor.index < this.#cursor.text.length ) {
+					acc += this.#cursor.text[ this.#cursor.index ];
+					this.#cursor.index++;
+				}
+			} else {
+				acc += ch;
+				this.#cursor.index++;
+			}
+		}
+
+		return acc;
+	} );
+
+	/**
+	 * Parses all attributes from an HTML tag.
+	 * Continues parsing until no more valid attributes are found.
+	 *
+	 * @returns An object containing all parsed attributes as key-value pairs.
+	 */
+	#eatAttributes = this.#tryOrRestoreCursor( () => {
+		const attributes = Object.create( null );
+
+		do {
+			const result = this.#cursor.try( () => {
+				const value = this.#eatAttribute();
+
+				if ( value ) {
+					this.#cursor.eatAllNextTerminals( ' \t\n' );
+					return value;
+				}
+			} );
+
+			if ( result ) {
+				attributes[ result.name ] = result.value;
+			} else {
+				break;
+			}
+		} while ( true );
+
+		return attributes;
+	} );
+
+	/**
+	 * Parses a single HTML attribute in the format name="value".
+	 * Handles quoted attribute values with proper escaping.
+	 *
+	 * @returns An object with name and value properties, or null if no valid attribute is found.
+	 */
+	#eatAttribute = this.#tryOrRestoreCursor( () => {
+		const name = this.#cursor.eatWord();
+
+		if ( !name ) {
+			return;
+		}
+
+		if ( !this.#cursor.eatTerminal( '=' ) ) {
+			return {
+				name,
+				value: true
+			};
+		}
+
+		const value = this.#cursor.eatQuote();
+
+		if ( value === null ) {
+			return;
+		}
+
+		return {
+			name,
+			value
+		};
+	} );
+};
+
+/**
+ * Checks if a given tag name is allowed based on the specified allowed tags.
+ *
+ * @param patternElements - An array of allowed tag names, functions, or regular expressions.
+ * @param tagName - The tag name to check.
+ * @returns True if the tag is allowed, false otherwise.
+ */
+function findMatchingAllowedTagPattern( patternElements, tagName ) {
+	if ( !patternElements || patternElements.length === 0 ) {
+		return { pattern: true };
+	}
+
+	for ( const element of patternElements ) {
+		const { pattern } = element;
+
+		if ( typeof pattern === 'string' && pattern === tagName ) {
+			return element;
+		}
+
+		if ( typeof pattern === 'function' && pattern( tagName ) ) {
+			return element;
+		}
+
+		if ( pattern instanceof RegExp && pattern.test( tagName ) ) {
+			return element;
+		}
+	}
+
+	return false;
+}

package/scripts/utils/pug-to-xml-binding/render-xml-pug-components-in-markdown.js

@@ -0,0 +1,193 @@
+/**
+ * @license Copyright (c) 2017-2025, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+'use strict';
+
+const transformXMLTreeToPug = require( './transform-xml-tree-to-pug.js' );
+const XMLComponentsParser = require( './parser/xml-components-parser.js' );
+const { walkXMLTree, TreeWalkActions } = require( './parser/walk-xml-tree.js' );
+
+/**
+ * Slices XML components from Markdown content and transforms them to Pug format.
+ *
+ * This function takes text containing XML-like components mixed with regular content,
+ * parses it into a tree structure, and then walks through the tree to transform
+ * XML elements into Pug mixin calls while preserving text content.
+ *
+ * @param xml - The input text containing XML components and regular content
+ * @param options - Configuration options for parsing and transformation
+ * @param options.hexo - The Hexo instance for context (required)
+ * @param options.sourceName - The name of the source file (for error reporting)
+ * @param options.mixinContextVars - Additional attributes to pass to the Pug mixin
+ * @param [options.parserConfig={}] - Configuration passed to XMLComponentsParser
+ * @param [options.pugTransformerConfig={}] - Configuration passed to transformXMLTreeToPug
+ * @returns The transformed content with XML components converted to Pug syntax
+ */
+function renderXMLPugComponentsInMarkdown( xml, options ) {
+	const {
+		sourceName,
+		mixinContextVars,
+		hexo,
+		basePath,
+		patternElements = [],
+		replaceFn = createReplacementBase64Tag
+	} = options;
+
+	const parser = XMLComponentsParser.parseText( xml, {
+		trimRootText: false,
+		patternElements
+	} );
+
+	const markdown = walkXMLTree( parser.tree, {
+		enter( node, depth ) {
+			switch ( node.type ) {
+				case 'text':
+					return TreeWalkActions.replace( node.content );
+
+				case 'element': {
+					const { patternElement } = node;
+
+					if ( patternElement.renderer ) {
+						return TreeWalkActions.replace(
+							patternElement.renderer( node )
+						);
+					}
+
+					// Hexo rendering engine is a little bit clumsy and rendered markdown within
+					// pug mixins often results in broken HTML. It's easier to do it manually, after
+					// the whole tree is rendered.
+					const postfixTexts = [];
+					const content = transformXMLTreeToPug( node, depth, {
+						fallbackGlobalRequire: '_components/index',
+						parseText: ( node, str ) => {
+							if ( !str.trim().length ) {
+								return '';
+							}
+
+							// Handle the text inside the pug.
+							if ( !node.patternElement?.allowMarkdownContent ) {
+								return str;
+							}
+
+							// If the content is a markdown text, we'll postprocess it after rendering whole tree.
+							postfixTexts.push( str );
+							return `__MARKDOWN_TEXT_${ postfixTexts.length - 1 }__`;
+						}
+					} );
+
+					// Let's render the mixin.
+					const locals = Object.assign(
+						{},
+						hexo.locals.toObject(),
+						hexo.extend.helper.list(),
+						{
+							...mixinContextVars,
+							config: hexo.config,
+							mixinContextVars: patternElement.mixinContextVars
+						}
+					);
+
+					// Render component wrapper
+					let finalReplacement = null;
+
+					try {
+						finalReplacement = hexo.render.renderSync(
+							{
+								engine: 'pug',
+								text: content,
+								path: basePath
+							},
+							locals
+						).trim();
+					} catch ( error ) {
+						console.error( error );
+						console.error( `Error rendering component "${ node.name }" in file ${ sourceName }` );
+						console.error( content );
+
+						throw error;
+					}
+
+					// Render markdown content inside the mixin, if present.
+					for ( let i = 0; i < postfixTexts.length; i++ ) {
+						const renderedInnerText = hexo.render.renderSync(
+							{
+								text: postfixTexts[ i ],
+								engine: 'markdown'
+							}
+						).trim();
+
+						finalReplacement = finalReplacement.replace(
+							`__MARKDOWN_TEXT_${ i }__`,
+							renderedInnerText
+						);
+					}
+
+					// Leave magic base64 tag that will be replaced AFTER processing the whole markdown document.
+					return TreeWalkActions.replace(
+						replaceFn( finalReplacement )
+					);
+				}
+			}
+		}
+	} ).join( '' );
+
+	return markdown;
+}
+
+/**
+ * Creates a Base64-encoded replacement tag for component content.
+ *
+ * This function encodes the provided content into a Base64 string and wraps it
+ * in an HTML comment marker that can be later extracted and replaced with the
+ * original content.
+ *
+ * @param content - The HTML/text content to be encoded and wrapped
+ * @returns A comment tag containing the Base64-encoded content
+ */
+function createReplacementBase64Tag( content ) {
+	const encodedContent = Buffer.from( content ).toString( 'base64' );
+
+	return `<!-- COMPONENT:${ encodedContent } -->`;
+}
+
+/**
+ * Extracts and decodes Base64-encoded component markers from page content.
+ *
+ * This function finds HTML comment markers containing Base64-encoded content,
+ * decodes them back to their original form, and replaces the markers with the
+ * decoded content. It processes the content iteratively to handle nested
+ * components and includes a maximum iteration limit to prevent infinite loops.
+ *
+ * @param options - Configuration object
+ * @param options.page - The page object containing the content to process
+ * @param options.maxIterations - Maximum number of processing iterations to prevent infinite loops
+ * @returns The processed content with all component markers replaced
+ */
+function extractReplacementBase64Tag( { page, maxIterations = 5 } ) {
+	let content = page.content;
+	let previousContent;
+
+	const markerRegex = /<!-- COMPONENT:(.*?) -->/g;
+
+	do {
+		previousContent = content;
+		content = content.replace(
+			/<!-- COMPONENT:([a-zA-Z0-9+/=]+) -->/g,
+			( match, encodedContent ) => {
+				try {
+					return Buffer.from( encodedContent, 'base64' ).toString();
+				} catch ( error ) {
+					console.error( `Error decoding component marker in file ${ page.source }`, error );
+					return match;
+				}
+			}
+		);
+	} while ( content !== previousContent && markerRegex.test( content ) && maxIterations-- > 0 );
+
+	return content;
+}
+
+exports.renderXMLPugComponentsInMarkdown = renderXMLPugComponentsInMarkdown;
+exports.extractReplacementBase64Tag = extractReplacementBase64Tag;