sommark 3.3.4 → 4.0.0

package/core/tokenTypes.js

@@ -1,11 +1,44 @@
-// Token Types in SomMark
+/**
+ * Token Types in SomMark.
+ * These represent the basic lexical atoms identified by the lexer.
+ * 
+ * @constant {Object}
+ * @property {string} OPEN_BRACKET - '[' char.
+ * @property {string} CLOSE_BRACKET - ']' char.
+ * @property {string} END_KEYWORD - 'end' value.
+ * @property {string} IDENTIFIER - Block or inline name (e.g. 'Person', 'import', '$use-module').
+ * @property {string} EQUAL - '=' char.
+ * @property {string} VALUE - Data values. Encapsulates Quoted Strings ("...") and Prefix Layers (js{}, p{}).
+ * @property {string} TEXT - Plain unformatted text content.
+ * @property {string} THIN_ARROW - '->' sequence.
+ * @property {string} OPEN_PAREN - '(' char.
+ * @property {string} CLOSE_PAREN - ')' char.
+ * @property {string} OPEN_AT - '@_' sequence (At-Block start).
+ * @property {string} CLOSE_AT - '_@' sequence (At-Header end).
+ * @property {string} COLON - ':' char.
+ * @property {string} COMMA - ',' char.
+ * @property {string} SEMICOLON - ';' char (At-Block separator).
+ * @property {string} COMMENT - '#' comments.
+ * @property {string} ESCAPE - '\' char. Used for literalizing structural chars like '\"' or '\['.
+ * @property {string} QUOTE - '"' delimiter.
+ * @property {string} IMPORT - 'import' keyword.
+ * @property {string} USE_MODULE - '$use-module' keyword.
+ * @property {string} PREFIX_JS - 'js{}' prefix layer.
+ * @property {string} PREFIX_P - 'p{}' placeholder layer.
+ * @property {string} EOF - End of File indicator.
+ */
 const TOKEN_TYPES = {
   OPEN_BRACKET: "OPEN_BRACKET",
   CLOSE_BRACKET: "CLOSE_BRACKET",
   END_KEYWORD: "END_KEYWORD",
+  IMPORT: "IMPORT",
+  USE_MODULE: "USE_MODULE",
   IDENTIFIER: "IDENTIFIER",
   EQUAL: "EQUAL",
   VALUE: "VALUE",
+  QUOTE: "QUOTE",
+  PREFIX_JS: "PREFIX_JS",
+  PREFIX_P: "PREFIX_P",
   TEXT: "TEXT",
   THIN_ARROW: "THIN_ARROW",
   OPEN_PAREN: "OPEN_PAREN",
@@ -17,6 +50,8 @@ const TOKEN_TYPES = {
   SEMICOLON: "SEMICOLON",
   COMMENT: "COMMENT",
   ESCAPE: "ESCAPE",
+  KEY: "KEY",
+  WHITESPACE: "WHITESPACE",
   EOF: "EOF"
 };
 

package/core/transpiler.js

@@ -1,206 +1,289 @@
 import { BLOCK, TEXT, INLINE, ATBLOCK, COMMENT } from "./labels.js";
-import escapeHTML from "../helpers/escapeHTML.js";
 import { transpilerError } from "./errors.js";
-import { textFormat, htmlFormat, markdownFormat, mdxFormat, jsonFormat } from "./formats.js";
+import { textFormat, htmlFormat, markdownFormat, mdxFormat, xmlFormat } from "./formats.js";
+import { matchedValue } from "../helpers/utils.js";
 
 /**
  * SomMark Transpiler
+ * This engine converts the AST into its final text format (like HTML or Markdown)
+ * using rules provided by a mapper.
  */
 
-const BODY_PLACEHOLDER = `__SOMMARK_BODY_PLACEHOLDER_${Math.random().toString(36).slice(2)}__`;
+const BODY_PLACEHOLDER = `SOMMARKBODYPLACEHOLDER${Math.random().toString(36).slice(2)}SOMMARK`;
 
-// ========================================================================== //
-//  Helpers                                                                  //
-// ========================================================================== //
-
-// Finds the matching output definition for a tag identifier
-function matchedValue(outputs, targetId) {
-	if (!outputs || !targetId) return undefined;
-	return outputs.find(output => {
-		if (Array.isArray(output.id)) {
-			return output.id.some(id => id === targetId);
-		}
-		return typeof output.id === "string" && output.id === targetId;
-	});
-}
-
-// Recursively collects all plain text from a node and its children
+/** 
+ * Extracts all plain text from a node and its children.
+ * 
+ * @param {Object} node - The node to read.
+ * @returns {string} - The extracted text.
+ */
 function getNodeText(node) {
-	if (!node.body) return "";
+	if (!node?.body && !node?.content) return "";
+	if (node.type === ATBLOCK) return node.content || "";
 	let text = "";
-	for (const child of node.body) {
-		if (child.type === TEXT) text += child.text;
-		else if (child.type === INLINE) text += child.value;
-		else if (child.type === ATBLOCK) text += child.content;
-		else if (child.type === BLOCK) text += getNodeText(child);
+	if (node.body) {
+		for (const child of node.body) {
+			if (child.type === TEXT) text += child.text || "";
+			else if (child.type === INLINE) text += child.value || "";
+			else if (child.type === ATBLOCK) text += child.content || "";
+			else if (child.type === BLOCK) text += getNodeText(child);
+		}
 	}
 	return text;
 }
 
-// ========================================================================== //
-//  Core Rendering Logic                                                     //
-// ========================================================================== //
 
-// Processes an individual node and its body to produce formatted output
+/** 
+ * Converts a code node into its final format (like HTML).
+ * 
+ * @param {Object|Object[]} ast - The node or list of nodes to convert.
+ * @param {number} i - The current position in the list.
+ * @param {string} format - The target format (e.g., 'html').
+ * @param {Object} mapper_file - The rules for how to convert each node.
+ * @returns {Promise<string>} - The final text for this node.
+ */
 async function generateOutput(ast, i, format, mapper_file) {
 	const node = Array.isArray(ast) ? ast[i] : ast;
+	if (!node) return "";
+
 	let result = "";
 	let context = "";
+	let isParentBlock = false;
+
+	if (node.id === mapper_file?.options?.moduleIdentityToken) {
+		const oldFilename = mapper_file.options.filename;
+		mapper_file.options.filename = node.args?.filename || oldFilename;
+		let bodyOutput = "";
+		if (node.body) {
+			for (let j = 0; j < node.body.length; j++) {
+				bodyOutput += await generateOutput(node.body, j, format, mapper_file);
+			}
+		}
+		mapper_file.options.filename = oldFilename;
+		return bodyOutput;
+	}
+
+	if (node.type === TEXT) {
+		const text = String(node.text || "");
+		return mapper_file ? mapper_file.text(text) : text;
+	}
+
+	if (node.type === COMMENT) {
+		if (mapper_file?.options?.removeComments) return "";
+		const cleanComment = String(node.text || "").replace(/^#/, "").trim();
+		return " ".repeat(node.depth) + `${mapper_file?.comment(cleanComment) || ""}`;
+	}
+
 	let target = mapper_file ? matchedValue(mapper_file.outputs, node.id) : null;
-	
-	if (!target) {
+	if (!target && mapper_file) {
 		target = mapper_file.getUnknownTag(node);
 	}
 
 	if (target) {
-		// ========================================================================== //
-		//  Always use placeholders for blocks to support wrapping                      //
-		// ========================================================================== //
-		const isParentBlock = format === mdxFormat && node.body.length > 1;
-		const placeholder = isParentBlock ? `\n${BODY_PLACEHOLDER}\n` : BODY_PLACEHOLDER;
+		const shouldResolveImmediate = target.options?.resolve === true;
 		const textContent = getNodeText(node);
 
-		result += target.render.call(mapper_file, { args: node.args, content: placeholder, textContent, ast: node });
+		let content = (node.body?.length === 0) ? "" :
+			(node.type === ATBLOCK ? (node.content || "") :
+				(node.type === INLINE ? (node.value || "") : BODY_PLACEHOLDER));
+
+		// Apply pipelines to format literal values
+		if (node.type === INLINE) {
+			content = String(content || "");
+			content = mapper_file ? mapper_file.inlineText(content, target.options) : content;
+		}
+
+		// 1. Determine if this is a parent block that needs newline wrapping (Trim-and-Wrap)
+		// Priority: Target options > Mapper global options
+		const effectiveTrimAndWrap = (target.options?.trimAndWrapBlocks !== undefined)
+			? target.options.trimAndWrapBlocks
+			: mapper_file?.options?.trimAndWrapBlocks;
+
+		isParentBlock = !shouldResolveImmediate && effectiveTrimAndWrap &&
+			(node.body?.length > 1 || (node.body?.length === 1 && textContent.trim().includes('\n')));
+
+		if (isParentBlock) {
+			content = `\n${BODY_PLACEHOLDER}\n`;
+		}
+
+		if (shouldResolveImmediate && node.body) {
+			let resolvedBody = "";
+			for (let j = 0; j < node.body.length; j++) {
+				resolvedBody += await generateOutput(node.body, j, format, mapper_file);
+			}
+			content = resolvedBody;
+		}
+
+		result += await target.render.call(mapper_file, { nodeType: node.type, args: node.args, content, textContent, ast: node });
 		if (isParentBlock) result = "\n" + result + "\n";
 
-		// ========================================================================== //
-		//  Process body nodes recursively                                           //
-		// ========================================================================== //
-		for (let j = 0; j < node.body.length; j++) {
-			const body_node = node.body[j];
-			switch (body_node.type) {
-				case TEXT:
-					const shouldEscapeText = target.options?.escape !== false;
-					context += (format === htmlFormat) && shouldEscapeText ? escapeHTML(body_node.text) : body_node.text;
-					break;
-
-				case INLINE:
-					let inlineTarget = matchedValue(mapper_file.outputs, body_node.id);
-					if (!inlineTarget) {
-						inlineTarget = mapper_file.getUnknownTag(body_node);
-					}
-
-					if (inlineTarget) {
-						const shouldEscapeInline = inlineTarget.options?.escape !== false;
-						context +=
-							inlineTarget.render.call(mapper_file, {
-								args: body_node.args.length > 0 ? body_node.args : [],
-								content: (format === htmlFormat) && shouldEscapeInline ? escapeHTML(body_node.value) : body_node.value,
+		if (shouldResolveImmediate) {
+			return result;
+		}
+
+		const isManualMode = target.options?.handleAst === true;
+
+		if (!isManualMode && node.body) {
+			let prev_body_node = null;
+			for (let j = 0; j < node.body.length; j++) {
+				const body_node = node.body[j];
+				let bodyOutput = "";
+
+				switch (body_node.type) {
+					case TEXT:
+						const text = String(body_node.text || "");
+						bodyOutput = mapper_file ? mapper_file.text(text, target?.options) : text;
+						break;
+
+					case INLINE:
+						let inlineTarget = matchedValue(mapper_file.outputs, body_node.id);
+						if (!inlineTarget) {
+							inlineTarget = mapper_file.getUnknownTag(body_node);
+						}
+
+						if (inlineTarget) {
+							let inlineValue = String(body_node.value || "").trim();
+							if (mapper_file) inlineValue = mapper_file.inlineText(inlineValue, inlineTarget.options);
+
+							const hasArgs = body_node.args && typeof body_node.args === "object" && Object.keys(body_node.args).length > 0;
+							bodyOutput = await inlineTarget.render.call(mapper_file, {
+								nodeType: body_node.type,
+								args: hasArgs ? body_node.args : {},
+								content: inlineValue,
 								ast: body_node
-							}) + (format === mdxFormat ? "\n" : "");
-					}
-					break;
-
-				case ATBLOCK:
-					let atTarget = matchedValue(mapper_file.outputs, body_node.id);
-					if (!atTarget) {
-						atTarget = mapper_file.getUnknownTag(body_node);
-					}
-					if (atTarget) {
-						const shouldEscapeAt = atTarget.options?.escape !== false;
-						let content = body_node.content;
-						if (shouldEscapeAt && (format === htmlFormat)) {
-							content = escapeHTML(content);
+							});
+						} else {
+							let fallback = body_node.value || "";
+							if (mapper_file) fallback = mapper_file.inlineText(fallback, {});
+							bodyOutput = fallback;
+						}
+						break;
+
+					case ATBLOCK:
+						console.log(`[TRANSPILER] Processing ATBLOCK: ${body_node.id}`);
+						let atTarget = matchedValue(mapper_file.outputs, body_node.id);
+						if (!atTarget) {
+							atTarget = mapper_file.getUnknownTag(body_node);
 						}
-						const rendered = atTarget.render.call(mapper_file, { args: body_node.args, content, ast: body_node });
-						const finalAtRendered = (format === mdxFormat) ? rendered : rendered.trimEnd() + "\n";
-						context = context.trim() ? context.trimEnd() + "\n" + finalAtRendered : context + finalAtRendered;
-					}
-					break;
-
-				case COMMENT:
-					context += " ".repeat(body_node.depth) + `\n${mapper_file.comment(body_node.text)}\n`;
-					break;
-
-				case BLOCK:
-					const blockOutput = await generateOutput(body_node, i, format, mapper_file);
-					const blockIsParent = format === mdxFormat && body_node.body.length > 1;
-					if (format === mdxFormat && !blockIsParent) {
-						context += blockOutput;
-					} else {
-						context = context.trim() ? context.trimEnd() + "\n" + blockOutput : context + blockOutput;
-					}
-					break;
+
+						let atContent = String(body_node.content || "").trim();
+						if (mapper_file) {
+							console.log(`[TRANSPILER] Calling atBlockBody for ${body_node.id}`);
+							atContent = mapper_file.atBlockBody(atContent, atTarget?.options || {});
+						}
+
+						// Removed multiline injection since atBlockBody handles formatting
+						bodyOutput = atTarget
+							? await atTarget.render.call(mapper_file, { nodeType: body_node.type, args: body_node.args, content: atContent, ast: body_node })
+							: atContent;
+						break;
+
+					case COMMENT:
+						if (mapper_file?.options?.removeComments) break;
+						const cleanComment = String(body_node.text || "").replace(/^#/, "").trim();
+						bodyOutput = " ".repeat(body_node.depth) + `${mapper_file.comment(cleanComment)}`;
+						break;
+
+					case BLOCK:
+						bodyOutput = await generateOutput(body_node, 0, format, mapper_file);
+						break;
+				}
+
+				if (bodyOutput) {
+					context += bodyOutput;
+				}
 			}
 		}
 
-		result = result.replaceAll(BODY_PLACEHOLDER, context);
+		// Trim only leading/trailing newlines and their surrounding spaces to preserve indentation
+		const finalContext = effectiveTrimAndWrap ? context.replace(/^\s*[\r\n]+|[\r\n]+\s*$/g, "") : context;
+
+		if (result.includes(BODY_PLACEHOLDER)) {
+			result = result.replaceAll(BODY_PLACEHOLDER, finalContext);
+		} else if (finalContext.trim()) {
+			result += finalContext;
+		}
 	}
 	else if (format === textFormat) {
-		for (const body_node of node.body) {
-			switch (body_node.type) {
-				case TEXT: context += body_node.text; break;
-				case INLINE: context += body_node.value + " "; break;
-				case ATBLOCK: context += body_node.content.trimEnd() + "\n"; break;
-				case BLOCK:
-					const textBlockOutput = await generateOutput(body_node, i, format, mapper_file);
-					context = context.trim() ? context.trimEnd() + "\n" + textBlockOutput : context + textBlockOutput;
-					break;
+		if (node.type === ATBLOCK) {
+			result = node.content || "";
+		} else if (node.type === INLINE) {
+			result = node.value || "";
+		} else if (node.body) {
+			for (const body_node of node.body) {
+				switch (body_node.type) {
+					case TEXT: context += body_node.text || ""; break;
+					case INLINE: context += (body_node.value || "") + " "; break;
+					case ATBLOCK: context += (body_node.content || "").trimEnd() + "\n"; break;
+					case BLOCK:
+						const textBlockOutput = await generateOutput(body_node, 0, format, mapper_file);
+						context = context.trim() ? context.trimEnd() + "\n" + textBlockOutput : context + textBlockOutput;
+						break;
+				}
 			}
+			result += context;
 		}
-		result += context;
 	} else {
 		transpilerError([
 			"{line}<$red:Invalid Identifier:$> ",
 			`<$yellow:Identifier$> <$blue:'${node.id}'$> <$yellow: is not found in mapping outputs$>{line}`
 		]);
 	}
-	const newline = (format === mdxFormat && node.body.length <= 1) ? "" : "\n";
-	const finalResult = (format === mdxFormat) ? result : result.trimEnd();
-	return finalResult + newline;
+
+	return result;
 }
 
-// ========================================================================== //
-//  Main Transpiler Entry Point                                              //
-// ========================================================================== //
+/** 
+ * The main entry point for the SomMark Transpiler.
+ * It takes an AST and turns it into the final formatted output.
+ * 
+ * @param {Object|Object[]} optionsOrAst - Either the full options object or just the AST.
+ * @param {string} [format] - The target format.
+ * @param {Object} [mapperFile] - The mapper rules to use.
+ * @returns {Promise<string>} - The final formatted document.
+ */
+export async function transpiler(optionsOrAst, format, mapperFile) {
+	let body = null;
+	let targetFormat = format;
+	let targetMapper = mapperFile;
+
+	if (typeof optionsOrAst === "object" && !Array.isArray(optionsOrAst) && (optionsOrAst.ast || Array.isArray(optionsOrAst))) {
+		if (optionsOrAst.ast) {
+			const root = optionsOrAst.ast;
+			body = Array.isArray(root) ? root : (root.body || [root]);
+			targetFormat = optionsOrAst.format;
+			targetMapper = optionsOrAst.mapperFile;
+		} else if (Array.isArray(optionsOrAst)) {
+			body = optionsOrAst;
+		}
+	} else if (Array.isArray(optionsOrAst)) {
+		body = optionsOrAst;
+	}
+
+	if (!body || !Array.isArray(body)) return "";
 
-async function transpiler({ ast, format, mapperFile, includeDocument = true }) {
 	let output = "";
-	for (let i = 0; i < ast.length; i++) {
-		const node = ast[i];
-		switch (node.type) {
-			case BLOCK:
-				output += await generateOutput(ast, i, format, mapperFile);
-				break;
-			case COMMENT:
-				output += mapperFile.comment(node.text);
-				break;
-			case TEXT:
-				const shouldEscapeText = (format === htmlFormat);
-				output += shouldEscapeText ? escapeHTML(node.text) : node.text;
-				break;
-			case INLINE:
-				let inlineTarget = matchedValue(mapperFile.outputs, node.id);
-				if (!inlineTarget) inlineTarget = mapperFile.getUnknownTag(node);
-				if (inlineTarget) {
-					const shouldEscapeInline = inlineTarget.options?.escape !== false;
-					output += inlineTarget.render.call(mapperFile, {
-						args: node.args.length > 0 ? node.args : [],
-						content: (format === htmlFormat) && shouldEscapeInline ? escapeHTML(node.value) : node.value,
-						ast: node
-					}) + (format === mdxFormat ? "\n" : "");
-				}
-				break;
-			case ATBLOCK:
-				let atTarget = matchedValue(mapperFile.outputs, node.id);
-				if (!atTarget) atTarget = mapperFile.getUnknownTag(node);
-				if (atTarget) {
-					const shouldEscapeAt = atTarget.options?.escape !== false;
-					let content = node.content;
-					if (shouldEscapeAt && (format === htmlFormat)) {
-						content = escapeHTML(content);
-					}
-					const rendered = atTarget.render.call(mapperFile, { args: node.args, content, ast: node });
-					const finalAtRendered = (format === mdxFormat) ? rendered : rendered.trimEnd() + "\n";
-					output = output.trim() ? output.trimEnd() + "\n" + finalAtRendered : output + finalAtRendered;
-				}
-				break;
+	let prev_body_node = null;
+	for (let i = 0; i < body.length; i++) {
+		const node = body[i];
+		const blockOutput = await generateOutput(body, i, targetFormat, targetMapper);
+
+		if (blockOutput) {
+			output += blockOutput;
+			if (node.type !== TEXT || node.text.trim().length > 0) {
+				prev_body_node = node;
+			}
+		} else if (node.type === COMMENT && targetMapper?.options?.removeComments) {
+			// If a comment is removed, check the next node.
+			// If it's just a blank line, skip it so we don't have extra gaps in the output.
+			const nextNode = body[i + 1];
+			if (nextNode && nextNode.type === TEXT && (nextNode.text === "\n" || nextNode.text === "\r\n")) {
+				i++; // Skip the next newline node
+			}
 		}
 	}
-	// ========================================================================== //
-	//  Final Post-Processing (Dynamic Formats)                                   //
-	// ========================================================================== //
-	return mapperFile.formatOutput(output, includeDocument);
+
+	return output.trim();
 }
 
 export default transpiler;

package/core/validator.js

@@ -0,0 +1,79 @@
+import { transpilerError } from "./errors.js";
+
+/**
+ * SomMark Rules Validator
+ * 
+ * This module ensures the AST matches the expected structure.
+ * It focuses on structural integrity (like self-closing tags)
+ * while leaving specific patterns to the mapper's discretion.
+ */
+
+/**
+ * Runs all validation rules against a single code node.
+ * 
+ * @param {Object} node - The code node to check.
+ * @param {Object} target - The rule definition for this node.
+ * @param {Object} instance - The current SomMark context.
+ */
+const runValidations = (node, target, instance) => {
+	if (!target || !target.options) return;
+	const rules = target.options.rules || {};
+	const id = (target.id) ? (Array.isArray(target.id) ? target.id.join(" | ") : target.id) : "Unknown";
+	const context = instance ? { src: instance.src, range: node.range, filename: instance.filename } : null;
+
+	// -- Structural Integrity (Self-Closing) ------------------------------ //
+	if (rules.is_self_closing && (node.type === "Block" && (node.body && node.body.length > 0))) {
+		transpilerError(
+			[
+				"<$red:Validation Error:$> ",
+				`<$yellow:Identifier$> <$blue:'${id}'$> <$yellow:is self-closing and cannot have children (body).$>`
+			],
+			context
+		);
+	}
+};
+
+/**
+ * Checks every node in the entire document for structural issues.
+ * 
+ * @param {Object|Array} ast - The Abstract Syntax Tree to validate.
+ * @param {Object} mapperFile - The mapper instance containing tag registrations.
+ * @param {Object} instance - The current SomMark context.
+ * @returns {Object|Array} - The validated AST.
+ */
+export function validateAST(ast, mapperFile, instance) {
+	if (!mapperFile) return ast;
+
+	const validateNode = (node) => {
+		if (!node) return;
+
+		// Handle filename context updates for module identity tokens
+		if (instance?.moduleIdentityToken && node.id === instance.moduleIdentityToken) {
+			const oldFilename = instance.filename;
+			instance.filename = node.args?.filename || oldFilename;
+			if (node.body) {
+				node.body.forEach(child => validateNode(child));
+			}
+			instance.filename = oldFilename;
+			return;
+		}
+
+		// 1. Identify Target
+		if (node.id) {
+			const target = mapperFile.get(node.id) || (mapperFile.getUnknownTag ? mapperFile.getUnknownTag(node) : null);
+			if (target) {
+				runValidations(node, target, instance);
+			}
+		}
+
+		// 2. Recursive Traversal
+		if (node.body && Array.isArray(node.body)) {
+			node.body.forEach(child => validateNode(child));
+		}
+	};
+
+	const rootNodes = Array.isArray(ast) ? ast : [ast];
+	rootNodes.forEach(node => validateNode(node));
+
+	return ast;
+}