sommark 3.3.4 → 4.0.0

package/core/lexer.js

@@ -5,247 +5,526 @@ import { lexerError } from "./errors.js";
 
 /**
  * SomMark Lexer
+ * 
+ * Transforms a raw SomMark source string into a stream of tokens.
+ * It uses a state-machine approach to handle complex contexts like At-Block bodies,
+ * quoted values, and hierarchical headers.
+ * 
+ * @param {string} src - The raw SomMark source code.
+ * @param {string} [filename="anonymous"] - Source filename for error reporting.
+ * @returns {Array<Object>} Array of token objects.
  */
+function lexer(src, filename = "anonymous") {
+	if (!src || typeof src !== "string") return [];
+	const tokens = [];
 
-// ========================================================================== //
-//  Helper Functions                                                         //
-// ========================================================================== //
+	let prev_type = "";
+	let last_non_junk_type = ""; // Tracks the last real token for context guessing
+	let i = 0;
+	let line = 0, character = 0;
 
-const atBlockEndRegex = new RegExp(`^@_\\s*${end_keyword}\\s*_@`);
+	// State Variables
+	let isInAtBlockBody = false;
+	let isInQuote = false;
+	let isInHeader = false; // Tracks if we are in a structural header context
+	let isInInlineHead = false; // Specific for (key:val) after ->
+	let parenDepth = 0; // To track balanced parentheses in inlines
+	let delimiterStack = []; // To track block nesting for body mode
 
-// Checks if we reached the end of an At-Block
-function isAtBlockEnd(input, index) {
-	const slice = typeof input === "string" ? input.slice(index, index + 100) : input.slice(index, index + 100).join("");
-	return atBlockEndRegex.test(slice);
-}
+	/**
+	 * Adds a token to the stream and updates the scanner's position tracking.
+	 * 
+	 * @param {string} type - The type of token (from TOKEN_TYPES).
+	 * @param {string} value - The literal text content of the token.
+	 */
+	function addToken(type, value) {
+		const start = { line, character };
 
-// Collects characters inside a quoted string
-function concatQuote(input, index) {
-	let text = "\"";
-	for (let i = index + 1; i < input.length; i++) {
-		const char = input[i];
-		if (char === "\\" && peek(input, i, 1) === "\"") {
-			text += "\\\"";
-			i++;
-			continue;
+		// Update position
+		const parts = value.split("\n");
+		if (parts.length > 1) {
+			line += parts.length - 1;
+			character = parts[parts.length - 1].length;
+		} else {
+			character += value.length;
+		}
+
+		const end = { line, character };
+		tokens.push({
+			type,
+			value,
+			source: filename,
+			range: { start, end },
+			depth: delimiterStack.length
+		});
+
+		prev_type = type;
+		if (type !== TOKEN_TYPES.WHITESPACE && type !== TOKEN_TYPES.COMMENT) {
+			if (type !== TOKEN_TYPES.TEXT || value.trim() !== "") {
+				last_non_junk_type = type;
+			}
 		}
-		text += char;
-		if (char === "\"") return text;
 	}
-	lexerError(["[Lexer Error]: Unclosed quote"]);
-	return text;
-}
 
-// Collects plain text until a special character is found
-function concatText(input, index, isInHeader, isInAtBlockBody, isLiberalValue = false) {
-	let text = "";
-	if (index >= input.length) return text;
-	for (let i = index; i < input.length; i++) {
-		const char = input[i];
-		const stopConditions = [
-			["[", !isInAtBlockBody],
-			["(", !isInAtBlockBody],
-			["#", !isInAtBlockBody && !isLiberalValue],
-			["=", isInHeader && !isInAtBlockBody],
-			["\"", isInHeader],
-			["]", isInHeader],
-			[")", isInHeader],
-			["-", peek(input, i, 1) === ">" && (isInHeader || true)],
-			["@", peek(input, i, 1) === "_" && (!isInAtBlockBody || isAtBlockEnd(input, i))],
-			["_", peek(input, i, 1) === "@" && isInHeader],
-			["\\", true],
-			[":", isInHeader && !isInAtBlockBody],
-			[";", isInHeader],
-			[",", isInHeader]
-		];
-		let shouldStop = false;
-		for (const [stopChar, conditionMet] of stopConditions) {
-			if (conditionMet && input.substring(i, i + stopChar.length) === stopChar) {
-				shouldStop = true;
-				break;
-			}
-		}
-		if (shouldStop) break;
-		text += char;
+	/**
+	 * Looks ahead to find the next structural character, skipping whitespace and comments.
+	 * Used for context-guessing (e.g., distinguishing KEY from VALUE).
+	 * 
+	 * @param {number} start - Index to start peeking from.
+	 * @returns {string|null} The next structural character or null if EOF.
+	 */
+	function peekStructural(start) {
+		let j = start;
+		while (j < src.length) {
+			const c = src[j];
+			if (c === " " || c === "\t" || c === "\n" || c === "\r") {
+				j++;
+				continue;
+			}
+			if (c === "#") {
+				while (j < src.length && src[j] !== "\n") j++;
+				continue;
+			}
+			if (c === "\\") {
+				// Escape sequence: jump over the backslash and the escaped char
+				j += 2;
+				continue;
+			}
+			return c;
+		}
+		return null;
 	}
-	return text;
-}
 
-// Handles backslash escapes in the text
-function concatEscape(input, index) {
-	if (index >= input.length) return "";
-	const nextChar = peek(input, index, 1);
-	const WHITESPACES = [" ", "\t", "\n", "\r", "\v", "\f"];
-	if (WHITESPACES.includes(nextChar)) lexerError(["[Lexer Error]: Invalid escape sequence (escaped whitespace)"]);
-	if (input[index] === "\\" && nextChar !== null) return "\\" + nextChar;
-	lexerError(["[Lexer Error]: Invalid escape sequence"]);
-	return "";
-}
+	while (i < src.length) {
+		// --- PHASE 1: AT-BLOCK BODY MODE ---
+		// In this mode, we consume everything as raw text until we hit the @_ marker.
+		if (isInAtBlockBody) {
+			if (src[i] === "@" && src[i + 1] === "_") {
+				isInAtBlockBody = false;
+			} else {
+				let body = "";
+				while (i < src.length) {
+					// Handle escapes in At-Block Body
+					if (src[i] === "\\" && i + 1 < src.length) {
+						body += src[i + 1];
+						i += 2;
+						continue;
+					}
+					// Stop at end marker
+					if (src[i] === "@" && src[i + 1] === "_") {
+						break;
+					}
+					body += src[i];
+					i++;
+				}
+				if (body.length > 0) {
+					addToken(TOKEN_TYPES.TEXT, body);
+				}
+				continue;
+			}
+		}
+		const char = src[i];
+		const next = src[i + 1];
 
-// ========================================================================== //
-//  Main Lexer Function                                                      //
-// ========================================================================== //
+		// --- PHASE 2: QUOTE MODE ---
+		// Handles balanced strings and allows prefix layers (js{}, p{}) inside them.
+		if (isInQuote) {
+			let quoteValue = "";
+			const quoteChar = tokens[tokens.length - 1].value;
+			while (i < src.length) {
+				if (src[i] === "\\" && i + 1 < src.length) {
+					// Inside quotes, we split escapes if we want to match reliability tests
+					if (quoteValue.length > 0) addToken(TOKEN_TYPES.VALUE, quoteValue);
+					addToken(TOKEN_TYPES.ESCAPE, "\\" + src[i + 1]);
+					quoteValue = "";
+					i += 2;
+					continue;
+				}
+				
+				// Support Prefix Layers inside quotes!
+				if ((src[i] === "j" && src[i+1] === "s" && src[i+2] === "{") || (src[i] === "p" && src[i+1] === "{")) {
+					const isJS = (src[i] === "j");
+					if (quoteValue.length > 0) {
+						addToken(TOKEN_TYPES.VALUE, quoteValue);
+						quoteValue = "";
+					}
+					
+					let braceDepth = 1;
+					let prefixValue = isJS ? "js{" : "p{";
+					i += isJS ? 3 : 2;
 
-function lexer(src) {
-	if (!src || typeof src !== "string") return [];
-	const tokens = [];
-	let isInHeader = false, isInAtBlockBody = false;
-	let line = 0, character = 0, depth_stack = [];
-
-	// ========================================================================== //
-	//  Token Creation Helpers                                                   //
-	// ========================================================================== //
-
-	function addToken(type, value, rawValue) {
-		if (typeof rawValue === "string" && typeof value === "string" && rawValue !== value) {
-			const offset = rawValue.indexOf(value);
-			if (offset !== -1) {
-				advance(rawValue.slice(0, offset));
-				const startPos = { line, character }; advance(value);
-				const endPos = { line, character };
-				tokens.push({ type, value, range: { start: startPos, end: endPos }, depth: depth_stack.length });
-				advance(rawValue.slice(offset + value.length));
-				return;
-			}
-		}
-		const startPos = { line, character }; advance(rawValue || value);
-		const endPos = { line, character };
-		tokens.push({ type, value, range: { start: startPos, end: endPos }, depth: depth_stack.length });
-	}
+					let internalString = null;
+					while (i < src.length && braceDepth > 0) {
+						const c = src[i];
+						const n = src[i + 1];
+						if (internalString) {
+							if (c === "\\" && (n === internalString || n === "\\")) {
+								prefixValue += c + n;
+								i += 2;
+								continue;
+							}
+							if (c === internalString) internalString = null;
+						} else {
+							if (c === "\"" || c === "'") internalString = c;
+							else if (c === "{") braceDepth++;
+							else if (c === "}") braceDepth--;
+						}
+						prefixValue += c;
+						i++;
+					}
+					addToken(isJS ? TOKEN_TYPES.PREFIX_JS : TOKEN_TYPES.PREFIX_P, prefixValue);
+					continue;
+				}
 
-	function advance(text) {
-		const newlines = (text.match(/\n/g) || []).length;
-		if (newlines > 0) { line += newlines; character = text.split("\n").pop().length; }
-		else character += text.length;
-	}
+				if (src[i] === quoteChar) {
+					// Guess role based on next structural character
+					let nextStructural = peekStructural(i + 1);
+					let tokenType = (isInHeader || isInInlineHead) && (nextStructural === ":" || nextStructural === "=")
+						? TOKEN_TYPES.KEY
+						: TOKEN_TYPES.VALUE;
 
-	function validateIdentifier(id, charPos) {
-		if (!/^[a-zA-Z0-9\-_$]+$/.test(id.trim())) {
-			lexerError([`[Lexer Error]: Invalid Identifier: '${id.trim()}' at line ${line + 1}, col ${charPos || character}`]);
+					if (quoteValue.length > 0) addToken(tokenType, quoteValue);
+					addToken(TOKEN_TYPES.QUOTE, quoteChar);
+					isInQuote = false;
+					i++;
+					break;
+				}
+				quoteValue += src[i];
+				i++;
+			}
+			if (!isInQuote) continue;
 		}
-	}
 
-	// ========================================================================== //
-	//  Main Tokenization Loop                                                   //
-	// ========================================================================== //
-
-	for (let i = 0; i < src.length; i++) {
-		const char = src[i];
-		const next = peek(src, i, 1);
+		// --- PHASE 3: STRUCTURAL PARSING ---
+		// Handles markers, whitespace, and structural symbols.
 		
-		// ========================================================================== //
-		//  Look back at previous tokens to determine current context                //
-		// ========================================================================== //
-		let prev_type = "", prev_prev_type = "", count = 0;
-		for (let j = tokens.length - 1; j >= 0; j--) {
-			const t = tokens[j];
-			if (t.type !== TOKEN_TYPES.TEXT && t.type !== TOKEN_TYPES.COMMENT) {
-				if (count === 0) prev_type = t.type;
-				else if (count === 1) prev_prev_type = t.type;
-				count++; if (count >= 2) break;
-			}
-		}
-
-		// ========================================================================== //
-		//  Check for structural characters ([ ], ( ), @_, _@)                      //
-		// ========================================================================== //
-
-		if (char === "[" && !isInAtBlockBody) {
-			let idPeek = ""; for (let j = i + 1; j < src.length && !/[=\]:#]/.test(src[j]); j++) idPeek += src[j];
-			if (idPeek.trim() !== end_keyword) depth_stack.push("B");
-			addToken(TOKEN_TYPES.OPEN_BRACKET, char); isInHeader = true;
-		} else if (char === "]" && isInHeader) {
-			addToken(TOKEN_TYPES.CLOSE_BRACKET, char); isInHeader = false;
-			// Reliable depth pop on [end]
-			for (let j = tokens.length - 1; j >= 0; j--) {
-				const t = tokens[j];
-				if (t.type === TOKEN_TYPES.IDENTIFIER || t.type === TOKEN_TYPES.END_KEYWORD) {
-					if (t.type === TOKEN_TYPES.END_KEYWORD || t.value.trim() === end_keyword) depth_stack.pop();
-					break;
+		// WHITESPACE
+		if (char === "\n") {
+			addToken(TOKEN_TYPES.WHITESPACE, char);
+			i++;
+			continue;
+		}
+
+		if (char === " " || char === "\t" || char === "\r") {
+			let ws = "";
+			while (i < src.length && (src[i] === " " || src[i] === "\t" || src[i] === "\r")) {
+				ws += src[i];
+				i++;
+			}
+			addToken(TOKEN_TYPES.WHITESPACE, ws);
+			continue;
+		}
+
+		// COMMENTS
+		if (char === "#") {
+			let comm = "";
+			while (i < src.length && src[i] !== "\n") {
+				comm += src[i];
+				i++;
+			}
+			addToken(TOKEN_TYPES.COMMENT, comm);
+			continue;
+		}
+
+		// ESCAPE CHARACTER (Sequence-based)
+		if (char === "\\") {
+			const seq = i + 1 < src.length ? "\\" + src[i + 1] : "\\";
+			addToken(TOKEN_TYPES.ESCAPE, seq);
+			i += seq.length;
+			continue;
+		}
+
+		// PREFIX LAYERS (js{...} or p{...})
+		if ((char === "j" && next === "s" && src[i+2] === "{") || (char === "p" && next === "{")) {
+			const isJS = (char === "j");
+			const isP = (char === "p");
+			
+			// Context Check
+			const top = (delimiterStack.length > 0) ? delimiterStack[delimiterStack.length - 1] : null;
+			const isInBlockHeader = isInHeader && top === "[";
+			const isInNormalText = !isInHeader && !isInInlineHead && !isInAtBlockBody && parenDepth === 0;
+			
+			let allowed = false;
+			if (isJS && isInBlockHeader) allowed = true;
+			if (isP && (isInBlockHeader || isInNormalText)) allowed = true;
+
+			if (allowed) {
+				let braceDepth = 1;
+				let prefixValue = isJS ? "js{" : "p{";
+				i += isJS ? 3 : 2;
+
+				let inString = null; // Track if we are inside " " or ' '
+				while (i < src.length && braceDepth > 0) {
+					const c = src[i];
+					const n = src[i + 1];
+
+					if (inString) {
+						if (c === "\\" && (n === inString || n === "\\")) {
+							prefixValue += c + n;
+							i += 2;
+							continue;
+						}
+						if (c === inString) inString = null;
+					} else {
+						if (c === "\"" || c === "'") inString = c;
+						else if (c === "{") braceDepth++;
+						else if (c === "}") braceDepth--;
+					}
+					prefixValue += c;
+					i++;
 				}
+				addToken(isJS ? TOKEN_TYPES.PREFIX_JS : TOKEN_TYPES.PREFIX_P, prefixValue);
+				continue;
 			}
-		} else if (char === "(" && !isInAtBlockBody) {
-			addToken(TOKEN_TYPES.OPEN_PAREN, char); isInHeader = true;
-		} else if (char === ")" && isInHeader) {
-			addToken(TOKEN_TYPES.CLOSE_PAREN, char); isInHeader = false;
-		} else if (char === "@" && next === "_" && (!isInAtBlockBody || isAtBlockEnd(src, i))) {
-			let idPeek = ""; for (let j = i + 2; j < src.length && !/[_@:#]/.test(src[j]); j++) idPeek += src[j];
-			if (idPeek.trim() !== end_keyword) depth_stack.push("A");
-			addToken(TOKEN_TYPES.OPEN_AT, "@_"); i++; isInHeader = true;
-		} else if (char === "_" && next === "@" && (isInHeader || isInAtBlockBody)) {
-			addToken(TOKEN_TYPES.CLOSE_AT, "_@"); i++;
-			for (let j = tokens.length - 1; j >= 0; j--) {
-				const t = tokens[j];
-				if (t.type === TOKEN_TYPES.IDENTIFIER || t.type === TOKEN_TYPES.END_KEYWORD) {
-					if (t.type === TOKEN_TYPES.END_KEYWORD || t.value.trim() === end_keyword) depth_stack.pop();
-					break;
+			// If not allowed, it will fall through to normal word scanning
+		}
+
+		// MULTI-CHAR MARKERS
+		if (char === "@" && next === "_") {
+			addToken(TOKEN_TYPES.OPEN_AT, "@_");
+			i += 2;
+			if (!isInAtBlockBody) delimiterStack.push("@");
+			isInHeader = true; // At-Blocks start with a header part
+			continue;
+		}
+		if (char === "-" && next === ">") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, "-");
+				i++; // Swallowed one char
+			} else {
+				addToken(TOKEN_TYPES.THIN_ARROW, "->");
+				i += 2;
+				isInInlineHead = true; // The following ( ) will be structural
+			}
+			continue;
+		}
+
+		// SINGLE-CHAR MARKERS
+		if (char === "[") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, "[");
+			} else {
+				addToken(TOKEN_TYPES.OPEN_BRACKET, "[");
+				delimiterStack.push("[");
+				isInHeader = true;
+			}
+			i++;
+			continue;
+		}
+		if (char === "_" && next === "@") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, "_@");
+			} else {
+				const lastRealType = last_non_junk_type;
+				addToken(TOKEN_TYPES.CLOSE_AT, "_@");
+				const top = delimiterStack[delimiterStack.length - 1];
+				if (top === "@") {
+					if (lastRealType === TOKEN_TYPES.END_KEYWORD) {
+						delimiterStack.pop();
+						isInAtBlockBody = false;
+						isInHeader = false;
+					}
 				}
 			}
-			isInHeader = true; isInAtBlockBody = false;
-		} else if (char === ";" && isInHeader) {
-			addToken(TOKEN_TYPES.SEMICOLON, char); isInHeader = false; isInAtBlockBody = true;
-		} else if (char === "=" && isInHeader && !isInAtBlockBody) {
-			addToken(TOKEN_TYPES.EQUAL, char);
-		} else if (char === ":" && isInHeader && !isInAtBlockBody && (prev_type === TOKEN_TYPES.IDENTIFIER || prev_type === TOKEN_TYPES.CLOSE_AT)) {
-			addToken(TOKEN_TYPES.COLON, char);
-		} else if (char === "," && isInHeader) {
-			addToken(TOKEN_TYPES.COMMA, char);
-		} else if (char === "-" && next === ">" && (isInHeader || prev_type === TOKEN_TYPES.CLOSE_PAREN)) {
-			addToken(TOKEN_TYPES.THIN_ARROW, "->"); i++;
-		} else if (char === "\"" && isInHeader) {
-			const quote = concatQuote(src, i); addToken(TOKEN_TYPES.VALUE, quote); i += quote.length - 1;
-		} else if (char === "\\") {
-			const esc = concatEscape(src, i); addToken(TOKEN_TYPES.ESCAPE, esc); i += esc.length - 1;
-		} else if (char === "#" && !isInAtBlockBody) {
-			let comm = ""; for (; i < src.length && src[i] !== "\n"; i++) comm += src[i];
-			addToken(TOKEN_TYPES.COMMENT, comm, comm); i--;
-		} else if (char === "\n" && !isInAtBlockBody) {
-			advance(char);
-		} else {
-			// ========================================================================== //
-			//  Capture plain text or Identifier values                                 //
-			// ========================================================================== //
-			const isValueContext = (prev_type === TOKEN_TYPES.COLON || prev_type === TOKEN_TYPES.EQUAL);
-			const context = concatText(src, i, isInHeader, isInAtBlockBody, isValueContext);
-			if (context.length > 0) {
-				if (isInHeader) {
-					const trimmed = context.trim();
-					if ((prev_type === TOKEN_TYPES.OPEN_BRACKET || prev_type === TOKEN_TYPES.OPEN_AT) && trimmed === end_keyword) {
-						addToken(TOKEN_TYPES.END_KEYWORD, trimmed, context);
-					} else if (trimmed.length > 0) {
-						let isNextColon = false;
-						for (let j = i + context.length; j < src.length; j++) {
-							const c = src[j];
-							if (c === " " || c === "\t" || c === "\n") continue;
-							if (c === ":") isNextColon = true;
-							break;
-						}
-						
-						const isBlockStart = (prev_type === TOKEN_TYPES.OPEN_BRACKET || prev_type === TOKEN_TYPES.OPEN_AT);
-						const isMapperHead = (prev_type === TOKEN_TYPES.OPEN_PAREN && prev_prev_type === TOKEN_TYPES.THIN_ARROW);
-						const isMandatoryId = (isNextColon || prev_type === TOKEN_TYPES.THIN_ARROW);
-						
-						if (isBlockStart || isMapperHead || isMandatoryId) {
-							validateIdentifier(trimmed, character + context.indexOf(trimmed));
-							addToken(TOKEN_TYPES.IDENTIFIER, trimmed, context);
-						} else {
-							addToken(TOKEN_TYPES.VALUE, trimmed, context);
+			i += 2;
+			continue;
+		}
+		if (char === "]") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, "]");
+			} else {
+				addToken(TOKEN_TYPES.CLOSE_BRACKET, "]");
+				isInHeader = false;
+			}
+			i++;
+			continue;
+		}
+		if (char === "(") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, "(");
+				parenDepth++;
+			} else {
+				addToken(TOKEN_TYPES.OPEN_PAREN, "(");
+				parenDepth++;
+			}
+			i++;
+			continue;
+		}
+		if (char === ")") {
+			if (isInAtBlockBody || (parenDepth > 1 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, ")");
+				parenDepth--;
+			} else if (parenDepth > 0) {
+				// This ends the content part if depth drops to 0
+				parenDepth--;
+				if (parenDepth === 0) {
+					addToken(TOKEN_TYPES.CLOSE_PAREN, ")");
+					if (isInInlineHead) isInInlineHead = false;
+				} else {
+					addToken(TOKEN_TYPES.TEXT, ")");
+				}
+			} else {
+				addToken(TOKEN_TYPES.TEXT, ")");
+			}
+			i++;
+			continue;
+		}
+		if (char === ":") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, ":");
+			} else {
+				const allowed = [TOKEN_TYPES.IDENTIFIER, TOKEN_TYPES.KEY, TOKEN_TYPES.CLOSE_AT, TOKEN_TYPES.VALUE, TOKEN_TYPES.ESCAPE, TOKEN_TYPES.QUOTE, TOKEN_TYPES.PREFIX_JS, TOKEN_TYPES.PREFIX_P, TOKEN_TYPES.IMPORT, TOKEN_TYPES.USE_MODULE, TOKEN_TYPES.END_KEYWORD, TOKEN_TYPES.TEXT];
+				if (allowed.includes(last_non_junk_type)) {
+					addToken(TOKEN_TYPES.COLON, ":");
+					isInHeader = true;
+				} else {
+					addToken(TOKEN_TYPES.TEXT, ":");
+				}
+			}
+			i++;
+			continue;
+		}
+		if (char === "=") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, "=");
+			} else {
+				const allowed = [TOKEN_TYPES.IDENTIFIER, TOKEN_TYPES.KEY, TOKEN_TYPES.ESCAPE, TOKEN_TYPES.QUOTE, TOKEN_TYPES.PREFIX_JS, TOKEN_TYPES.PREFIX_P, TOKEN_TYPES.IMPORT, TOKEN_TYPES.USE_MODULE, TOKEN_TYPES.END_KEYWORD, TOKEN_TYPES.TEXT];
+				if (allowed.includes(last_non_junk_type)) {
+					addToken(TOKEN_TYPES.EQUAL, "=");
+				} else {
+					addToken(TOKEN_TYPES.TEXT, "=");
+				}
+			}
+			i++;
+			continue;
+		}
+		if (char === ",") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, ",");
+			} else {
+				const allowed = [TOKEN_TYPES.VALUE, TOKEN_TYPES.IDENTIFIER, TOKEN_TYPES.QUOTE, TOKEN_TYPES.ESCAPE, TOKEN_TYPES.PREFIX_JS, TOKEN_TYPES.PREFIX_P, TOKEN_TYPES.IMPORT, TOKEN_TYPES.USE_MODULE, TOKEN_TYPES.END_KEYWORD, TOKEN_TYPES.TEXT];
+				if (allowed.includes(last_non_junk_type)) {
+					addToken(TOKEN_TYPES.COMMA, ",");
+				} else {
+					addToken(TOKEN_TYPES.TEXT, ",");
+				}
+			}
+			i++;
+			continue;
+		}
+		if (char === ";") {
+			if (isInAtBlockBody || (parenDepth > 0 && !isInInlineHead)) {
+				addToken(TOKEN_TYPES.TEXT, ";");
+			} else {
+				const allowed = [TOKEN_TYPES.IDENTIFIER, TOKEN_TYPES.VALUE, TOKEN_TYPES.CLOSE_AT, TOKEN_TYPES.CLOSE_PAREN, TOKEN_TYPES.ESCAPE, TOKEN_TYPES.QUOTE, TOKEN_TYPES.PREFIX_JS, TOKEN_TYPES.PREFIX_P, TOKEN_TYPES.IMPORT, TOKEN_TYPES.USE_MODULE, TOKEN_TYPES.END_KEYWORD, TOKEN_TYPES.TEXT];
+				if (allowed.includes(last_non_junk_type)) {
+					addToken(TOKEN_TYPES.SEMICOLON, ";");
+					isInHeader = false; // Semicolon ends the At-Block header
+					// Trigger body mode for At-Blocks
+					if (delimiterStack.length > 0) {
+						const top = delimiterStack[delimiterStack.length - 1];
+						if (top === "@") {
+							isInAtBlockBody = true;
 						}
-					} else {
-						advance(context);
 					}
 				} else {
-					addToken(TOKEN_TYPES.TEXT, context);
+					addToken(TOKEN_TYPES.TEXT, ";");
+				}
+			}
+			i++;
+			continue;
+		}
+		if (char === "\"" || char === "'") {
+			const valTriggers = [TOKEN_TYPES.COLON, TOKEN_TYPES.EQUAL, TOKEN_TYPES.COMMA, TOKEN_TYPES.ESCAPE, TOKEN_TYPES.OPEN_BRACKET, TOKEN_TYPES.OPEN_AT];
+			const wasValueTrigger = valTriggers.includes(last_non_junk_type);
+			addToken(TOKEN_TYPES.QUOTE, char);
+			i++;
+			// Enable quote mode
+			// NOTE: We allow quotes basically anywhere in headers as values/keys
+			if (isInHeader || wasValueTrigger) {
+				isInQuote = true;
+			}
+			continue;
+		}
+
+		// --- PHASE 4: WORD / TEXT SCANNING ---
+		// This is the "Fallback" mode where we scan for identifiers, keys, or values.
+		// It uses lookahead and context variables to guess the role of a word.
+		let word = "";
+		// Only Blocks ([ ]) allow ':' in their main identifier.
+		// At-Blocks (@_) and Inlines (->( )) do NOT allow ':' in the ID.
+		const isStartOfBlockId = (last_non_junk_type === TOKEN_TYPES.OPEN_BRACKET);
+
+		let stopChars = "[](){}:=;,@_>\"'#\\ \t\n\r";
+		if (isStartOfBlockId || (parenDepth > 0 && !isInInlineHead)) {
+			stopChars = stopChars.replace(":", "");
+		}
+		if (!isInHeader && !isInInlineHead) {
+			stopChars = "[]@_()\\#\n\r"; // In normal text, stop at markers, comments and newlines
+		}
+
+		while (i < src.length && !stopChars.includes(src[i])) { 
+			// Lookahead for -> marker in normal text
+			if (!isInHeader && src[i] === "-" && src[i+1] === ">") break;
+
+			// Stop if we hit an ALLOWED prefix trigger
+			if ((src[i] === "p" && src[i+1] === "{")) {
+				const top = (delimiterStack.length > 0) ? delimiterStack[delimiterStack.length - 1] : null;
+				const isInBlockHeader = isInHeader && top === "[";
+				const isInNormalText = !isInHeader && !isInInlineHead && !isInAtBlockBody && parenDepth === 0;
+				if (isInBlockHeader || isInNormalText) break;
+			}
+			if (src[i] === "j" && src[i+1] === "s" && src[i+2] === "{") {
+				const top = (delimiterStack.length > 0) ? delimiterStack[delimiterStack.length - 1] : null;
+				if (isInHeader && top === "[") break;
+			}
+			word += src[i];
+			i++;
+		}
+
+		if (word.length > 0) {
+			// Guess role based on context
+			if (parenDepth > 0 && !isInInlineHead) {
+				// Inside Inline Content (raw text)
+				addToken(TOKEN_TYPES.TEXT, word);
+			} else if (isInHeader || isInInlineHead) {
+				// Inside a structural header context
+				const isMainIdentifier = (
+					last_non_junk_type === TOKEN_TYPES.OPEN_BRACKET || 
+					last_non_junk_type === TOKEN_TYPES.OPEN_AT ||
+					(last_non_junk_type === TOKEN_TYPES.OPEN_PAREN && isInInlineHead)
+				);
+
+				if (isMainIdentifier) {
+					if (word === end_keyword) {
+						addToken(TOKEN_TYPES.END_KEYWORD, word);
+						if (delimiterStack[delimiterStack.length - 1] === "[") delimiterStack.pop();
+					}
+					else if (word === "import") addToken(TOKEN_TYPES.IMPORT, word);
+					else if (word === "$use-module") addToken(TOKEN_TYPES.USE_MODULE, word);
+					else addToken(TOKEN_TYPES.IDENTIFIER, word);
+				} else {
+					// Use lookahead to distinguish KEY from VALUE
+					const p = peekStructural(i);
+					if (p === ":") {
+						addToken(TOKEN_TYPES.KEY, word);
+					} else {
+						addToken(TOKEN_TYPES.VALUE, word);
+					}
 				}
-				i += context.length - 1;
 			} else {
-				addToken(TOKEN_TYPES.TEXT, char);
+				// Normal text
+				addToken(TOKEN_TYPES.TEXT, word);
+			}
+		} else {
+			// Fallback for any unhandled characters
+			if (i < src.length) {
+				addToken(TOKEN_TYPES.TEXT, src[i]);
+				i++;
 			}
 		}
 	}
-	// ========================================================================== //
-	//  Finalize with End-of-File token                                          //
-	// ========================================================================== //
+
 	addToken(TOKEN_TYPES.EOF, "");
 	return tokens;
 }