xmlui 0.4.0

package/dist/scripts/src/parsers/ueml/UemlLexer.js

@@ -0,0 +1,868 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UemlLexer = exports.SCRIPT_LITERAL_END = exports.SCRIPT_LITERAL_START = exports.HARD_LITERAL_END = exports.HARD_LITERAL_START = void 0;
+const UemlToken_1 = require("./UemlToken");
+// States of the lexer's state machine
+var LexerPhase;
+(function (LexerPhase) {
+    LexerPhase[LexerPhase["Start"] = 0] = "Start";
+    LexerPhase[LexerPhase["InWhiteSpace"] = 1] = "InWhiteSpace";
+    LexerPhase[LexerPhase["LeftAngle"] = 2] = "LeftAngle";
+    LexerPhase[LexerPhase["LeftAngleExcl"] = 3] = "LeftAngleExcl";
+    LexerPhase[LexerPhase["LeftAngleExclDash"] = 4] = "LeftAngleExclDash";
+    LexerPhase[LexerPhase["Comment"] = 5] = "Comment";
+    LexerPhase[LexerPhase["CommentDash"] = 6] = "CommentDash";
+    LexerPhase[LexerPhase["CommentTwoDash"] = 7] = "CommentTwoDash";
+    LexerPhase[LexerPhase["SingleWord"] = 8] = "SingleWord";
+    LexerPhase[LexerPhase["CloseStart"] = 9] = "CloseStart";
+    LexerPhase[LexerPhase["HardLiteral"] = 10] = "HardLiteral";
+    LexerPhase[LexerPhase["ScriptLiteral"] = 11] = "ScriptLiteral";
+    LexerPhase[LexerPhase["InNestedWhitespace"] = 12] = "InNestedWhitespace";
+    LexerPhase[LexerPhase["NestedText"] = 13] = "NestedText";
+    // --- String processing phases
+    LexerPhase[LexerPhase["String"] = 14] = "String";
+    LexerPhase[LexerPhase["StringBackSlash"] = 15] = "StringBackSlash";
+    LexerPhase[LexerPhase["StringHexa1"] = 16] = "StringHexa1";
+    LexerPhase[LexerPhase["StringHexa2"] = 17] = "StringHexa2";
+    LexerPhase[LexerPhase["StringUHexa1"] = 18] = "StringUHexa1";
+    LexerPhase[LexerPhase["StringUHexa2"] = 19] = "StringUHexa2";
+    LexerPhase[LexerPhase["StringUHexa3"] = 20] = "StringUHexa3";
+    LexerPhase[LexerPhase["StringUHexa4"] = 21] = "StringUHexa4";
+    LexerPhase[LexerPhase["StringUcp1"] = 22] = "StringUcp1";
+    LexerPhase[LexerPhase["StringUcp2"] = 23] = "StringUcp2";
+    LexerPhase[LexerPhase["StringUcp3"] = 24] = "StringUcp3";
+    LexerPhase[LexerPhase["StringUcp4"] = 25] = "StringUcp4";
+    LexerPhase[LexerPhase["StringUcp5"] = 26] = "StringUcp5";
+    LexerPhase[LexerPhase["StringUcp6"] = 27] = "StringUcp6";
+    LexerPhase[LexerPhase["StringUcpTail"] = 28] = "StringUcpTail";
+    LexerPhase[LexerPhase["BackSlash"] = 29] = "BackSlash";
+    LexerPhase[LexerPhase["Ampersand"] = 30] = "Ampersand";
+})(LexerPhase || (LexerPhase = {}));
+// --- Regex used to recognize UEML identifiers
+const uemlIdRegex = /^[A-Z_][$0-9A-Z_.\-]*$/i;
+exports.HARD_LITERAL_START = "<![CDATA[";
+exports.HARD_LITERAL_END = "]]>";
+exports.SCRIPT_LITERAL_START = "<script>";
+exports.SCRIPT_LITERAL_END = "</script>";
+/**
+ * This class implements the lexer of binding expressions
+ */
+class UemlLexer {
+    /**
+     * Initializes the tokenizer with the input stream
+     * @param input Input source code stream
+     */
+    constructor(input) {
+        this.input = input;
+        // --- Already fetched tokens
+        this._ahead = [];
+        // --- Prefetched character (from the next token)
+        this._prefetched = null;
+        // --- Prefetched character position (from the next token)
+        this._prefetchedPos = null;
+        // --- Prefetched character column (from the next token)
+        this._prefetchedColumn = null;
+    }
+    /**
+     * Fetches the next token without advancing to its position
+     * @param ws If true, retrieve whitespaces too
+     */
+    peek(ws = false) {
+        return this.ahead(0, ws);
+    }
+    /**
+     * Reads tokens ahead
+     * @param n Number of token positions to read ahead
+     * @param ws If true, retrieve whitespaces too
+     */
+    ahead(n = 1, ws = false) {
+        if (n > 16) {
+            throw new Error("Cannot look ahead more than 16 tokens");
+        }
+        // --- Prefetch missing tokens
+        while (this._ahead.length <= n) {
+            const token = this.fetch();
+            if (isEof(token)) {
+                return token;
+            }
+            if (ws || (!ws && !isWs(token))) {
+                this._ahead.push(token);
+            }
+        }
+        return this._ahead[n];
+    }
+    /**
+     * Fetches the next token and advances the stream position
+     * @param ws If true, retrieve whitespaces too
+     @param searchForNested Search for a nested element token   */
+    get(ws = false, searchForNested = false) {
+        if (this._ahead.length > 0) {
+            const token = this._ahead.shift();
+            if (!token) {
+                throw new Error("Token expected");
+            }
+            return token;
+        }
+        while (true) {
+            const token = this.fetch(searchForNested);
+            if (isEof(token) || ws || (!ws && !isWs(token))) {
+                return token;
+            }
+        }
+    }
+    /**
+     * Fetches the next child element token without advancing to its position
+     */
+    peekNested() {
+        if (!this._ahead.length) {
+            const token = this.fetch(true);
+            if (isEof(token)) {
+                return token;
+            }
+            this._ahead.push(token);
+        }
+        return this._ahead[0];
+    }
+    /**
+     * Fetches the next token from the input stream
+     * @param searchForNested Search for a nested element token
+     */
+    fetch(searchForNested = false) {
+        var _a;
+        // --- Captured constants used in nested functions
+        const lexer = this;
+        const input = this.input;
+        const startPos = this._prefetchedPos || input.position;
+        const line = input.line;
+        const startColumn = this._prefetchedColumn || input.column;
+        // --- State variables
+        let stringState = null;
+        let text = "";
+        let tokenType = UemlToken_1.UemlTokenType.Eof;
+        let lastEndPos = input.position;
+        let lastEndColumn = input.column;
+        let ch = null;
+        let skipTextAppend = false;
+        // --- Start from the beginning
+        let phase = LexerPhase.Start;
+        // --- Process all token characters
+        while (true) {
+            // --- Get the next character
+            ch = fetchNextChar();
+            skipTextAppend = false;
+            // --- In case of EOF, return the current token data
+            if (ch === null) {
+                return makeToken();
+            }
+            // --- Set the initial token type to unknown for the other characters
+            if (tokenType === UemlToken_1.UemlTokenType.Eof) {
+                tokenType = UemlToken_1.UemlTokenType.Unknown;
+            }
+            // --- Follow the lexer state machine
+            switch (phase) {
+                // ====================================================================
+                // Process the first character
+                case LexerPhase.Start:
+                    switch (ch) {
+                        // --- Go on with whitespaces
+                        case " ":
+                        case "\t":
+                        case "\n":
+                        case "\r":
+                            if (searchForNested) {
+                                phase = searchForNested ? LexerPhase.InNestedWhitespace : LexerPhase.InWhiteSpace;
+                                tokenType = UemlToken_1.UemlTokenType.NestedText;
+                            }
+                            else {
+                                phase = LexerPhase.InWhiteSpace;
+                                tokenType = UemlToken_1.UemlTokenType.Ws;
+                            }
+                            break;
+                        case "<":
+                            phase = handleHardLiteralCase();
+                            if (phase === LexerPhase.LeftAngle) {
+                                phase = handleScriptLiteralCase();
+                            }
+                            break;
+                        case ">":
+                            return completeToken(UemlToken_1.UemlTokenType.NodeEnd);
+                        case "/":
+                            if (searchForNested) {
+                                phase = LexerPhase.NestedText;
+                                tokenType = UemlToken_1.UemlTokenType.NestedText;
+                                break;
+                            }
+                            phase = LexerPhase.CloseStart;
+                            break;
+                        case "=":
+                            if (searchForNested) {
+                                phase = LexerPhase.NestedText;
+                                tokenType = UemlToken_1.UemlTokenType.NestedText;
+                                break;
+                            }
+                            return completeToken(UemlToken_1.UemlTokenType.Equal);
+                        case ":":
+                            if (searchForNested) {
+                                phase = LexerPhase.NestedText;
+                                tokenType = UemlToken_1.UemlTokenType.NestedText;
+                                break;
+                            }
+                            return completeToken(UemlToken_1.UemlTokenType.Colon);
+                        case "'":
+                        case '"':
+                        case "`":
+                            phase = LexerPhase.String;
+                            stringState = ch;
+                            break;
+                        case "&":
+                            if (searchForNested) {
+                                phase = LexerPhase.Ampersand;
+                                tokenType = UemlToken_1.UemlTokenType.NestedText;
+                                break;
+                            }
+                            return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                        case "\\":
+                            if (searchForNested) {
+                                phase = LexerPhase.BackSlash;
+                                tokenType = UemlToken_1.UemlTokenType.NestedText;
+                                break;
+                            }
+                            return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                        default:
+                            if (isSingleWordChar(ch)) {
+                                if (searchForNested) {
+                                    phase = LexerPhase.NestedText;
+                                    tokenType = UemlToken_1.UemlTokenType.NestedText;
+                                }
+                                else {
+                                    phase = LexerPhase.SingleWord;
+                                    tokenType = UemlToken_1.UemlTokenType.SingleWord;
+                                }
+                                break;
+                            }
+                            if (searchForNested) {
+                                phase = LexerPhase.NestedText;
+                                tokenType = UemlToken_1.UemlTokenType.NestedText;
+                                break;
+                            }
+                            return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // ====================================================================
+                // Whitespace processing
+                case LexerPhase.InWhiteSpace:
+                    if (ch !== " " && ch !== "\t" && ch !== "\r" && ch !== "\n") {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.InNestedWhitespace:
+                    if (ch === " " || ch === "\t" || ch === "\r" || ch === "\n") {
+                        break;
+                    }
+                    if (ch === "<") {
+                        phase = handleHardLiteralCase();
+                        if (phase === LexerPhase.LeftAngle) {
+                            phase = handleScriptLiteralCase();
+                        }
+                        break;
+                    }
+                    if (ch === "'" || ch === '"' || ch === "`") {
+                        phase = LexerPhase.String;
+                        stringState = ch;
+                        break;
+                    }
+                    // --- This is the start of a nested text
+                    phase = LexerPhase.NestedText;
+                    tokenType = UemlToken_1.UemlTokenType.NestedText;
+                    break;
+                // ====================================================================
+                // Hard literal processing
+                case LexerPhase.HardLiteral:
+                    if (ch === exports.HARD_LITERAL_END[0]) {
+                        // --- Recognize if hard literal end found
+                        let isHardLiteral = true;
+                        for (let i = 1; i < exports.HARD_LITERAL_END.length; i++) {
+                            isHardLiteral = isHardLiteral && input.ahead(i - 1) === exports.HARD_LITERAL_END[i];
+                            if (!isHardLiteral)
+                                break;
+                        }
+                        if (isHardLiteral) {
+                            text += exports.HARD_LITERAL_END;
+                            skipTextAppend = true;
+                            for (let i = 1; i < exports.HARD_LITERAL_END.length; i++) {
+                                input.get();
+                            }
+                            return completeToken(UemlToken_1.UemlTokenType.HardLiteral);
+                        }
+                        break;
+                    }
+                    // --- We're still within a hard literal
+                    break;
+                // ====================================================================
+                // Hard literal processing
+                case LexerPhase.ScriptLiteral:
+                    if (ch === exports.SCRIPT_LITERAL_END[0]) {
+                        // --- Recognize if hard literal end found
+                        let isScriptLiteral = true;
+                        for (let i = 1; i < exports.SCRIPT_LITERAL_END.length; i++) {
+                            isScriptLiteral = isScriptLiteral && input.ahead(i - 1) === exports.SCRIPT_LITERAL_END[i];
+                            if (!isScriptLiteral)
+                                break;
+                        }
+                        if (isScriptLiteral) {
+                            text += exports.SCRIPT_LITERAL_END;
+                            skipTextAppend = true;
+                            for (let i = 1; i < exports.SCRIPT_LITERAL_END.length; i++) {
+                                input.get();
+                            }
+                            return completeToken(UemlToken_1.UemlTokenType.ScriptLiteral);
+                        }
+                        break;
+                    }
+                    // --- We're still within a script literal
+                    break;
+                // ====================================================================
+                // Nested text processing
+                case LexerPhase.BackSlash:
+                    if (ch === "&") {
+                        // --- Remove the last "\"
+                        text = text.substring(0, text.length - 1);
+                    }
+                    phase = LexerPhase.NestedText;
+                    tokenType = UemlToken_1.UemlTokenType.NestedText;
+                    break;
+                case LexerPhase.Ampersand:
+                    const tail = input.getTail(input.position - 1);
+                    if (tail.startsWith("amp;")) {
+                        text = text.substring(0, text.length - 1);
+                        ch = "&";
+                        input.get(); // m
+                        input.get(); // p
+                        input.get(); // ;
+                    }
+                    else if (tail.startsWith("gt;")) {
+                        text = text.substring(0, text.length - 1);
+                        ch = ">";
+                        input.get(); // t
+                        input.get(); // ;
+                    }
+                    else if (tail.startsWith("lt;")) {
+                        text = text.substring(0, text.length - 1);
+                        ch = "<";
+                        input.get(); // t
+                        input.get(); // ;
+                    }
+                    else if (tail.startsWith("apos;")) {
+                        text = text.substring(0, text.length - 1);
+                        ch = "'";
+                        input.get(); // p
+                        input.get(); // o
+                        input.get(); // s
+                        input.get(); // ;
+                    }
+                    else if (tail.startsWith("quot;")) {
+                        text = text.substring(0, text.length - 1);
+                        ch = '"';
+                        input.get(); // u
+                        input.get(); // o
+                        input.get(); // t
+                        input.get(); // ;
+                    }
+                    phase = LexerPhase.NestedText;
+                    break;
+                case LexerPhase.NestedText:
+                    if (ch === "\\") {
+                        phase = LexerPhase.BackSlash;
+                    }
+                    else if (ch === "&") {
+                        phase = LexerPhase.Ampersand;
+                    }
+                    else if (ch === "<") {
+                        return makeToken();
+                    }
+                    // --- We're still within a nested text
+                    break;
+                // ====================================================================
+                // Opening and closing tags
+                case LexerPhase.LeftAngle:
+                    if (ch === "!") {
+                        phase = LexerPhase.LeftAngleExcl;
+                        tokenType = UemlToken_1.UemlTokenType.Unknown;
+                    }
+                    else if (ch === "/") {
+                        return completeToken(UemlToken_1.UemlTokenType.CloseNodeStart);
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                // --- Testing for closing tags
+                case LexerPhase.CloseStart:
+                    return ch === ">" ? completeToken(UemlToken_1.UemlTokenType.NodeClose) : makeToken();
+                // ====================================================================
+                // Collecting "single word" characters
+                case LexerPhase.SingleWord:
+                    if (!isSingleWordChar(ch)) {
+                        return makeToken();
+                    }
+                    break;
+                // ====================================================================
+                // Comment processing
+                case LexerPhase.LeftAngleExcl:
+                    if (ch === "-") {
+                        phase = LexerPhase.LeftAngleExclDash;
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.LeftAngleExclDash:
+                    if (ch === "-") {
+                        phase = LexerPhase.Comment;
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.Comment:
+                    if (ch === "-") {
+                        phase = LexerPhase.CommentDash;
+                    }
+                    break;
+                case LexerPhase.CommentDash:
+                    phase = ch === "-" ? LexerPhase.CommentTwoDash : LexerPhase.Comment;
+                    break;
+                case LexerPhase.CommentTwoDash:
+                    if (ch === ">") {
+                        return completeToken(UemlToken_1.UemlTokenType.Comment);
+                    }
+                    else {
+                        phase = LexerPhase.Comment;
+                    }
+                    break;
+                // ====================================================================
+                // String processing
+                case LexerPhase.String:
+                    if (ch === stringState) {
+                        if (searchForNested) {
+                            const tail = (_a = input.getTail(input.position)) === null || _a === void 0 ? void 0 : _a.trim();
+                            if ((tail === null || tail === void 0 ? void 0 : tail[0]) === "<") {
+                                return completeToken(UemlToken_1.UemlTokenType.StringLiteral);
+                            }
+                            phase = LexerPhase.NestedText;
+                            tokenType = UemlToken_1.UemlTokenType.NestedText;
+                            break;
+                        }
+                        else {
+                            return completeToken(UemlToken_1.UemlTokenType.StringLiteral);
+                        }
+                    }
+                    else if (ch === "\\") {
+                        const tail = input.getTail(input.position);
+                        if (tail.startsWith("&amp;")) {
+                            text += "&amp";
+                            ch = ";";
+                            input.get(); // &
+                            input.get(); // a
+                            input.get(); // m
+                            input.get(); // p
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&gt;")) {
+                            text += "&gt";
+                            ch = ";";
+                            input.get(); // &
+                            input.get(); // g
+                            input.get(); // t
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&lt;")) {
+                            text += "&lt";
+                            ch = ";";
+                            input.get(); // &
+                            input.get(); // l
+                            input.get(); // t
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&apos;")) {
+                            text += "&apos";
+                            ch = ";";
+                            input.get(); // &
+                            input.get(); // a
+                            input.get(); // p
+                            input.get(); // o
+                            input.get(); // s
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&quot;")) {
+                            text += "&quot";
+                            ch = ";";
+                            input.get(); // &
+                            input.get(); // q
+                            input.get(); // u
+                            input.get(); // o
+                            input.get(); // t
+                            input.get(); // ;
+                        }
+                        else {
+                            phase = LexerPhase.StringBackSlash;
+                            tokenType = UemlToken_1.UemlTokenType.Unknown;
+                        }
+                    }
+                    else {
+                        const tail = input.getTail(input.position - 1);
+                        if (tail.startsWith("&amp;")) {
+                            ch = "&";
+                            input.get(); // a
+                            input.get(); // m
+                            input.get(); // p
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&gt;")) {
+                            ch = ">";
+                            input.get(); // g
+                            input.get(); // t
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&lt;")) {
+                            ch = "<";
+                            input.get(); // l
+                            input.get(); // t
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&apos;")) {
+                            ch = "'";
+                            input.get(); // a
+                            input.get(); // p
+                            input.get(); // o
+                            input.get(); // s
+                            input.get(); // ;
+                        }
+                        else if (tail.startsWith("&quot;")) {
+                            ch = '"';
+                            input.get(); // q
+                            input.get(); // u
+                            input.get(); // o
+                            input.get(); // t
+                            input.get(); // ;
+                        }
+                    }
+                    break;
+                // Start of string character escape
+                case LexerPhase.StringBackSlash:
+                    switch (ch) {
+                        case "b":
+                        case "f":
+                        case "n":
+                        case "r":
+                        case "t":
+                        case "v":
+                        case "S":
+                        case "0":
+                        case "'":
+                        case '"':
+                        case "`":
+                        case "\\":
+                            phase = LexerPhase.String;
+                            break;
+                        case "x":
+                            phase = LexerPhase.StringHexa1;
+                            break;
+                        case "u":
+                            phase = LexerPhase.StringUHexa1;
+                            break;
+                        default:
+                            phase = LexerPhase.String;
+                            break;
+                    }
+                    break;
+                // --- First hexadecimal digit of string character escape
+                case LexerPhase.StringHexa1:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringHexa2;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Second hexadecimal digit of character escape
+                case LexerPhase.StringHexa2:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.String;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- First hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa1:
+                    if (ch === "{") {
+                        phase = LexerPhase.StringUcp1;
+                        break;
+                    }
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUHexa2;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Second hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa2:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUHexa3;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Third hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa3:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUHexa4;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Fourth hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa4:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.String;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- First hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp1:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp2;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Second hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp2:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp3;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Third hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp3:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp4;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Fourth hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp4:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp5;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Fifth hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp5:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp6;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Sixth hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp6:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcpTail;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+                // --- Closing bracket of Unicode codepoint string character escape
+                case LexerPhase.StringUcpTail:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else {
+                        return completeToken(UemlToken_1.UemlTokenType.Unknown);
+                    }
+                    break;
+            }
+            // --- Append the char to the current text
+            appendTokenChar();
+            // --- Go on with parsing the next character
+        }
+        // --- Handle hard literal checking
+        function handleHardLiteralCase() {
+            let isHardLiteral = true;
+            for (let i = 1; i < exports.HARD_LITERAL_START.length; i++) {
+                const ahead = input.ahead(i - 1);
+                isHardLiteral = isHardLiteral && ahead === exports.HARD_LITERAL_START[i];
+                if (!isHardLiteral)
+                    break;
+            }
+            if (isHardLiteral) {
+                text += exports.HARD_LITERAL_START;
+                skipTextAppend = true;
+                for (let i = 1; i < exports.HARD_LITERAL_START.length; i++) {
+                    input.get();
+                }
+                return LexerPhase.HardLiteral;
+            }
+            tokenType = UemlToken_1.UemlTokenType.OpenNodeStart;
+            return LexerPhase.LeftAngle;
+        }
+        // --- Handle hard literal checking
+        function handleScriptLiteralCase() {
+            let isScriptLiteral = true;
+            for (let i = 1; i < exports.SCRIPT_LITERAL_START.length; i++) {
+                const ahead = input.ahead(i - 1);
+                isScriptLiteral = isScriptLiteral && ahead === exports.SCRIPT_LITERAL_START[i];
+                if (!isScriptLiteral)
+                    break;
+            }
+            if (isScriptLiteral) {
+                text += exports.SCRIPT_LITERAL_START;
+                skipTextAppend = true;
+                for (let i = 1; i < exports.SCRIPT_LITERAL_START.length; i++) {
+                    input.get();
+                }
+                return LexerPhase.ScriptLiteral;
+            }
+            tokenType = UemlToken_1.UemlTokenType.OpenNodeStart;
+            return LexerPhase.LeftAngle;
+        }
+        /**
+         * Appends the last character to the token, and manages positions
+         */
+        function appendTokenChar() {
+            if (!skipTextAppend) {
+                text += ch;
+            }
+            lexer._prefetched = null;
+            lexer._prefetchedPos = null;
+            lexer._prefetchedColumn = null;
+            lastEndPos = input.position;
+            lastEndColumn = input.position;
+        }
+        /**
+         * Fetches the next character from the input stream
+         */
+        function fetchNextChar() {
+            if (!lexer._prefetched) {
+                lexer._prefetchedPos = input.position;
+                lexer._prefetched = input.get();
+            }
+            return lexer._prefetched;
+        }
+        /**
+         * Packs the specified type of token to send back
+         */
+        function makeToken() {
+            if (tokenType === UemlToken_1.UemlTokenType.SingleWord) {
+                // --- A single word may be an identifier
+                if (uemlIdRegex.test(text)) {
+                    tokenType = UemlToken_1.UemlTokenType.Identifier;
+                }
+                else {
+                    tokenType = UemlToken_1.UemlTokenType.Unknown;
+                }
+            }
+            return {
+                text,
+                type: tokenType,
+                location: {
+                    startPosition: startPos,
+                    endPosition: lastEndPos,
+                    startLine: line,
+                    endLine: line,
+                    startColumn,
+                    endColumn: lastEndColumn,
+                },
+            };
+        }
+        /**
+         * Add the last character to the token and return it
+         */
+        function completeToken(suggestedType) {
+            appendTokenChar();
+            // --- Send back the token
+            if (suggestedType !== undefined) {
+                tokenType = suggestedType;
+            }
+            return makeToken();
+        }
+    }
+}
+exports.UemlLexer = UemlLexer;
+/**
+ * Tests if a token id EOF
+ * @param t Token instance
+ */
+function isEof(t) {
+    return t.type === UemlToken_1.UemlTokenType.Eof;
+}
+/**
+ * Tests if a token is whitespace
+ * @param t Token instance
+ */
+function isWs(t) {
+    return t.type <= UemlToken_1.UemlTokenType.Ws;
+}
+/**
+ * Tests if a character is an identifier start character
+ * @param ch Character to test
+ */
+function isSingleWordChar(ch) {
+    return ((ch >= "a" && ch <= "z") ||
+        (ch >= "A" && ch <= "Z") ||
+        (ch >= "0" && ch <= "9") ||
+        ch === "$" ||
+        ch === "#" ||
+        ch === "@" ||
+        ch === "_" ||
+        ch === "." ||
+        ch === "," ||
+        ch === "%" ||
+        ch === "(" ||
+        ch === ")" ||
+        ch === "{" ||
+        ch === "}" ||
+        ch === "[" ||
+        ch === "]" ||
+        ch === "+" ||
+        ch === "*" ||
+        ch === "-");
+}
+/**
+ * Tests if a character is a hexadecimal digit
+ * @param ch Character to test
+ */
+function isHexadecimalDigit(ch) {
+    return (ch >= "0" && ch <= "9") || (ch >= "A" && ch <= "F") || (ch >= "a" && ch <= "f");
+}