xmlui 0.4.0

package/dist/scripts/src/parsers/scripting/Lexer.js

@@ -0,0 +1,1028 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lexer = void 0;
+const regexpp_1 = require("@eslint-community/regexpp");
+const Token_1 = require("../../abstractions/scripting/Token");
+/**
+ * This enum indicates the current lexer phase
+ */
+var LexerPhase;
+(function (LexerPhase) {
+    // Start getting a token
+    LexerPhase[LexerPhase["Start"] = 0] = "Start";
+    // Collecting whitespace
+    LexerPhase[LexerPhase["InWhiteSpace"] = 1] = "InWhiteSpace";
+    // Comment phases
+    LexerPhase[LexerPhase["InlineCommentTrail"] = 2] = "InlineCommentTrail";
+    LexerPhase[LexerPhase["BlockCommentTrail1"] = 3] = "BlockCommentTrail1";
+    LexerPhase[LexerPhase["BlockCommentTrail2"] = 4] = "BlockCommentTrail2";
+    // Multi-char tokens
+    LexerPhase[LexerPhase["Slash"] = 5] = "Slash";
+    LexerPhase[LexerPhase["Or"] = 6] = "Or";
+    LexerPhase[LexerPhase["Asterisk"] = 7] = "Asterisk";
+    LexerPhase[LexerPhase["Ampersand"] = 8] = "Ampersand";
+    LexerPhase[LexerPhase["Equal"] = 9] = "Equal";
+    LexerPhase[LexerPhase["DoubleEqual"] = 10] = "DoubleEqual";
+    LexerPhase[LexerPhase["NotEqual"] = 11] = "NotEqual";
+    LexerPhase[LexerPhase["Exclamation"] = 12] = "Exclamation";
+    LexerPhase[LexerPhase["AngleLeft"] = 13] = "AngleLeft";
+    LexerPhase[LexerPhase["AngleRight"] = 14] = "AngleRight";
+    LexerPhase[LexerPhase["SignedShiftRight"] = 15] = "SignedShiftRight";
+    LexerPhase[LexerPhase["IdTail"] = 16] = "IdTail";
+    LexerPhase[LexerPhase["Dot"] = 17] = "Dot";
+    LexerPhase[LexerPhase["DotDot"] = 18] = "DotDot";
+    LexerPhase[LexerPhase["Colon"] = 19] = "Colon";
+    LexerPhase[LexerPhase["Zero"] = 20] = "Zero";
+    LexerPhase[LexerPhase["QuestionMark"] = 21] = "QuestionMark";
+    LexerPhase[LexerPhase["HexaFirst"] = 22] = "HexaFirst";
+    LexerPhase[LexerPhase["HexaTail"] = 23] = "HexaTail";
+    LexerPhase[LexerPhase["BinaryFirst"] = 24] = "BinaryFirst";
+    LexerPhase[LexerPhase["BinaryTail"] = 25] = "BinaryTail";
+    LexerPhase[LexerPhase["DecimalOrReal"] = 26] = "DecimalOrReal";
+    LexerPhase[LexerPhase["RealFractionalFirst"] = 27] = "RealFractionalFirst";
+    LexerPhase[LexerPhase["RealFractionalTail"] = 28] = "RealFractionalTail";
+    LexerPhase[LexerPhase["RealExponent"] = 29] = "RealExponent";
+    LexerPhase[LexerPhase["RealExponentSign"] = 30] = "RealExponentSign";
+    LexerPhase[LexerPhase["RealExponentTail"] = 31] = "RealExponentTail";
+    LexerPhase[LexerPhase["String"] = 32] = "String";
+    LexerPhase[LexerPhase["StringBackSlash"] = 33] = "StringBackSlash";
+    LexerPhase[LexerPhase["StringHexa1"] = 34] = "StringHexa1";
+    LexerPhase[LexerPhase["StringHexa2"] = 35] = "StringHexa2";
+    LexerPhase[LexerPhase["StringUHexa1"] = 36] = "StringUHexa1";
+    LexerPhase[LexerPhase["StringUHexa2"] = 37] = "StringUHexa2";
+    LexerPhase[LexerPhase["StringUHexa3"] = 38] = "StringUHexa3";
+    LexerPhase[LexerPhase["StringUHexa4"] = 39] = "StringUHexa4";
+    LexerPhase[LexerPhase["StringUcp1"] = 40] = "StringUcp1";
+    LexerPhase[LexerPhase["StringUcp2"] = 41] = "StringUcp2";
+    LexerPhase[LexerPhase["StringUcp3"] = 42] = "StringUcp3";
+    LexerPhase[LexerPhase["StringUcp4"] = 43] = "StringUcp4";
+    LexerPhase[LexerPhase["StringUcp5"] = 44] = "StringUcp5";
+    LexerPhase[LexerPhase["StringUcp6"] = 45] = "StringUcp6";
+    LexerPhase[LexerPhase["StringUcpTail"] = 46] = "StringUcpTail";
+    // --- Assignments
+    LexerPhase[LexerPhase["Exponent"] = 47] = "Exponent";
+    LexerPhase[LexerPhase["Plus"] = 48] = "Plus";
+    LexerPhase[LexerPhase["Minus"] = 49] = "Minus";
+    LexerPhase[LexerPhase["Divide"] = 50] = "Divide";
+    LexerPhase[LexerPhase["Remainder"] = 51] = "Remainder";
+    LexerPhase[LexerPhase["ShiftLeft"] = 52] = "ShiftLeft";
+    LexerPhase[LexerPhase["ShiftRight"] = 53] = "ShiftRight";
+    LexerPhase[LexerPhase["LogicalAnd"] = 54] = "LogicalAnd";
+    LexerPhase[LexerPhase["BitwiseXor"] = 55] = "BitwiseXor";
+    LexerPhase[LexerPhase["LogicalOr"] = 56] = "LogicalOr";
+    LexerPhase[LexerPhase["NullCoalesce"] = 57] = "NullCoalesce";
+    // --- Other
+    LexerPhase[LexerPhase["RegEx"] = 58] = "RegEx";
+})(LexerPhase || (LexerPhase = {}));
+/**
+ * This class implements the lexer of binding expressions
+ */
+class Lexer {
+    /**
+     * Initializes the tokenizer with the input stream
+     * @param input Input source code stream
+     */
+    constructor(input) {
+        this.input = input;
+        // --- Already fetched tokens
+        this._ahead = [];
+        // --- Prefetched character (from the next token)
+        this._prefetched = null;
+        // --- Prefetched character position (from the next token)
+        this._prefetchedPos = null;
+        // --- Prefetched character column (from the next token)
+        this._prefetchedColumn = null;
+        // --- input position at the beginning of last fetch
+        this._lastFetchPosition = 0;
+    }
+    /**
+     * Fetches the next token without advancing to its position
+     * @param ws If true, retrieve whitespaces too
+     */
+    peek(ws = false) {
+        return this.ahead(0, ws);
+    }
+    /**
+     * Reads tokens ahead
+     * @param n Number of token positions to read ahead
+     * @param ws If true, retrieve whitespaces too
+     */
+    ahead(n = 1, ws = false) {
+        if (n > 16) {
+            throw new Error("Cannot look ahead more than 16 tokens");
+        }
+        // --- Prefetch missing tokens
+        while (this._ahead.length <= n) {
+            const token = this.fetch();
+            if (isEof(token)) {
+                return token;
+            }
+            if (ws || (!ws && !isWs(token))) {
+                this._ahead.push(token);
+            }
+        }
+        return this._ahead[n];
+    }
+    /**
+     * Fetches the next token and advances the stream position
+     * @param ws If true, retrieve whitespaces too
+     */
+    get(ws = false) {
+        if (this._ahead.length > 0) {
+            const token = this._ahead.shift();
+            if (!token) {
+                throw new Error("Token expected");
+            }
+            return token;
+        }
+        while (true) {
+            const token = this.fetch();
+            if (isEof(token) || ws || (!ws && !isWs(token))) {
+                return token;
+            }
+        }
+    }
+    /**
+     * Gets a RegEx from the current position
+     */
+    getRegEx() {
+        return this.fetchRegEx();
+    }
+    /**
+     * Gets the remaining characters after the parsing phase
+     */
+    getTail() {
+        return this._ahead.length > 0
+            ? this.input.getTail(this._ahead[0].location.startPosition)
+            : this.input.getTail(this._lastFetchPosition);
+    }
+    /**
+     * Fetches the next character from the input stream
+     */
+    fetchNextChar() {
+        if (!this._prefetched) {
+            this._prefetchedPos = this.input.position;
+            this._prefetchedColumn = this.input.column;
+            this._prefetched = this.input.get();
+        }
+        return this._prefetched;
+    }
+    /**
+     * Fetches the next token from the input stream
+     */
+    fetch() {
+        // --- Captured constants used in nested functions
+        const lexer = this;
+        const input = this.input;
+        const startPos = this._prefetchedPos || input.position;
+        const line = input.line;
+        const startColumn = this._prefetchedColumn || input.column;
+        this._lastFetchPosition = this.input.position;
+        // --- State variables
+        let stringState = null;
+        let text = "";
+        let tokenType = Token_1.TokenType.Eof;
+        let lastEndPos = input.position;
+        let lastEndColumn = input.column;
+        let ch = null;
+        let useResolver = false;
+        // --- Start from the beginning
+        let phase = LexerPhase.Start;
+        // --- Process all token characters
+        while (true) {
+            // --- Get the next character
+            ch = this.fetchNextChar();
+            // --- In case of EOF, return the current token data
+            if (ch === null) {
+                return makeToken();
+            }
+            // --- Set the initial token type to unknown for the other characters
+            if (tokenType === Token_1.TokenType.Eof) {
+                tokenType = Token_1.TokenType.Unknown;
+            }
+            // --- Follow the lexer state machine
+            switch (phase) {
+                // ====================================================================
+                // Process the first character
+                case LexerPhase.Start:
+                    switch (ch) {
+                        // --- Go on with whitespaces
+                        case " ":
+                        case "\t":
+                        case "\n":
+                        case "\r":
+                            phase = LexerPhase.InWhiteSpace;
+                            tokenType = Token_1.TokenType.Ws;
+                            break;
+                        // --- Divide, BlockComment, or EolComment
+                        case "/":
+                            phase = LexerPhase.Slash;
+                            tokenType = Token_1.TokenType.Divide;
+                            break;
+                        case "*":
+                            phase = LexerPhase.Asterisk;
+                            tokenType = Token_1.TokenType.Multiply;
+                            break;
+                        case "%":
+                            phase = LexerPhase.Remainder;
+                            tokenType = Token_1.TokenType.Remainder;
+                            break;
+                        case "+":
+                            phase = LexerPhase.Plus;
+                            tokenType = Token_1.TokenType.Plus;
+                            break;
+                        case "-":
+                            phase = LexerPhase.Minus;
+                            tokenType = Token_1.TokenType.Minus;
+                            break;
+                        case "^":
+                            phase = LexerPhase.BitwiseXor;
+                            tokenType = Token_1.TokenType.BitwiseXor;
+                            break;
+                        // --- BitwiseOr, LogicalOr
+                        case "|":
+                            phase = LexerPhase.Or;
+                            tokenType = Token_1.TokenType.BitwiseOr;
+                            break;
+                        // --- BitwiseAnd, LogicalAnd
+                        case "&":
+                            phase = LexerPhase.Ampersand;
+                            tokenType = Token_1.TokenType.BitwiseAnd;
+                            break;
+                        // --- "?", "?", or "?."
+                        case "?":
+                            phase = LexerPhase.QuestionMark;
+                            tokenType = Token_1.TokenType.QuestionMark;
+                            break;
+                        case ";":
+                            return completeToken(Token_1.TokenType.Semicolon);
+                        case ",":
+                            return completeToken(Token_1.TokenType.Comma);
+                        case "(":
+                            return completeToken(Token_1.TokenType.LParent);
+                        case ")":
+                            return completeToken(Token_1.TokenType.RParent);
+                        // --- ":" or "::"
+                        case ":":
+                            phase = LexerPhase.Colon;
+                            tokenType = Token_1.TokenType.Colon;
+                            break;
+                        case "[":
+                            return completeToken(Token_1.TokenType.LSquare);
+                        case "]":
+                            return completeToken(Token_1.TokenType.RSquare);
+                        case "~":
+                            return completeToken(Token_1.TokenType.BinaryNot);
+                        case "{":
+                            return completeToken(Token_1.TokenType.LBrace);
+                        case "}":
+                            return completeToken(Token_1.TokenType.RBrace);
+                        // --- "=","==", "===", or "=>"
+                        case "=":
+                            phase = LexerPhase.Equal;
+                            tokenType = Token_1.TokenType.Assignment;
+                            break;
+                        // --- "!", "!=", or "!=="
+                        case "!":
+                            phase = LexerPhase.Exclamation;
+                            tokenType = Token_1.TokenType.LogicalNot;
+                            break;
+                        // --- "<", "<=", or "<<"
+                        case "<":
+                            phase = LexerPhase.AngleLeft;
+                            tokenType = Token_1.TokenType.LessThan;
+                            break;
+                        // --- ">", ">=", ">>", or ">>>"
+                        case ">":
+                            phase = LexerPhase.AngleRight;
+                            tokenType = Token_1.TokenType.GreaterThan;
+                            break;
+                        // --- Decimal or Real literal
+                        case "0":
+                            phase = LexerPhase.Zero;
+                            tokenType = Token_1.TokenType.DecimalLiteral;
+                            break;
+                        case ".":
+                            phase = LexerPhase.Dot;
+                            tokenType = Token_1.TokenType.Dot;
+                            break;
+                        // --- String (both " and ' are accepted as string wrappers
+                        case '"':
+                        case "'":
+                            stringState = ch;
+                            phase = LexerPhase.String;
+                            break;
+                        default:
+                            if (isIdStart(ch)) {
+                                useResolver = true;
+                                phase = LexerPhase.IdTail;
+                                tokenType = Token_1.TokenType.Identifier;
+                            }
+                            else if (isDecimalDigit(ch)) {
+                                phase = LexerPhase.DecimalOrReal;
+                                tokenType = Token_1.TokenType.DecimalLiteral;
+                            }
+                            else {
+                                completeToken(Token_1.TokenType.Unknown);
+                            }
+                            break;
+                    }
+                    break;
+                // ====================================================================
+                // Process comments
+                // --- Looking for the end of whitespace
+                case LexerPhase.InWhiteSpace:
+                    if (ch !== " " && ch !== "\t" && ch !== "\r" && ch !== "\n") {
+                        return makeToken();
+                    }
+                    break;
+                // --- Wait for an "*" that may complete a block comment
+                case LexerPhase.BlockCommentTrail1:
+                    if (ch === "*") {
+                        phase = LexerPhase.BlockCommentTrail2;
+                    }
+                    break;
+                // --- Wait for a "/" that may complete a block comment
+                case LexerPhase.BlockCommentTrail2:
+                    if (ch === "/") {
+                        return completeToken(Token_1.TokenType.BlockComment);
+                    }
+                    break;
+                case LexerPhase.InlineCommentTrail:
+                    if (ch === "\n") {
+                        return completeToken();
+                    }
+                    break;
+                // ====================================================================
+                // Process identifiers
+                case LexerPhase.IdTail:
+                    if (!isIdContinuation(ch)) {
+                        return makeToken();
+                    }
+                    break;
+                // ====================================================================
+                // Process miscellaneous tokens
+                case LexerPhase.Colon:
+                    return ch === ":" ? completeToken(Token_1.TokenType.Global) : makeToken();
+                case LexerPhase.Slash:
+                    if (ch === "*") {
+                        phase = LexerPhase.BlockCommentTrail1;
+                    }
+                    else if (ch === "/") {
+                        phase = LexerPhase.InlineCommentTrail;
+                        tokenType = Token_1.TokenType.EolComment;
+                    }
+                    else if (ch === "=") {
+                        return completeToken(Token_1.TokenType.DivideAssignment);
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.Plus:
+                    if (ch === "+") {
+                        return completeToken(Token_1.TokenType.IncOp);
+                    }
+                    return ch === "=" ? completeToken(Token_1.TokenType.AddAssignment) : makeToken();
+                case LexerPhase.Minus:
+                    if (ch === "-") {
+                        return completeToken(Token_1.TokenType.DecOp);
+                    }
+                    return ch === "=" ? completeToken(Token_1.TokenType.SubtractAssignment) : makeToken();
+                case LexerPhase.Remainder:
+                    return ch === "=" ? completeToken(Token_1.TokenType.RemainderAssignment) : makeToken();
+                case LexerPhase.BitwiseXor:
+                    return ch === "=" ? completeToken(Token_1.TokenType.BitwiseXorAssignment) : makeToken();
+                case LexerPhase.Or:
+                    if (ch === "=") {
+                        return completeToken(Token_1.TokenType.BitwiseOrAssignment);
+                    }
+                    if (ch === "|") {
+                        phase = LexerPhase.LogicalOr;
+                        tokenType = Token_1.TokenType.LogicalOr;
+                        break;
+                    }
+                    return makeToken();
+                case LexerPhase.LogicalOr:
+                    return ch === "=" ? completeToken(Token_1.TokenType.LogicalOrAssignment) : makeToken();
+                case LexerPhase.Ampersand:
+                    if (ch === "=") {
+                        return completeToken(Token_1.TokenType.BitwiseAndAssignment);
+                    }
+                    if (ch === "&") {
+                        phase = LexerPhase.LogicalAnd;
+                        tokenType = Token_1.TokenType.LogicalAnd;
+                        break;
+                    }
+                    return makeToken();
+                case LexerPhase.LogicalAnd:
+                    return ch === "=" ? completeToken(Token_1.TokenType.LogicalAndAssignment) : makeToken();
+                case LexerPhase.Asterisk:
+                    if (ch === "*") {
+                        phase = LexerPhase.Exponent;
+                        tokenType = Token_1.TokenType.Exponent;
+                        break;
+                    }
+                    else if (ch === "=") {
+                        return completeToken(Token_1.TokenType.MultiplyAssignment);
+                    }
+                    return makeToken();
+                case LexerPhase.Exponent:
+                    return ch === "=" ? completeToken(Token_1.TokenType.ExponentAssignment) : makeToken();
+                case LexerPhase.QuestionMark:
+                    if (ch === "?") {
+                        phase = LexerPhase.NullCoalesce;
+                        tokenType = Token_1.TokenType.NullCoalesce;
+                        break;
+                    }
+                    if (ch === ".") {
+                        return completeToken(Token_1.TokenType.OptionalChaining);
+                    }
+                    return makeToken();
+                case LexerPhase.NullCoalesce:
+                    return ch === "=" ? completeToken(Token_1.TokenType.NullCoalesceAssignment) : makeToken();
+                case LexerPhase.Equal:
+                    if (ch === ">") {
+                        return completeToken(Token_1.TokenType.Arrow);
+                    }
+                    if (ch === "=") {
+                        phase = LexerPhase.DoubleEqual;
+                        tokenType = Token_1.TokenType.Equal;
+                        break;
+                    }
+                    return makeToken();
+                case LexerPhase.DoubleEqual:
+                    return ch === "=" ? completeToken(Token_1.TokenType.StrictEqual) : makeToken();
+                case LexerPhase.Exclamation:
+                    if (ch === "=") {
+                        phase = LexerPhase.NotEqual;
+                        tokenType = Token_1.TokenType.NotEqual;
+                        break;
+                    }
+                    return makeToken();
+                case LexerPhase.NotEqual:
+                    return ch === "=" ? completeToken(Token_1.TokenType.StrictNotEqual) : makeToken();
+                case LexerPhase.AngleLeft:
+                    if (ch === "=") {
+                        return completeToken(Token_1.TokenType.LessThanOrEqual);
+                    }
+                    if (ch === "<") {
+                        phase = LexerPhase.ShiftLeft;
+                        tokenType = Token_1.TokenType.ShiftLeft;
+                        break;
+                    }
+                    return makeToken();
+                case LexerPhase.ShiftLeft:
+                    return ch === "=" ? completeToken(Token_1.TokenType.ShiftLeftAssignment) : makeToken();
+                case LexerPhase.AngleRight:
+                    if (ch === "=") {
+                        return completeToken(Token_1.TokenType.GreaterThanOrEqual);
+                    }
+                    if (ch === ">") {
+                        phase = LexerPhase.SignedShiftRight;
+                        tokenType = Token_1.TokenType.SignedShiftRight;
+                        break;
+                    }
+                    return makeToken();
+                case LexerPhase.SignedShiftRight:
+                    if (ch === ">") {
+                        phase = LexerPhase.ShiftRight;
+                        tokenType = Token_1.TokenType.ShiftRight;
+                        break;
+                    }
+                    if (ch === "=") {
+                        return completeToken(Token_1.TokenType.SignedShiftRightAssignment);
+                    }
+                    return makeToken();
+                case LexerPhase.ShiftRight:
+                    return ch === "=" ? completeToken(Token_1.TokenType.ShiftRightAssignment) : makeToken();
+                // ====================================================================
+                // --- Literals
+                case LexerPhase.Zero:
+                    if (ch === "x") {
+                        phase = LexerPhase.HexaFirst;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    else if (ch === "b") {
+                        phase = LexerPhase.BinaryFirst;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    else if (isDecimalDigit(ch) || ch === "_") {
+                        phase = LexerPhase.DecimalOrReal;
+                    }
+                    else if (ch === ".") {
+                        phase = LexerPhase.RealFractionalFirst;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    else if (ch === "e" || ch === "E") {
+                        phase = LexerPhase.RealExponent;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.Dot:
+                    if (ch === ".") {
+                        phase = LexerPhase.DotDot;
+                        tokenType = Token_1.TokenType.Unknown;
+                        break;
+                    }
+                    if (!isDecimalDigit(ch)) {
+                        return makeToken();
+                    }
+                    phase = LexerPhase.RealFractionalTail;
+                    tokenType = Token_1.TokenType.RealLiteral;
+                    break;
+                case LexerPhase.DotDot:
+                    return ch === "." ? completeToken(Token_1.TokenType.Spread) : makeToken();
+                case LexerPhase.HexaFirst:
+                    if (ch === "_") {
+                        break;
+                    }
+                    if (!isHexadecimalDigit(ch)) {
+                        return makeToken();
+                    }
+                    phase = LexerPhase.HexaTail;
+                    tokenType = Token_1.TokenType.HexadecimalLiteral;
+                    break;
+                case LexerPhase.HexaTail:
+                    if (!isHexadecimalDigit(ch) && ch !== "_") {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.BinaryFirst:
+                    if (ch === "_") {
+                        break;
+                    }
+                    if (!isBinaryDigit(ch)) {
+                        return makeToken();
+                    }
+                    phase = LexerPhase.BinaryTail;
+                    tokenType = Token_1.TokenType.BinaryLiteral;
+                    break;
+                case LexerPhase.BinaryTail:
+                    if (!isBinaryDigit(ch) && ch !== "_") {
+                        return makeToken();
+                    }
+                    tokenType = Token_1.TokenType.BinaryLiteral;
+                    break;
+                case LexerPhase.DecimalOrReal:
+                    if (isDecimalDigit(ch) || ch === "_") {
+                        break;
+                    }
+                    else if (ch === "." && (this.input.peek() === null || isDecimalDigit(this.input.peek()))) {
+                        phase = LexerPhase.RealFractionalFirst;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    else if (ch === "e" || ch === "E") {
+                        phase = LexerPhase.RealExponent;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.RealFractionalFirst:
+                    if (isDecimalDigit(ch)) {
+                        phase = LexerPhase.RealFractionalTail;
+                        tokenType = Token_1.TokenType.RealLiteral;
+                    }
+                    else if (ch === "e" || ch === "E") {
+                        phase = LexerPhase.RealExponent;
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.RealFractionalTail:
+                    if (ch === "e" || ch === "E") {
+                        phase = LexerPhase.RealExponent;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    else if (!isDecimalDigit(ch) && ch !== "_") {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.RealExponent:
+                    if (ch === "+" || ch === "-") {
+                        phase = LexerPhase.RealExponentSign;
+                    }
+                    else if (isDecimalDigit(ch)) {
+                        phase = LexerPhase.RealExponentTail;
+                        tokenType = Token_1.TokenType.RealLiteral;
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.RealExponentSign:
+                    if (isDecimalDigit(ch)) {
+                        phase = LexerPhase.RealExponentTail;
+                        tokenType = Token_1.TokenType.RealLiteral;
+                    }
+                    else {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.RealExponentTail:
+                    if (!isDecimalDigit(ch)) {
+                        return makeToken();
+                    }
+                    break;
+                case LexerPhase.String:
+                    if (ch === stringState) {
+                        return completeToken(Token_1.TokenType.StringLiteral);
+                    }
+                    else if (isRestrictedInString(ch)) {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    else if (ch === "\\") {
+                        phase = LexerPhase.StringBackSlash;
+                        tokenType = Token_1.TokenType.Unknown;
+                    }
+                    break;
+                // Start of string character escape
+                case LexerPhase.StringBackSlash:
+                    switch (ch) {
+                        case "b":
+                        case "f":
+                        case "n":
+                        case "r":
+                        case "t":
+                        case "v":
+                        case "S":
+                        case "0":
+                        case "'":
+                        case '"':
+                        case "`":
+                        case "\\":
+                            phase = LexerPhase.String;
+                            break;
+                        case "x":
+                            phase = LexerPhase.StringHexa1;
+                            break;
+                        case "u":
+                            phase = LexerPhase.StringUHexa1;
+                            break;
+                        default:
+                            phase = LexerPhase.String;
+                            break;
+                    }
+                    break;
+                // --- First hexadecimal digit of string character escape
+                case LexerPhase.StringHexa1:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringHexa2;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Second hexadecimal digit of character escape
+                case LexerPhase.StringHexa2:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.String;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- First hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa1:
+                    if (ch === "{") {
+                        phase = LexerPhase.StringUcp1;
+                        break;
+                    }
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUHexa2;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Second hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa2:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUHexa3;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Third hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa3:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUHexa4;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Fourth hexadecimal digit of Unicode string character escape
+                case LexerPhase.StringUHexa4:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.String;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- First hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp1:
+                    if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp2;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Second hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp2:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp3;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Third hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp3:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp4;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Fourth hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp4:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp5;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Fifth hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp5:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcp6;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Sixth hexadecimal digit of Unicode codepoint string character escape
+                case LexerPhase.StringUcp6:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else if (isHexadecimalDigit(ch)) {
+                        phase = LexerPhase.StringUcpTail;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // --- Closing bracket of Unicode codepoint string character escape
+                case LexerPhase.StringUcpTail:
+                    if (ch === "}") {
+                        phase = LexerPhase.String;
+                    }
+                    else {
+                        return completeToken(Token_1.TokenType.Unknown);
+                    }
+                    break;
+                // ====================================================================
+                // --- We cannot continue
+                default:
+                    return makeToken();
+            }
+            // --- Append the char to the current text
+            appendTokenChar();
+            // --- Go on with parsing the next character
+        }
+        /**
+         * Appends the last character to the token, and manages positions
+         */
+        function appendTokenChar() {
+            text += ch;
+            lexer._prefetched = null;
+            lexer._prefetchedPos = null;
+            lexer._prefetchedColumn = null;
+            lastEndPos = input.position;
+            lastEndColumn = input.position;
+        }
+        /**
+         * Packs the specified type of token to send back
+         */
+        function makeToken() {
+            var _a;
+            if (useResolver) {
+                tokenType =
+                    (_a = resolverHash.get(text)) !== null && _a !== void 0 ? _a : (isIdStart(text[0]) && text[text.length - 1] !== "'" ? Token_1.TokenType.Identifier : Token_1.TokenType.Unknown);
+            }
+            return {
+                text,
+                type: tokenType,
+                location: {
+                    startPosition: startPos,
+                    endPosition: lastEndPos,
+                    startLine: line,
+                    endLine: line,
+                    startColumn,
+                    endColumn: lastEndColumn,
+                },
+            };
+        }
+        /**
+         * Add the last character to the token and return it
+         */
+        function completeToken(suggestedType) {
+            appendTokenChar();
+            // --- Send back the token
+            if (suggestedType !== undefined) {
+                tokenType = suggestedType;
+            }
+            return makeToken();
+        }
+    }
+    /**
+     * Fetches the next RegEx token from the input stream
+     */
+    fetchRegEx() {
+        // --- Get the tail
+        const tailPosition = this._ahead.length > 0 ? this._ahead[0].location.startPosition : this._lastFetchPosition;
+        const tail = this.input.getTail(tailPosition);
+        // --- Parse the tail. If no error, the entire tail is the RegExp
+        try {
+            const regexpResult = (0, regexpp_1.parseRegExpLiteral)(tail);
+            const text = regexpResult.raw;
+            // --- Consume the characters parsed successfully
+            for (let i = 1; i < text.length; i++) {
+                this.fetchNextChar();
+                this._prefetched = null;
+                this._prefetchedPos = null;
+                this._prefetchedColumn = null;
+            }
+            this._ahead.length = 0;
+            // --- Return the token
+            return {
+                success: true,
+                pattern: regexpResult.pattern.raw,
+                flags: regexpResult.flags.raw,
+                length: text.length,
+            };
+        }
+        catch (parseErr) {
+            let errorIndex = parseErr.index;
+            if (parseErr.toString().includes("Invalid flag")) {
+                while (errorIndex < tail.length && "dgimsuy".includes(tail[errorIndex])) {
+                    errorIndex++;
+                }
+            }
+            // --- If there is no error, something is wrong
+            if (errorIndex === undefined) {
+                return {
+                    success: false,
+                    pattern: tail[0],
+                };
+            }
+            // --- Try to parse the tail before the error position
+            const tailBeforeError = tail.substring(0, errorIndex);
+            try {
+                const regexpResult = (0, regexpp_1.parseRegExpLiteral)(tailBeforeError);
+                const text = regexpResult.raw;
+                // --- Consume the characters parsed successfully
+                for (let i = 1; i < text.length; i++) {
+                    this.fetchNextChar();
+                    this._prefetched = null;
+                    this._prefetchedPos = null;
+                    this._prefetchedColumn = null;
+                }
+                this._ahead.length = 0;
+                // --- Return the token
+                return {
+                    success: true,
+                    pattern: regexpResult.pattern.raw,
+                    flags: regexpResult.flags.raw,
+                    length: text.length,
+                };
+            }
+            catch (parseErr2) {
+                // --- This is really not a regexp
+                return {
+                    success: false,
+                    pattern: tailBeforeError,
+                };
+            }
+        }
+    }
+}
+exports.Lexer = Lexer;
+/**
+ * Reserved ID-like tokens
+ */
+const resolverHash = new Map();
+resolverHash.set("typeof", Token_1.TokenType.Typeof);
+resolverHash.set("Infinity", Token_1.TokenType.Infinity);
+resolverHash.set("NaN", Token_1.TokenType.NaN);
+resolverHash.set("true", Token_1.TokenType.True);
+resolverHash.set("false", Token_1.TokenType.False);
+resolverHash.set("undefined", Token_1.TokenType.Undefined);
+resolverHash.set("null", Token_1.TokenType.Null);
+resolverHash.set("in", Token_1.TokenType.In);
+resolverHash.set("let", Token_1.TokenType.Let);
+resolverHash.set("const", Token_1.TokenType.Const);
+resolverHash.set("var", Token_1.TokenType.Var);
+resolverHash.set("if", Token_1.TokenType.If);
+resolverHash.set("else", Token_1.TokenType.Else);
+resolverHash.set("return", Token_1.TokenType.Return);
+resolverHash.set("break", Token_1.TokenType.Break);
+resolverHash.set("continue", Token_1.TokenType.Continue);
+resolverHash.set("do", Token_1.TokenType.Do);
+resolverHash.set("while", Token_1.TokenType.While);
+resolverHash.set("for", Token_1.TokenType.For);
+resolverHash.set("of", Token_1.TokenType.Of);
+resolverHash.set("try", Token_1.TokenType.Try);
+resolverHash.set("catch", Token_1.TokenType.Catch);
+resolverHash.set("finally", Token_1.TokenType.Finally);
+resolverHash.set("throw", Token_1.TokenType.Throw);
+resolverHash.set("switch", Token_1.TokenType.Switch);
+resolverHash.set("case", Token_1.TokenType.Case);
+resolverHash.set("default", Token_1.TokenType.Default);
+resolverHash.set("delete", Token_1.TokenType.Delete);
+resolverHash.set("function", Token_1.TokenType.Function);
+resolverHash.set("export", Token_1.TokenType.Export);
+resolverHash.set("import", Token_1.TokenType.Import);
+resolverHash.set("as", Token_1.TokenType.As);
+resolverHash.set("from", Token_1.TokenType.From);
+/**
+ * Tests if a token id EOF
+ * @param t Token instance
+ */
+function isEof(t) {
+    return t.type === Token_1.TokenType.Eof;
+}
+/**
+ * Tests if a token is whitespace
+ * @param t Token instance
+ */
+function isWs(t) {
+    return t.type <= Token_1.TokenType.Ws;
+}
+/**
+ * Tests if a character is an identifier start character
+ * @param ch Character to test
+ */
+function isIdStart(ch) {
+    return (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || ch === "_" || ch === "$";
+}
+/**
+ * Tests if a character is an identifier continuation character
+ * @param ch Character to test
+ */
+function isIdContinuation(ch) {
+    return (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || (ch >= "0" && ch <= "9") || ch === "_" || ch === "$";
+}
+/**
+ * Tests if a character is a binary digit
+ * @param ch Character to test
+ */
+function isBinaryDigit(ch) {
+    return ch === "0" || ch === "1";
+}
+/**
+ * Tests if a character is a decimal digit
+ * @param ch Character to test
+ */
+function isDecimalDigit(ch) {
+    return ch >= "0" && ch <= "9";
+}
+/**
+ * Tests if a character is a hexadecimal digit
+ * @param ch Character to test
+ */
+function isHexadecimalDigit(ch) {
+    return (ch >= "0" && ch <= "9") || (ch >= "A" && ch <= "F") || (ch >= "a" && ch <= "f");
+}
+/**
+ * Tests if a character is restricted in a string
+ * @param ch Character to test
+ */
+function isRestrictedInString(ch) {
+    return ch === "\r" || ch === "\n" || ch === "\u0085" || ch === "\u2028" || ch === "\u2029";
+}