xmlui 0.9.1 → 0.9.2

package/dist/scripts/src/parsers/scripting-exp/Lexer.js

@@ -1,1092 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Lexer = void 0;
-const regexpp_1 = require("@eslint-community/regexpp");
-const TokenType_1 = require("./TokenType");
-/**
- * This enum indicates the current lexer phase
- */
-var LexerPhase;
-(function (LexerPhase) {
-    // Start getting a token
-    LexerPhase[LexerPhase["Start"] = 0] = "Start";
-    // Collecting whitespace
-    LexerPhase[LexerPhase["InWhiteSpace"] = 1] = "InWhiteSpace";
-    // Comment phases
-    LexerPhase[LexerPhase["InlineCommentTrail"] = 2] = "InlineCommentTrail";
-    LexerPhase[LexerPhase["BlockCommentTrail1"] = 3] = "BlockCommentTrail1";
-    LexerPhase[LexerPhase["BlockCommentTrail2"] = 4] = "BlockCommentTrail2";
-    // Multi-char tokens
-    LexerPhase[LexerPhase["Slash"] = 5] = "Slash";
-    LexerPhase[LexerPhase["Dollar"] = 6] = "Dollar";
-    LexerPhase[LexerPhase["Or"] = 7] = "Or";
-    LexerPhase[LexerPhase["Asterisk"] = 8] = "Asterisk";
-    LexerPhase[LexerPhase["Ampersand"] = 9] = "Ampersand";
-    LexerPhase[LexerPhase["Equal"] = 10] = "Equal";
-    LexerPhase[LexerPhase["DoubleEqual"] = 11] = "DoubleEqual";
-    LexerPhase[LexerPhase["NotEqual"] = 12] = "NotEqual";
-    LexerPhase[LexerPhase["Exclamation"] = 13] = "Exclamation";
-    LexerPhase[LexerPhase["AngleLeft"] = 14] = "AngleLeft";
-    LexerPhase[LexerPhase["AngleRight"] = 15] = "AngleRight";
-    LexerPhase[LexerPhase["SignedShiftRight"] = 16] = "SignedShiftRight";
-    LexerPhase[LexerPhase["IdTail"] = 17] = "IdTail";
-    LexerPhase[LexerPhase["Dot"] = 18] = "Dot";
-    LexerPhase[LexerPhase["DotDot"] = 19] = "DotDot";
-    LexerPhase[LexerPhase["Colon"] = 20] = "Colon";
-    LexerPhase[LexerPhase["Zero"] = 21] = "Zero";
-    LexerPhase[LexerPhase["QuestionMark"] = 22] = "QuestionMark";
-    LexerPhase[LexerPhase["HexaFirst"] = 23] = "HexaFirst";
-    LexerPhase[LexerPhase["HexaTail"] = 24] = "HexaTail";
-    LexerPhase[LexerPhase["BinaryFirst"] = 25] = "BinaryFirst";
-    LexerPhase[LexerPhase["BinaryTail"] = 26] = "BinaryTail";
-    LexerPhase[LexerPhase["DecimalOrReal"] = 27] = "DecimalOrReal";
-    LexerPhase[LexerPhase["RealFractionalFirst"] = 28] = "RealFractionalFirst";
-    LexerPhase[LexerPhase["RealFractionalTail"] = 29] = "RealFractionalTail";
-    LexerPhase[LexerPhase["RealExponent"] = 30] = "RealExponent";
-    LexerPhase[LexerPhase["RealExponentSign"] = 31] = "RealExponentSign";
-    LexerPhase[LexerPhase["RealExponentTail"] = 32] = "RealExponentTail";
-    LexerPhase[LexerPhase["StringTemplateLiteralBackSlash"] = 33] = "StringTemplateLiteralBackSlash";
-    LexerPhase[LexerPhase["StringTemplateLiteral"] = 34] = "StringTemplateLiteral";
-    LexerPhase[LexerPhase["String"] = 35] = "String";
-    LexerPhase[LexerPhase["StringBackSlash"] = 36] = "StringBackSlash";
-    LexerPhase[LexerPhase["StringHexa1"] = 37] = "StringHexa1";
-    LexerPhase[LexerPhase["StringHexa2"] = 38] = "StringHexa2";
-    LexerPhase[LexerPhase["StringUHexa1"] = 39] = "StringUHexa1";
-    LexerPhase[LexerPhase["StringUHexa2"] = 40] = "StringUHexa2";
-    LexerPhase[LexerPhase["StringUHexa3"] = 41] = "StringUHexa3";
-    LexerPhase[LexerPhase["StringUHexa4"] = 42] = "StringUHexa4";
-    LexerPhase[LexerPhase["StringUcp1"] = 43] = "StringUcp1";
-    LexerPhase[LexerPhase["StringUcp2"] = 44] = "StringUcp2";
-    LexerPhase[LexerPhase["StringUcp3"] = 45] = "StringUcp3";
-    LexerPhase[LexerPhase["StringUcp4"] = 46] = "StringUcp4";
-    LexerPhase[LexerPhase["StringUcp5"] = 47] = "StringUcp5";
-    LexerPhase[LexerPhase["StringUcp6"] = 48] = "StringUcp6";
-    LexerPhase[LexerPhase["StringUcpTail"] = 49] = "StringUcpTail";
-    // --- Assignments
-    LexerPhase[LexerPhase["Exponent"] = 50] = "Exponent";
-    LexerPhase[LexerPhase["Plus"] = 51] = "Plus";
-    LexerPhase[LexerPhase["Minus"] = 52] = "Minus";
-    LexerPhase[LexerPhase["Divide"] = 53] = "Divide";
-    LexerPhase[LexerPhase["Remainder"] = 54] = "Remainder";
-    LexerPhase[LexerPhase["ShiftLeft"] = 55] = "ShiftLeft";
-    LexerPhase[LexerPhase["ShiftRight"] = 56] = "ShiftRight";
-    LexerPhase[LexerPhase["LogicalAnd"] = 57] = "LogicalAnd";
-    LexerPhase[LexerPhase["BitwiseXor"] = 58] = "BitwiseXor";
-    LexerPhase[LexerPhase["LogicalOr"] = 59] = "LogicalOr";
-    LexerPhase[LexerPhase["NullCoalesce"] = 60] = "NullCoalesce";
-    // --- Other
-    LexerPhase[LexerPhase["RegEx"] = 61] = "RegEx";
-})(LexerPhase || (LexerPhase = {}));
-/**
- * This class implements the lexer of binding expressions
- */
-class Lexer {
-    /**
-     * Initializes the tokenizer with the input stream
-     * @param input Input source code stream
-     */
-    constructor(input) {
-        this.input = input;
-        // --- Already fetched tokens
-        this._ahead = [];
-        // --- Prefetched character (from the next token)
-        this._prefetched = null;
-        // --- Prefetched character position (from the next token)
-        this._prefetchedPos = null;
-        // --- Prefetched character column (from the next token)
-        this._prefetchedColumn = null;
-        // --- input position at the beginning of last fetch
-        this._lastFetchPosition = 0;
-        this._phaseExternallySet = null;
-    }
-    /**
-     * Fetches the next token without advancing to its position
-     * @param ws If true, retrieve whitespaces too
-     */
-    peek(ws = false) {
-        return this.ahead(0, ws);
-    }
-    /**
-     * Reads tokens ahead
-     * @param n Number of token positions to read ahead
-     * @param ws If true, retrieve whitespaces too
-     */
-    ahead(n = 1, ws = false) {
-        if (n > 16) {
-            throw new Error("Cannot look ahead more than 16 tokens");
-        }
-        // --- Prefetch missing tokens
-        while (this._ahead.length <= n) {
-            const token = this.fetch();
-            if (isEof(token)) {
-                return token;
-            }
-            if (ws || (!ws && !isWs(token))) {
-                this._ahead.push(token);
-            }
-        }
-        return this._ahead[n];
-    }
-    /**
-     * Fetches the next token and advances the stream position
-     * @param ws If true, retrieve whitespaces too
-     */
-    get(ws = false) {
-        if (this._ahead.length > 0) {
-            const token = this._ahead.shift();
-            if (!token) {
-                throw new Error("Token expected");
-            }
-            return token;
-        }
-        while (true) {
-            const token = this.fetch();
-            if (isEof(token) || ws || (!ws && !isWs(token))) {
-                return token;
-            }
-        }
-    }
-    /**
-     * Gets a RegEx from the current position
-     */
-    getRegEx() {
-        return this.fetchRegEx();
-    }
-    /**
-     * Gets the remaining characters after the parsing phase
-     */
-    getTail() {
-        return this._ahead.length > 0
-            ? this.input.getTail(this._ahead[0].startPosition)
-            : this.input.getTail(this._lastFetchPosition);
-    }
-    /** Parsing template literals requires a context sensitive lexer.
-     * This method has to be called by the parser when the lexer needs to scan a string inside a template literal.
-     * Call this after the first opening backing and after the parser is done with parsing a placeholder, after the right brace.
-     */
-    setStartingPhaseToTemplateLiteral() {
-        this._phaseExternallySet = LexerPhase.StringTemplateLiteral;
-    }
-    /**
-     * Fetches the next character from the input stream
-     */
-    fetchNextChar() {
-        if (!this._prefetched) {
-            this._prefetchedPos = this.input.position;
-            this._prefetchedColumn = this.input.column;
-            this._prefetched = this.input.get();
-        }
-        return this._prefetched;
-    }
-    /**
-     * Fetches the next token from the input stream
-     */
-    fetch() {
-        // --- Captured constants used in nested functions
-        const lexer = this;
-        const input = this.input;
-        const startPos = this._prefetchedPos || input.position;
-        const line = input.line;
-        const startColumn = this._prefetchedColumn || input.column;
-        this._lastFetchPosition = this.input.position;
-        // --- State variables
-        let stringState = null;
-        let text = "";
-        let tokenType = TokenType_1.TokenType.Eof;
-        let lastEndPos = input.position;
-        let lastEndColumn = input.column;
-        let ch = null;
-        let useResolver = false;
-        // --- Start from the beginning
-        let phase = this.getStartingPhaseThenReset();
-        // --- Process all token characters
-        while (true) {
-            // --- Get the next character
-            ch = this.fetchNextChar();
-            // --- In case of EOF, return the current token data
-            if (ch === null) {
-                return makeToken();
-            }
-            // --- Set the initial token type to unknown for the other characters
-            if (tokenType === TokenType_1.TokenType.Eof) {
-                tokenType = TokenType_1.TokenType.Unknown;
-            }
-            // --- Follow the lexer state machine
-            phaseSwitch: switch (phase) {
-                // ====================================================================
-                // Process the first character
-                case LexerPhase.Start:
-                    switch (ch) {
-                        // --- Go on with whitespaces
-                        case " ":
-                        case "\t":
-                        case "\n":
-                        case "\r":
-                            phase = LexerPhase.InWhiteSpace;
-                            tokenType = TokenType_1.TokenType.Ws;
-                            break;
-                        // --- Divide, BlockComment, or EolComment
-                        case "/":
-                            phase = LexerPhase.Slash;
-                            tokenType = TokenType_1.TokenType.Divide;
-                            break;
-                        case "$":
-                            phase = LexerPhase.Dollar;
-                            tokenType = TokenType_1.TokenType.Identifier;
-                            break;
-                        case "*":
-                            phase = LexerPhase.Asterisk;
-                            tokenType = TokenType_1.TokenType.Multiply;
-                            break;
-                        case "%":
-                            phase = LexerPhase.Remainder;
-                            tokenType = TokenType_1.TokenType.Remainder;
-                            break;
-                        case "+":
-                            phase = LexerPhase.Plus;
-                            tokenType = TokenType_1.TokenType.Plus;
-                            break;
-                        case "-":
-                            phase = LexerPhase.Minus;
-                            tokenType = TokenType_1.TokenType.Minus;
-                            break;
-                        case "^":
-                            phase = LexerPhase.BitwiseXor;
-                            tokenType = TokenType_1.TokenType.BitwiseXor;
-                            break;
-                        // --- BitwiseOr, LogicalOr
-                        case "|":
-                            phase = LexerPhase.Or;
-                            tokenType = TokenType_1.TokenType.BitwiseOr;
-                            break;
-                        // --- BitwiseAnd, LogicalAnd
-                        case "&":
-                            phase = LexerPhase.Ampersand;
-                            tokenType = TokenType_1.TokenType.BitwiseAnd;
-                            break;
-                        // --- "?", "?", or "?."
-                        case "?":
-                            phase = LexerPhase.QuestionMark;
-                            tokenType = TokenType_1.TokenType.QuestionMark;
-                            break;
-                        case ";":
-                            return completeToken(TokenType_1.TokenType.Semicolon);
-                        case ",":
-                            return completeToken(TokenType_1.TokenType.Comma);
-                        case "(":
-                            return completeToken(TokenType_1.TokenType.LParent);
-                        case ")":
-                            return completeToken(TokenType_1.TokenType.RParent);
-                        // --- ":" or "::"
-                        case ":":
-                            phase = LexerPhase.Colon;
-                            tokenType = TokenType_1.TokenType.Colon;
-                            break;
-                        case "`":
-                            return completeToken(TokenType_1.TokenType.Backtick);
-                        case "[":
-                            return completeToken(TokenType_1.TokenType.LSquare);
-                        case "]":
-                            return completeToken(TokenType_1.TokenType.RSquare);
-                        case "~":
-                            return completeToken(TokenType_1.TokenType.BinaryNot);
-                        case "{":
-                            return completeToken(TokenType_1.TokenType.LBrace);
-                        case "}":
-                            return completeToken(TokenType_1.TokenType.RBrace);
-                        // --- "=","==", "===", or "=>"
-                        case "=":
-                            phase = LexerPhase.Equal;
-                            tokenType = TokenType_1.TokenType.Assignment;
-                            break;
-                        // --- "!", "!=", or "!=="
-                        case "!":
-                            phase = LexerPhase.Exclamation;
-                            tokenType = TokenType_1.TokenType.LogicalNot;
-                            break;
-                        // --- "<", "<=", or "<<"
-                        case "<":
-                            phase = LexerPhase.AngleLeft;
-                            tokenType = TokenType_1.TokenType.LessThan;
-                            break;
-                        // --- ">", ">=", ">>", or ">>>"
-                        case ">":
-                            phase = LexerPhase.AngleRight;
-                            tokenType = TokenType_1.TokenType.GreaterThan;
-                            break;
-                        // --- Decimal or Real literal
-                        case "0":
-                            phase = LexerPhase.Zero;
-                            tokenType = TokenType_1.TokenType.DecimalLiteral;
-                            break;
-                        case ".":
-                            phase = LexerPhase.Dot;
-                            tokenType = TokenType_1.TokenType.Dot;
-                            break;
-                        // --- String (both " and ' are accepted as string wrappers
-                        case '"':
-                        case "'":
-                            stringState = ch;
-                            phase = LexerPhase.String;
-                            break;
-                        default:
-                            if (isIdStart(ch)) {
-                                useResolver = true;
-                                phase = LexerPhase.IdTail;
-                                tokenType = TokenType_1.TokenType.Identifier;
-                            }
-                            else if (isDecimalDigit(ch)) {
-                                phase = LexerPhase.DecimalOrReal;
-                                tokenType = TokenType_1.TokenType.DecimalLiteral;
-                            }
-                            else {
-                                completeToken(TokenType_1.TokenType.Unknown);
-                            }
-                            break;
-                    }
-                    break;
-                // ====================================================================
-                // Process comments
-                // --- Looking for the end of whitespace
-                case LexerPhase.InWhiteSpace:
-                    if (ch !== " " && ch !== "\t" && ch !== "\r" && ch !== "\n") {
-                        return makeToken();
-                    }
-                    break;
-                // --- Wait for an "*" that may complete a block comment
-                case LexerPhase.BlockCommentTrail1:
-                    if (ch === "*") {
-                        phase = LexerPhase.BlockCommentTrail2;
-                    }
-                    break;
-                // --- Wait for a "/" that may complete a block comment
-                case LexerPhase.BlockCommentTrail2:
-                    if (ch === "/") {
-                        return completeToken(TokenType_1.TokenType.BlockComment);
-                    }
-                    break;
-                case LexerPhase.InlineCommentTrail:
-                    if (ch === "\n") {
-                        return completeToken();
-                    }
-                    break;
-                // ====================================================================
-                // Process identifiers
-                case LexerPhase.IdTail:
-                    if (!isIdContinuation(ch)) {
-                        return makeToken();
-                    }
-                    break;
-                // ====================================================================
-                // Process miscellaneous tokens
-                case LexerPhase.Colon:
-                    return ch === ":" ? completeToken(TokenType_1.TokenType.Global) : makeToken();
-                case LexerPhase.Slash:
-                    if (ch === "*") {
-                        phase = LexerPhase.BlockCommentTrail1;
-                    }
-                    else if (ch === "/") {
-                        phase = LexerPhase.InlineCommentTrail;
-                        tokenType = TokenType_1.TokenType.EolComment;
-                    }
-                    else if (ch === "=") {
-                        return completeToken(TokenType_1.TokenType.DivideAssignment);
-                    }
-                    else {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.Plus:
-                    if (ch === "+") {
-                        return completeToken(TokenType_1.TokenType.IncOp);
-                    }
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.AddAssignment) : makeToken();
-                case LexerPhase.Minus:
-                    if (ch === "-") {
-                        return completeToken(TokenType_1.TokenType.DecOp);
-                    }
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.SubtractAssignment) : makeToken();
-                case LexerPhase.Remainder:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.RemainderAssignment) : makeToken();
-                case LexerPhase.BitwiseXor:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.BitwiseXorAssignment) : makeToken();
-                case LexerPhase.Or:
-                    if (ch === "=") {
-                        return completeToken(TokenType_1.TokenType.BitwiseOrAssignment);
-                    }
-                    if (ch === "|") {
-                        phase = LexerPhase.LogicalOr;
-                        tokenType = TokenType_1.TokenType.LogicalOr;
-                        break;
-                    }
-                    return makeToken();
-                case LexerPhase.LogicalOr:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.LogicalOrAssignment) : makeToken();
-                case LexerPhase.Ampersand:
-                    if (ch === "=") {
-                        return completeToken(TokenType_1.TokenType.BitwiseAndAssignment);
-                    }
-                    if (ch === "&") {
-                        phase = LexerPhase.LogicalAnd;
-                        tokenType = TokenType_1.TokenType.LogicalAnd;
-                        break;
-                    }
-                    return makeToken();
-                case LexerPhase.LogicalAnd:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.LogicalAndAssignment) : makeToken();
-                case LexerPhase.Asterisk:
-                    if (ch === "*") {
-                        phase = LexerPhase.Exponent;
-                        tokenType = TokenType_1.TokenType.Exponent;
-                        break;
-                    }
-                    else if (ch === "=") {
-                        return completeToken(TokenType_1.TokenType.MultiplyAssignment);
-                    }
-                    return makeToken();
-                case LexerPhase.Exponent:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.ExponentAssignment) : makeToken();
-                case LexerPhase.QuestionMark:
-                    if (ch === "?") {
-                        phase = LexerPhase.NullCoalesce;
-                        tokenType = TokenType_1.TokenType.NullCoalesce;
-                        break;
-                    }
-                    if (ch === ".") {
-                        return completeToken(TokenType_1.TokenType.OptionalChaining);
-                    }
-                    return makeToken();
-                case LexerPhase.NullCoalesce:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.NullCoalesceAssignment) : makeToken();
-                case LexerPhase.Equal:
-                    if (ch === ">") {
-                        return completeToken(TokenType_1.TokenType.Arrow);
-                    }
-                    if (ch === "=") {
-                        phase = LexerPhase.DoubleEqual;
-                        tokenType = TokenType_1.TokenType.Equal;
-                        break;
-                    }
-                    return makeToken();
-                case LexerPhase.DoubleEqual:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.StrictEqual) : makeToken();
-                case LexerPhase.Exclamation:
-                    if (ch === "=") {
-                        phase = LexerPhase.NotEqual;
-                        tokenType = TokenType_1.TokenType.NotEqual;
-                        break;
-                    }
-                    return makeToken();
-                case LexerPhase.NotEqual:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.StrictNotEqual) : makeToken();
-                case LexerPhase.AngleLeft:
-                    if (ch === "=") {
-                        return completeToken(TokenType_1.TokenType.LessThanOrEqual);
-                    }
-                    if (ch === "<") {
-                        phase = LexerPhase.ShiftLeft;
-                        tokenType = TokenType_1.TokenType.ShiftLeft;
-                        break;
-                    }
-                    return makeToken();
-                case LexerPhase.ShiftLeft:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.ShiftLeftAssignment) : makeToken();
-                case LexerPhase.AngleRight:
-                    if (ch === "=") {
-                        return completeToken(TokenType_1.TokenType.GreaterThanOrEqual);
-                    }
-                    if (ch === ">") {
-                        phase = LexerPhase.SignedShiftRight;
-                        tokenType = TokenType_1.TokenType.SignedShiftRight;
-                        break;
-                    }
-                    return makeToken();
-                case LexerPhase.SignedShiftRight:
-                    if (ch === ">") {
-                        phase = LexerPhase.ShiftRight;
-                        tokenType = TokenType_1.TokenType.ShiftRight;
-                        break;
-                    }
-                    if (ch === "=") {
-                        return completeToken(TokenType_1.TokenType.SignedShiftRightAssignment);
-                    }
-                    return makeToken();
-                case LexerPhase.ShiftRight:
-                    return ch === "=" ? completeToken(TokenType_1.TokenType.ShiftRightAssignment) : makeToken();
-                // ====================================================================
-                // --- Literals
-                case LexerPhase.Zero:
-                    if (ch === "x") {
-                        phase = LexerPhase.HexaFirst;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    else if (ch === "b") {
-                        phase = LexerPhase.BinaryFirst;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    else if (isDecimalDigit(ch) || ch === "_") {
-                        phase = LexerPhase.DecimalOrReal;
-                    }
-                    else if (ch === ".") {
-                        phase = LexerPhase.RealFractionalFirst;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    else if (ch === "e" || ch === "E") {
-                        phase = LexerPhase.RealExponent;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    else {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.Dot:
-                    if (ch === ".") {
-                        phase = LexerPhase.DotDot;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                        break;
-                    }
-                    if (!isDecimalDigit(ch)) {
-                        return makeToken();
-                    }
-                    phase = LexerPhase.RealFractionalTail;
-                    tokenType = TokenType_1.TokenType.RealLiteral;
-                    break;
-                case LexerPhase.DotDot:
-                    return ch === "." ? completeToken(TokenType_1.TokenType.Spread) : makeToken();
-                case LexerPhase.HexaFirst:
-                    if (ch === "_") {
-                        break;
-                    }
-                    if (!isHexadecimalDigit(ch)) {
-                        return makeToken();
-                    }
-                    phase = LexerPhase.HexaTail;
-                    tokenType = TokenType_1.TokenType.HexadecimalLiteral;
-                    break;
-                case LexerPhase.HexaTail:
-                    if (!isHexadecimalDigit(ch) && ch !== "_") {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.BinaryFirst:
-                    if (ch === "_") {
-                        break;
-                    }
-                    if (!isBinaryDigit(ch)) {
-                        return makeToken();
-                    }
-                    phase = LexerPhase.BinaryTail;
-                    tokenType = TokenType_1.TokenType.BinaryLiteral;
-                    break;
-                case LexerPhase.BinaryTail:
-                    if (!isBinaryDigit(ch) && ch !== "_") {
-                        return makeToken();
-                    }
-                    tokenType = TokenType_1.TokenType.BinaryLiteral;
-                    break;
-                case LexerPhase.DecimalOrReal:
-                    if (isDecimalDigit(ch) || ch === "_") {
-                        break;
-                    }
-                    else if (ch === "." && (this.input.peek() === null || isDecimalDigit(this.input.peek()))) {
-                        phase = LexerPhase.RealFractionalFirst;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    else if (ch === "e" || ch === "E") {
-                        phase = LexerPhase.RealExponent;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    else {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.RealFractionalFirst:
-                    if (isDecimalDigit(ch)) {
-                        phase = LexerPhase.RealFractionalTail;
-                        tokenType = TokenType_1.TokenType.RealLiteral;
-                    }
-                    else if (ch === "e" || ch === "E") {
-                        phase = LexerPhase.RealExponent;
-                    }
-                    else {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.RealFractionalTail:
-                    if (ch === "e" || ch === "E") {
-                        phase = LexerPhase.RealExponent;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    else if (!isDecimalDigit(ch) && ch !== "_") {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.RealExponent:
-                    if (ch === "+" || ch === "-") {
-                        phase = LexerPhase.RealExponentSign;
-                    }
-                    else if (isDecimalDigit(ch)) {
-                        phase = LexerPhase.RealExponentTail;
-                        tokenType = TokenType_1.TokenType.RealLiteral;
-                    }
-                    else {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.RealExponentSign:
-                    if (isDecimalDigit(ch)) {
-                        phase = LexerPhase.RealExponentTail;
-                        tokenType = TokenType_1.TokenType.RealLiteral;
-                    }
-                    else {
-                        return makeToken();
-                    }
-                    break;
-                case LexerPhase.RealExponentTail:
-                    if (!isDecimalDigit(ch)) {
-                        return makeToken();
-                    }
-                    break;
-                // A dollar sign is also a valid variable name. When it isn't a dollar left brace, we continue as if it was an identifier
-                case LexerPhase.Dollar:
-                    if (ch === "{") {
-                        return completeToken(TokenType_1.TokenType.DollarLBrace);
-                    }
-                    phase = LexerPhase.IdTail;
-                    useResolver = true;
-                    tokenType = TokenType_1.TokenType.Identifier;
-                    if (!isIdContinuation(ch)) {
-                        makeToken();
-                    }
-                    break;
-                case LexerPhase.StringTemplateLiteralBackSlash: {
-                    phase = LexerPhase.StringTemplateLiteral;
-                    const charAhead1 = this.input.ahead(0);
-                    const charAhead2 = this.input.ahead(1);
-                    if (charAhead1 === "`" || (charAhead1 === "$" && charAhead2 === "{")) {
-                        return completeToken(TokenType_1.TokenType.StringLiteral);
-                    }
-                    break;
-                }
-                case LexerPhase.StringTemplateLiteral:
-                    switch (ch) {
-                        case "\\":
-                            phase = LexerPhase.StringTemplateLiteralBackSlash;
-                            tokenType = TokenType_1.TokenType.Unknown;
-                            break phaseSwitch;
-                        case "`":
-                            return completeToken(TokenType_1.TokenType.Backtick);
-                        case "$":
-                            const charAhead = this.input.ahead(0);
-                            if (charAhead === "{") {
-                                appendTokenChar();
-                                this.fetchNextChar();
-                                return completeToken(TokenType_1.TokenType.DollarLBrace);
-                            }
-                    }
-                    const charAhead1 = this.input.ahead(0);
-                    const charAhead2 = this.input.ahead(1);
-                    if (charAhead1 === "`" || (charAhead1 === "$" && charAhead2 === "{")) {
-                        return completeToken(TokenType_1.TokenType.StringLiteral);
-                    }
-                    break;
-                case LexerPhase.String:
-                    if (ch === stringState) {
-                        return completeToken(TokenType_1.TokenType.StringLiteral);
-                    }
-                    else if (isRestrictedInString(ch)) {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    else if (ch === "\\") {
-                        phase = LexerPhase.StringBackSlash;
-                        tokenType = TokenType_1.TokenType.Unknown;
-                    }
-                    break;
-                // Start of string character escape
-                case LexerPhase.StringBackSlash:
-                    switch (ch) {
-                        case "b":
-                        case "f":
-                        case "n":
-                        case "r":
-                        case "t":
-                        case "v":
-                        case "S":
-                        case "0":
-                        case "'":
-                        case '"':
-                        case "`":
-                        case "\\":
-                            phase = LexerPhase.String;
-                            break;
-                        case "x":
-                            phase = LexerPhase.StringHexa1;
-                            break;
-                        case "u":
-                            phase = LexerPhase.StringUHexa1;
-                            break;
-                        default:
-                            phase = LexerPhase.String;
-                            break;
-                    }
-                    break;
-                // --- First hexadecimal digit of string character escape
-                case LexerPhase.StringHexa1:
-                    if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringHexa2;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Second hexadecimal digit of character escape
-                case LexerPhase.StringHexa2:
-                    if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.String;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- First hexadecimal digit of Unicode string character escape
-                case LexerPhase.StringUHexa1:
-                    if (ch === "{") {
-                        phase = LexerPhase.StringUcp1;
-                        break;
-                    }
-                    if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUHexa2;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Second hexadecimal digit of Unicode string character escape
-                case LexerPhase.StringUHexa2:
-                    if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUHexa3;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Third hexadecimal digit of Unicode string character escape
-                case LexerPhase.StringUHexa3:
-                    if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUHexa4;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Fourth hexadecimal digit of Unicode string character escape
-                case LexerPhase.StringUHexa4:
-                    if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.String;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- First hexadecimal digit of Unicode codepoint string character escape
-                case LexerPhase.StringUcp1:
-                    if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUcp2;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Second hexadecimal digit of Unicode codepoint string character escape
-                case LexerPhase.StringUcp2:
-                    if (ch === "}") {
-                        phase = LexerPhase.String;
-                    }
-                    else if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUcp3;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Third hexadecimal digit of Unicode codepoint string character escape
-                case LexerPhase.StringUcp3:
-                    if (ch === "}") {
-                        phase = LexerPhase.String;
-                    }
-                    else if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUcp4;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Fourth hexadecimal digit of Unicode codepoint string character escape
-                case LexerPhase.StringUcp4:
-                    if (ch === "}") {
-                        phase = LexerPhase.String;
-                    }
-                    else if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUcp5;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Fifth hexadecimal digit of Unicode codepoint string character escape
-                case LexerPhase.StringUcp5:
-                    if (ch === "}") {
-                        phase = LexerPhase.String;
-                    }
-                    else if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUcp6;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Sixth hexadecimal digit of Unicode codepoint string character escape
-                case LexerPhase.StringUcp6:
-                    if (ch === "}") {
-                        phase = LexerPhase.String;
-                    }
-                    else if (isHexadecimalDigit(ch)) {
-                        phase = LexerPhase.StringUcpTail;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // --- Closing bracket of Unicode codepoint string character escape
-                case LexerPhase.StringUcpTail:
-                    if (ch === "}") {
-                        phase = LexerPhase.String;
-                    }
-                    else {
-                        return completeToken(TokenType_1.TokenType.Unknown);
-                    }
-                    break;
-                // ====================================================================
-                // --- We cannot continue
-                default:
-                    return makeToken();
-            }
-            // --- Append the char to the current text
-            appendTokenChar();
-            // --- Go on with parsing the next character
-        }
-        /**
-         * Appends the last character to the token, and manages positions
-         */
-        function appendTokenChar() {
-            text += ch;
-            lexer._prefetched = null;
-            lexer._prefetchedPos = null;
-            lexer._prefetchedColumn = null;
-            lastEndPos = input.position;
-            lastEndColumn = input.position;
-        }
-        /**
-         * Packs the specified type of token to send back
-         */
-        function makeToken() {
-            var _a;
-            if (useResolver) {
-                tokenType =
-                    (_a = resolverHash.get(text)) !== null && _a !== void 0 ? _a : (isIdStart(text[0]) && text[text.length - 1] !== "'" ? TokenType_1.TokenType.Identifier : TokenType_1.TokenType.Unknown);
-            }
-            return {
-                text,
-                type: tokenType,
-                startPosition: startPos,
-                endPosition: lastEndPos,
-                startLine: line,
-                endLine: line,
-                startColumn,
-                endColumn: lastEndColumn,
-            };
-        }
-        /**
-         * Add the last character to the token and return it
-         */
-        function completeToken(suggestedType) {
-            appendTokenChar();
-            // --- Send back the token
-            if (suggestedType !== undefined) {
-                tokenType = suggestedType;
-            }
-            return makeToken();
-        }
-    }
-    getStartingPhaseThenReset() {
-        if (this._phaseExternallySet !== null) {
-            const phase = this._phaseExternallySet;
-            this._phaseExternallySet = null;
-            return phase;
-        }
-        return LexerPhase.Start;
-    }
-    /**
-     * Fetches the next RegEx token from the input stream
-     */
-    fetchRegEx() {
-        // --- Get the tail
-        const tailPosition = this._ahead.length > 0 ? this._ahead[0].startPosition : this._lastFetchPosition;
-        const tail = this.input.getTail(tailPosition);
-        // --- Parse the tail. If no error, the entire tail is the RegExp
-        try {
-            const regexpResult = (0, regexpp_1.parseRegExpLiteral)(tail);
-            const text = regexpResult.raw;
-            // --- Consume the characters parsed successfully
-            for (let i = 1; i < text.length; i++) {
-                this.fetchNextChar();
-                this._prefetched = null;
-                this._prefetchedPos = null;
-                this._prefetchedColumn = null;
-            }
-            this._ahead.length = 0;
-            // --- Return the token
-            return {
-                success: true,
-                pattern: regexpResult.pattern.raw,
-                flags: regexpResult.flags.raw,
-                length: text.length,
-            };
-        }
-        catch (parseErr) {
-            let errorIndex = parseErr.index;
-            if (parseErr.toString().includes("Invalid flag")) {
-                while (errorIndex < tail.length && "dgimsuy".includes(tail[errorIndex])) {
-                    errorIndex++;
-                }
-            }
-            // --- If there is no error, something is wrong
-            if (errorIndex === undefined) {
-                return {
-                    success: false,
-                    pattern: tail[0],
-                };
-            }
-            // --- Try to parse the tail before the error position
-            const tailBeforeError = tail.substring(0, errorIndex);
-            try {
-                const regexpResult = (0, regexpp_1.parseRegExpLiteral)(tailBeforeError);
-                const text = regexpResult.raw;
-                // --- Consume the characters parsed successfully
-                for (let i = 1; i < text.length; i++) {
-                    this.fetchNextChar();
-                    this._prefetched = null;
-                    this._prefetchedPos = null;
-                    this._prefetchedColumn = null;
-                }
-                this._ahead.length = 0;
-                // --- Return the token
-                return {
-                    success: true,
-                    pattern: regexpResult.pattern.raw,
-                    flags: regexpResult.flags.raw,
-                    length: text.length,
-                };
-            }
-            catch (parseErr2) {
-                // --- This is really not a regexp
-                return {
-                    success: false,
-                    pattern: tailBeforeError,
-                };
-            }
-        }
-    }
-}
-exports.Lexer = Lexer;
-/**
- * Reserved ID-like tokens
- */
-const resolverHash = new Map();
-resolverHash.set("typeof", TokenType_1.TokenType.Typeof);
-resolverHash.set("Infinity", TokenType_1.TokenType.Infinity);
-resolverHash.set("NaN", TokenType_1.TokenType.NaN);
-resolverHash.set("true", TokenType_1.TokenType.True);
-resolverHash.set("false", TokenType_1.TokenType.False);
-resolverHash.set("undefined", TokenType_1.TokenType.Undefined);
-resolverHash.set("null", TokenType_1.TokenType.Null);
-resolverHash.set("in", TokenType_1.TokenType.In);
-resolverHash.set("let", TokenType_1.TokenType.Let);
-resolverHash.set("const", TokenType_1.TokenType.Const);
-resolverHash.set("var", TokenType_1.TokenType.Var);
-resolverHash.set("if", TokenType_1.TokenType.If);
-resolverHash.set("else", TokenType_1.TokenType.Else);
-resolverHash.set("return", TokenType_1.TokenType.Return);
-resolverHash.set("break", TokenType_1.TokenType.Break);
-resolverHash.set("continue", TokenType_1.TokenType.Continue);
-resolverHash.set("do", TokenType_1.TokenType.Do);
-resolverHash.set("while", TokenType_1.TokenType.While);
-resolverHash.set("for", TokenType_1.TokenType.For);
-resolverHash.set("of", TokenType_1.TokenType.Of);
-resolverHash.set("try", TokenType_1.TokenType.Try);
-resolverHash.set("catch", TokenType_1.TokenType.Catch);
-resolverHash.set("finally", TokenType_1.TokenType.Finally);
-resolverHash.set("throw", TokenType_1.TokenType.Throw);
-resolverHash.set("switch", TokenType_1.TokenType.Switch);
-resolverHash.set("case", TokenType_1.TokenType.Case);
-resolverHash.set("default", TokenType_1.TokenType.Default);
-resolverHash.set("delete", TokenType_1.TokenType.Delete);
-resolverHash.set("function", TokenType_1.TokenType.Function);
-resolverHash.set("as", TokenType_1.TokenType.As);
-resolverHash.set("from", TokenType_1.TokenType.From);
-/**
- * Tests if a token id EOF
- * @param t Token instance
- */
-function isEof(t) {
-    return t.type === TokenType_1.TokenType.Eof;
-}
-/**
- * Tests if a token is whitespace
- * @param t Token instance
- */
-function isWs(t) {
-    return t.type <= TokenType_1.TokenType.Ws;
-}
-/**
- * Tests if a character is an identifier start character
- * @param ch Character to test
- */
-function isIdStart(ch) {
-    return (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || ch === "_" || ch === "$";
-}
-/**
- * Tests if a character is an identifier continuation character
- * @param ch Character to test
- */
-function isIdContinuation(ch) {
-    return (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z") || (ch >= "0" && ch <= "9") || ch === "_" || ch === "$";
-}
-/**
- * Tests if a character is a binary digit
- * @param ch Character to test
- */
-function isBinaryDigit(ch) {
-    return ch === "0" || ch === "1";
-}
-/**
- * Tests if a character is a decimal digit
- * @param ch Character to test
- */
-function isDecimalDigit(ch) {
-    return ch >= "0" && ch <= "9";
-}
-/**
- * Tests if a character is a hexadecimal digit
- * @param ch Character to test
- */
-function isHexadecimalDigit(ch) {
-    return (ch >= "0" && ch <= "9") || (ch >= "A" && ch <= "F") || (ch >= "a" && ch <= "f");
-}
-/**
- * Tests if a character is restricted in a string
- * @param ch Character to test
- */
-function isRestrictedInString(ch) {
-    return ch === "\r" || ch === "\n" || ch === "\u0085" || ch === "\u2028" || ch === "\u2029";
-}