skir 1.2.4 → 1.2.5

package/dist/tokenizer.js

@@ -1,244 +0,0 @@
-import { unquoteAndUnescape, } from "skir-internal";
-import { parseDocComment } from "./doc_comment_parser.js";
-/** Tokenizes the given module. */
-export function tokenizeModule(sourceCode, modulePath, completionMode) {
-    const tokens = [];
-    const errors = [];
-    const lines = new Lines(sourceCode, modulePath);
-    // Multiline comment:            \/\*([^*]|\*[^\/])*(\*\/)?
-    // Single-line comment:          \/\/[^\n\r]*
-    // Number:                       -?(0|[1-9][0-9]*)(\.[0-9]+)?\b
-    // Word:                         \b\w+\b
-    // Whitespaces:                  [ \n\r\t]+
-    // Symbol:                       \.\.|\{\||\|\}|[{}[\]()*.:=;|?,]
-    // Double-quote string literal:  "(\\(\n|\r|\n\r|.)|[^\\\"\n\r])*"?
-    // Single-quote string literal:  '(\\(\n|\r|\n\r|.)|[^\\\'\n\r])*'?
-    //
-    // To iterate on this regex, use https://regex101.com/.
-    const re = /(\/\*([^*]|\*[^/])*(\*\/)?)|(\/\/[^\n\r]*)|(-?(0|[1-9][0-9]*)(\.[0-9]+)?\b)|\b(\w+)\b|([ \n\r\t]+)|(\.\.|\{\||\|\}|[{}[\]()*.:=;|?,])|("(\\(\n|\r|\n\r|.)|[^\\"\n\r])*"?)|('(\\(\n|\r|\n\r|.)|[^\\'\n\r])*'?)|($)/g;
-    // 1    2             3        4              5  6              7                8       9            10                                 11   13                             14   16                             17
-    let group;
-    let expectedPosition = 0;
-    while ((group = re.exec(sourceCode)) !== null) {
-        const position = re.lastIndex - group[0].length;
-        // Check that all the regex matches are consecutive.
-        // If there is a gap between two matches, it means that we have an invalid
-        // sequence of characters.
-        if (position !== expectedPosition) {
-            const line = lines.advancePosition(expectedPosition);
-            const colNumber = expectedPosition - line.position;
-            const text = sourceCode.substring(expectedPosition, position);
-            errors.push({
-                token: {
-                    text: text,
-                    originalText: text,
-                    position: expectedPosition,
-                    line: line,
-                    colNumber: colNumber,
-                },
-                message: "Invalid sequence of characters",
-            });
-        }
-        expectedPosition = re.lastIndex;
-        const line = lines.advancePosition(position);
-        const colNumber = position - line.position;
-        const token = {
-            text: group[0],
-            originalText: group[0],
-            position: position,
-            line: line,
-            colNumber: colNumber,
-        };
-        if (group[1] !== undefined && !group[1].endsWith("*/")) {
-            errors.push({
-                token: token,
-                message: "Unterminated multi-line comment",
-            });
-        }
-        if (group[8] !== undefined) {
-            if (!validateWord(token, errors)) {
-                continue;
-            }
-        }
-        // Skip whitespaces.
-        if (group[9] !== undefined)
-            continue;
-        // Validate string literals.
-        const stringLiteral = group[11] || group[14];
-        if (stringLiteral !== undefined) {
-            if (!stringLiteral.endsWith(stringLiteral[0])) {
-                errors.push({
-                    token: token,
-                    message: "Unterminated string literal",
-                });
-                continue;
-            }
-            const stringLiteralRe = /^["'](\\(\r\n|u[0-9A-Fa-f]{4}|["'\\/bfnrt\n\r])|[^\\])*["']$/;
-            if (!stringLiteralRe.test(stringLiteral)) {
-                errors.push({
-                    token: token,
-                    message: "String literal contains invalid escape sequence",
-                });
-                continue;
-            }
-            // Check that the string doesn't contain lone surrogates.
-            // See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/isWellFormed
-            const unquoted = unquoteAndUnescape(stringLiteral);
-            try {
-                encodeURIComponent(unquoted);
-            }
-            catch {
-                errors.push({
-                    token: token,
-                    message: "String literal contains lone surrogates",
-                });
-                continue;
-            }
-        }
-        tokens.push(token);
-        if (group[17] !== undefined) {
-            // The end of the string has been reached.
-            // The last token we added is an empty string.
-            break;
-        }
-    }
-    if (completionMode !== undefined) {
-        const { position } = completionMode;
-        const existingToken = tokens.find((t) => {
-            const start = t.position;
-            const end = t.position + t.text.length;
-            if (/^\w/.test(t.text)) {
-                // Word token: inclusive comparisons.
-                return position >= start && position <= end;
-            }
-            else {
-                // Non-word token: exclusive comparisons.
-                return position > start && position < end;
-            }
-        });
-        if (existingToken === undefined) {
-            const fakeLines = new Lines(sourceCode, modulePath);
-            const line = fakeLines.advancePosition(position);
-            const colNumber = position - line.position;
-            const fakeToken = {
-                text: "...",
-                originalText: "",
-                position: position,
-                line: line,
-                colNumber: colNumber,
-            };
-            const insertIndex = tokens.findIndex((t) => t.position >= position);
-            if (insertIndex === -1) {
-                tokens.push(fakeToken);
-            }
-            else {
-                tokens.splice(insertIndex, 0, fakeToken);
-            }
-        }
-    }
-    // Parse the doc comments.
-    const posToDoc = {};
-    for (const token of tokens) {
-        if (!token.text.startsWith("///")) {
-            continue;
-        }
-        const docResult = parseDocComment(token, completionMode);
-        posToDoc[token.position] = docResult.result;
-        docResult.errors.forEach((e) => errors.push(e));
-    }
-    return {
-        result: {
-            tokens: tokens.filter((t) => !isRegularComment(t.originalText)),
-            tokensWithComments: tokens,
-            posToDoc: posToDoc,
-            sourceCode: sourceCode,
-            modulePath: modulePath,
-        },
-        errors: errors,
-    };
-}
-function validateWord(token, errors) {
-    if (/^[0-9]/.test(token.text)) {
-        errors.push({
-            token: token,
-            message: "Invalid number",
-        });
-        return false;
-    }
-    if (token.text.startsWith("_")) {
-        errors.push({
-            token: token,
-            message: "Identifier cannot start with _",
-        });
-        return false;
-    }
-    if (token.text.endsWith("_")) {
-        errors.push({
-            token: token,
-            message: "Identifier cannot end with _",
-        });
-        return false;
-    }
-    if (token.text.includes("__")) {
-        errors.push({
-            token: token,
-            message: "Identifier cannot contain __ sequence",
-        });
-        return false;
-    }
-    if (/_[0-9]/.test(token.text)) {
-        // The problem is we could end up with duplicate field names when converting
-        // to camel case. For example: "field0" and "field_0" would both be
-        // converted to "field0".
-        errors.push({
-            token: token,
-            message: "Digit cannot follow _",
-        });
-        return false;
-    }
-    return true;
-}
-function isRegularComment(token) {
-    return (token.startsWith("/*") ||
-        (token.startsWith("//") && !token.startsWith("///")));
-}
-class Lines {
-    /** Splits the given code into multiple lines. */
-    constructor(code, modulePath) {
-        this.lines = [];
-        const lines = this.lines;
-        let lineStart = 0;
-        let group;
-        const re = /\n|\r|\n\r|$/g;
-        while ((group = re.exec(code)) !== null) {
-            const match = group[0];
-            const lineEnd = re.lastIndex - match.length;
-            const lineString = code.substring(lineStart, lineEnd);
-            lines.push({
-                lineNumber: lines.length,
-                line: lineString,
-                position: lineStart,
-                modulePath: modulePath,
-            });
-            // Record the position after the EOL or EOF.
-            lineStart = re.lastIndex;
-            if (match === "") {
-                break;
-            }
-        }
-    }
-    /**
-     * Returns the line at the given position, where the position is the char
-     * index within `code`.
-     * Requires the position to be >= the position given the last time the method
-     * was called.
-     */
-    advancePosition(position) {
-        const lines = this.lines;
-        while (lines.length >= 2 && position >= this.lines[1].position) {
-            lines.shift();
-        }
-        return lines[0];
-    }
-}
-//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,GAOnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAe1D,kCAAkC;AAClC,MAAM,UAAU,cAAc,CAC5B,UAAkB,EAClB,UAAkB,EAClB,cAEC;IAED,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAEhD,2DAA2D;IAC3D,6CAA6C;IAC7C,+DAA+D;IAC/D,wCAAwC;IACxC,2CAA2C;IAC3C,iEAAiE;IACjE,mEAAmE;IACnE,mEAAmE;IACnE,EAAE;IACF,uDAAuD;IACvD,MAAM,EAAE,GACN,oNAAoN,CAAC;IACvN,mNAAmN;IAEnN,IAAI,KAA6B,CAAC;IAClC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAEhD,oDAAoD;QACpD,0EAA0E;QAC1E,0BAA0B;QAC1B,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,KAAK,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YACrD,MAAM,SAAS,GAAG,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC;YACnD,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,IAAI;oBACV,YAAY,EAAE,IAAI;oBAClB,QAAQ,EAAE,gBAAgB;oBAC1B,IAAI,EAAE,IAAI;oBACV,SAAS,EAAE,SAAS;iBACrB;gBACD,OAAO,EAAE,gCAAgC;aAC1C,CAAC,CAAC;QACL,CAAC;QACD,gBAAgB,GAAG,EAAE,CAAC,SAAS,CAAC;QAEhC,MAAM,IAAI,GAAG,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC3C,MAAM,KAAK,GAAU;YACnB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YACd,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC;YACtB,QAAQ,EAAE,QAAQ;YAClB,IAAI,EAAE,IAAI;YACV,SAAS,EAAE,SAAS;SACrB,CAAC;QAEF,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,iCAAiC;aAC3C,CAAC,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;YAAE,SAAS;QAErC,4BAA4B;QAC5B,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,KAAK;oBACZ,OAAO,EAAE,6BAA6B;iBACvC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,MAAM,eAAe,GACnB,8DAA8D,CAAC;YACjE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,KAAK;oBACZ,OAAO,EAAE,iDAAiD;iBAC3D,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,yDAAyD;YACzD,2GAA2G;YAC3G,MAAM,QAAQ,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;YACnD,IAAI,CAAC;gBACH,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,KAAK;oBACZ,OAAO,EAAE,yCAAyC;iBACnD,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,CAAC,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC;YAC5B,0CAA0C;YAC1C,8CAA8C;YAC9C,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC;QACpC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YACtC,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,CAAC;YACzB,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACvC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,qCAAqC;gBACrC,OAAO,QAAQ,IAAI,KAAK,IAAI,QAAQ,IAAI,GAAG,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,OAAO,QAAQ,GAAG,KAAK,IAAI,QAAQ,GAAG,GAAG,CAAC;YAC5C,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,SAAS,GAAG,IAAI,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YACpD,MAAM,IAAI,GAAG,SAAS,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC3C,MAAM,SAAS,GAAU;gBACvB,IAAI,EAAE,KAAK;gBACX,YAAY,EAAE,EAAE;gBAChB,QAAQ,EAAE,QAAQ;gBAClB,IAAI,EAAE,IAAI;gBACV,SAAS,EAAE,SAAS;aACrB,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC;YACpE,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QACzD,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;QAC5C,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YAC/D,kBAAkB,EAAE,MAAM;YAC1B,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,UAAU;YACtB,UAAU,EAAE,UAAU;SACvB;QACD,MAAM,EAAE,MAAM;KACf,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAY,EAAE,MAAiB;IACnD,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,gBAAgB;SAC1B,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,gCAAgC;SAC1C,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,8BAA8B;SACxC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,uCAAuC;SACjD,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,4EAA4E;QAC5E,mEAAmE;QACnE,yBAAyB;QACzB,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,uBAAuB;SACjC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa;IACrC,OAAO,CACL,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QACtB,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CACrD,CAAC;AACJ,CAAC;AAED,MAAM,KAAK;IACT,iDAAiD;IACjD,YAAY,IAAY,EAAE,UAAkB;QAqC3B,UAAK,GAAe,EAAE,CAAC;QApCtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAA6B,CAAC;QAClC,MAAM,EAAE,GAAG,eAAe,CAAC;QAC3B,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,OAAO,GAAG,EAAE,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,IAAI,CAAC;gBACT,UAAU,EAAE,KAAK,CAAC,MAAM;gBACxB,IAAI,EAAE,UAAU;gBAChB,QAAQ,EAAE,SAAS;gBACnB,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;YACH,4CAA4C;YAC5C,SAAS,GAAG,EAAE,CAAC,SAAS,CAAC;YACzB,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,QAAgB;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,CAAC;YAChE,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC;QACD,OAAO,KAAK,CAAC,CAAC,CAAE,CAAC;IACnB,CAAC;CAGF"}