skir 0.0.1

package/dist/tokenizer.test.js

@@ -0,0 +1,425 @@
+import { expect } from "buckwheat";
+import { describe, it } from "mocha";
+import { tokenizeModule } from "./tokenizer.js";
+describe("tokenizer", () => {
+    it("tokenizes module with simple struct", () => {
+        const code = [
+            "import foo from './path/😊/foo';",
+            'import * as bar from "path/to/bar";',
+            "",
+            "// Single-line comment",
+            "struct Point2d { /* Multi-line",
+            "  comment */",
+            "  x: float32 = 1;",
+            "  y: float32 = 2;",
+            "  foos: [foo.Foo]? = 4;",
+            "  bars: [Bar|key] = 8;",
+            "  removed 3, 5..7;",
+            "}",
+            "",
+            "const MINUS_ONE: int32 = -1;",
+            "const MINUS_ONE_AND_A_HALF: float32 = -1.5;",
+            "const STRUCT: Point2d = {| |};",
+        ].join("\n");
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [],
+            result: [
+                {
+                    text: "import",
+                    position: 0,
+                    line: {
+                        lineNumber: 0,
+                        line: "import foo from './path/😊/foo';",
+                        position: 0,
+                        modulePath: "path/to/module",
+                    },
+                    colNumber: 0,
+                },
+                {
+                    text: "foo",
+                    position: 7,
+                    colNumber: 7,
+                },
+                {
+                    text: "from",
+                    position: 11,
+                    colNumber: 11,
+                },
+                {
+                    text: "'./path/😊/foo'",
+                    position: 16,
+                    colNumber: 16,
+                },
+                {
+                    text: ";",
+                    position: 31,
+                    colNumber: 31,
+                },
+                {
+                    text: "import",
+                    position: 33,
+                    colNumber: 0,
+                },
+                {
+                    text: "*",
+                    position: 40,
+                    line: {
+                        lineNumber: 1,
+                        line: 'import * as bar from "path/to/bar";',
+                        position: 33,
+                        modulePath: "path/to/module",
+                    },
+                    colNumber: 7,
+                },
+                {
+                    text: "as",
+                    position: 42,
+                    colNumber: 9,
+                },
+                {
+                    text: "bar",
+                    position: 45,
+                    colNumber: 12,
+                },
+                {
+                    text: "from",
+                    position: 49,
+                    colNumber: 16,
+                },
+                {
+                    text: '"path/to/bar"',
+                    position: 54,
+                    colNumber: 21,
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "struct",
+                },
+                {
+                    text: "Point2d",
+                },
+                {
+                    text: "{",
+                },
+                {
+                    text: "x",
+                },
+                {
+                    text: ":",
+                },
+                {
+                    text: "float32",
+                },
+                {
+                    text: "=",
+                },
+                {
+                    text: "1",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "y",
+                },
+                {
+                    text: ":",
+                },
+                {
+                    text: "float32",
+                },
+                {
+                    text: "=",
+                },
+                {
+                    text: "2",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "foos",
+                },
+                {
+                    text: ":",
+                },
+                {
+                    text: "[",
+                },
+                {
+                    text: "foo",
+                },
+                {
+                    text: ".",
+                },
+                {
+                    text: "Foo",
+                },
+                {
+                    text: "]",
+                },
+                {
+                    text: "?",
+                },
+                {
+                    text: "=",
+                },
+                {
+                    text: "4",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "bars",
+                },
+                {
+                    text: ":",
+                },
+                {
+                    text: "[",
+                },
+                {
+                    text: "Bar",
+                },
+                {
+                    text: "|",
+                },
+                {
+                    text: "key",
+                },
+                {
+                    text: "]",
+                },
+                {
+                    text: "=",
+                },
+                {
+                    text: "8",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "removed",
+                },
+                {
+                    text: "3",
+                },
+                {
+                    text: ",",
+                },
+                {
+                    text: "5",
+                },
+                {
+                    text: "..",
+                },
+                {
+                    text: "7",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "}",
+                },
+                {
+                    text: "const",
+                },
+                {
+                    text: "MINUS_ONE",
+                },
+                {
+                    text: ":",
+                },
+                {
+                    text: "int32",
+                },
+                {
+                    text: "=",
+                },
+                {
+                    text: "-1",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "const",
+                },
+                {
+                    text: "MINUS_ONE_AND_A_HALF",
+                },
+                {
+                    text: ":",
+                },
+                {
+                    text: "float32",
+                },
+                {
+                    text: "=",
+                },
+                {
+                    text: "-1.5",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "const",
+                },
+                {
+                    text: "STRUCT",
+                },
+                {
+                    text: ":",
+                },
+                {
+                    text: "Point2d",
+                },
+                {
+                    text: "=",
+                },
+                {
+                    text: "{|",
+                },
+                {
+                    text: "|}",
+                },
+                {
+                    text: ";",
+                },
+                {
+                    text: "",
+                },
+            ],
+        });
+    });
+    it("tokenizes module with single-quoted string", () => {
+        const code = ['const FOO: string = \'"\\\\\\""\''].join("\n");
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [],
+        });
+    });
+    it("tokenizes module with unterminated multi-line comment", () => {
+        const code = "  /*";
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [
+                {
+                    token: {
+                        text: "/*",
+                        position: 2,
+                    },
+                    message: "Unterminated multi-line comment",
+                },
+            ],
+        });
+    });
+    it("tokenizes module with unterminated string literal", () => {
+        const code = "import 'foo";
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [
+                {
+                    token: {
+                        text: "'foo",
+                        position: 7,
+                    },
+                    message: "Unterminated string literal",
+                },
+            ],
+        });
+    });
+    it("tokenizes module with invalid char sequence", () => {
+        const code = "  ##";
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [
+                {
+                    token: {
+                        text: "##",
+                        position: 2,
+                    },
+                    message: "Invalid sequence of characters",
+                },
+            ],
+        });
+    });
+    it("tokenizes module with invalid escape sequence in string literal", () => {
+        const code = "import 'foo\\u0ffg';";
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [
+                {
+                    token: {
+                        text: "'foo\\u0ffg'",
+                        position: 7,
+                    },
+                    message: "String literal contains invalid escape sequence",
+                },
+            ],
+        });
+    });
+    it("tokenizes module with lone surrogate in string literal", () => {
+        const code = "'\uD800a'";
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [
+                {
+                    token: {
+                        text: "'\ud800a'",
+                        position: 0,
+                    },
+                    message: "String literal contains lone surrogates",
+                },
+            ],
+        });
+    });
+    it("tokenizes module with invalid word", () => {
+        const code = "00 1_ 2a";
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [
+                {
+                    token: { text: "00" },
+                    message: "Invalid number",
+                },
+                {
+                    token: { text: "1_" },
+                    message: "Invalid number",
+                },
+                {
+                    token: { text: "2a" },
+                    message: "Invalid number",
+                },
+            ],
+        });
+    });
+    it("tokenizes module with invalid identifiers", () => {
+        const code = "_a a_ a__a a_0";
+        const actual = tokenizeModule(code, "path/to/module");
+        expect(actual).toMatch({
+            errors: [
+                {
+                    message: "Identifier cannot start with _",
+                },
+                {
+                    message: "Identifier cannot end with _",
+                },
+                {
+                    message: "Identifier cannot contain __ sequence",
+                },
+                {
+                    message: "Digit cannot follow _",
+                },
+            ],
+        });
+    });
+});
+//# sourceMappingURL=tokenizer.test.js.map

package/dist/tokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.test.js","sourceRoot":"","sources":["../src/tokenizer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,IAAI,GAAG;YACX,kCAAkC;YAClC,qCAAqC;YACrC,EAAE;YACF,wBAAwB;YACxB,gCAAgC;YAChC,cAAc;YACd,mBAAmB;YACnB,mBAAmB;YACnB,yBAAyB;YACzB,wBAAwB;YACxB,oBAAoB;YACpB,GAAG;YACH,EAAE;YACF,8BAA8B;YAC9B,6CAA6C;YAC7C,gCAAgC;SACjC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,EAAE;YACV,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,CAAC;oBACX,IAAI,EAAE;wBACJ,UAAU,EAAE,CAAC;wBACb,IAAI,EAAE,kCAAkC;wBACxC,QAAQ,EAAE,CAAC;wBACX,UAAU,EAAE,gBAAgB;qBAC7B;oBACD,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,CAAC;oBACX,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,iBAAiB;oBACvB,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,GAAG;oBACT,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;oBACT,QAAQ,EAAE,EAAE;oBACZ,IAAI,EAAE;wBACJ,UAAU,EAAE,CAAC;wBACb,IAAI,EAAE,qCAAqC;wBAC3C,QAAQ,EAAE,EAAE;wBACZ,UAAU,EAAE,gBAAgB;qBAC7B;oBACD,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,eAAe;oBACrB,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,QAAQ;iBACf;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,MAAM;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,MAAM;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,WAAW;iBAClB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,sBAAsB;iBAC7B;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,MAAM;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,QAAQ;iBACf;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,EAAE;iBACT;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,IAAI,GAAG,CAAC,mCAAmC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,EAAE;SACX,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,IAAI,GAAG,MAAM,CAAC;QAEpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,IAAI;wBACV,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,iCAAiC;iBAC3C;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,IAAI,GAAG,aAAa,CAAC;QAE3B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,MAAM;wBACZ,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,6BAA6B;iBACvC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,IAAI,GAAG,MAAM,CAAC;QAEpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,IAAI;wBACV,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,gCAAgC;iBAC1C;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,IAAI,GAAG,sBAAsB,CAAC;QAEpC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,cAAc;wBACpB,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,iDAAiD;iBAC3D;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,IAAI,GAAG,WAAW,CAAC;QAEzB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,yCAAyC;iBACnD;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,IAAI,GAAG,UAAU,CAAC;QAExB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;gBACD;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;gBACD;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,IAAI,GAAG,gBAAgB,CAAC;QAE9B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,OAAO,EAAE,gCAAgC;iBAC1C;gBACD;oBACE,OAAO,EAAE,8BAA8B;iBACxC;gBACD;oBACE,OAAO,EAAE,uCAAuC;iBACjD;gBACD;oBACE,OAAO,EAAE,uBAAuB;iBACjC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}