skir 0.0.1 → 0.0.2

package/dist/tokenizer.test.js

@@ -19,285 +19,301 @@ describe("tokenizer", () => {
             "",
             "const MINUS_ONE: int32 = -1;",
             "const MINUS_ONE_AND_A_HALF: float32 = -1.5;",
+            "/// Doc comment for STRUCT",
             "const STRUCT: Point2d = {| |};",
         ].join("\n");
         const actual = tokenizeModule(code, "path/to/module");
         expect(actual).toMatch({
             errors: [],
-            result: [
-                {
-                    text: "import",
-                    position: 0,
-                    line: {
-                        lineNumber: 0,
-                        line: "import foo from './path/😊/foo';",
+            result: {
+                tokens: [
+                    {
+                        text: "import",
                         position: 0,
-                        modulePath: "path/to/module",
+                        line: {
+                            lineNumber: 0,
+                            line: "import foo from './path/😊/foo';",
+                            position: 0,
+                            modulePath: "path/to/module",
+                        },
+                        colNumber: 0,
                     },
-                    colNumber: 0,
-                },
-                {
-                    text: "foo",
-                    position: 7,
-                    colNumber: 7,
-                },
-                {
-                    text: "from",
-                    position: 11,
-                    colNumber: 11,
-                },
-                {
-                    text: "'./path/😊/foo'",
-                    position: 16,
-                    colNumber: 16,
-                },
-                {
-                    text: ";",
-                    position: 31,
-                    colNumber: 31,
-                },
-                {
-                    text: "import",
-                    position: 33,
-                    colNumber: 0,
-                },
-                {
-                    text: "*",
-                    position: 40,
-                    line: {
-                        lineNumber: 1,
-                        line: 'import * as bar from "path/to/bar";',
+                    {
+                        text: "foo",
+                        position: 7,
+                        colNumber: 7,
+                    },
+                    {
+                        text: "from",
+                        position: 11,
+                        colNumber: 11,
+                    },
+                    {
+                        text: "'./path/😊/foo'",
+                        position: 16,
+                        colNumber: 16,
+                    },
+                    {
+                        text: ";",
+                        position: 31,
+                        colNumber: 31,
+                    },
+                    {
+                        text: "import",
                         position: 33,
-                        modulePath: "path/to/module",
+                        colNumber: 0,
                     },
-                    colNumber: 7,
-                },
-                {
-                    text: "as",
-                    position: 42,
-                    colNumber: 9,
-                },
-                {
-                    text: "bar",
-                    position: 45,
-                    colNumber: 12,
-                },
-                {
-                    text: "from",
-                    position: 49,
-                    colNumber: 16,
-                },
-                {
-                    text: '"path/to/bar"',
-                    position: 54,
-                    colNumber: 21,
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "struct",
-                },
-                {
-                    text: "Point2d",
-                },
-                {
-                    text: "{",
-                },
-                {
-                    text: "x",
-                },
-                {
-                    text: ":",
-                },
-                {
-                    text: "float32",
-                },
-                {
-                    text: "=",
-                },
-                {
-                    text: "1",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "y",
-                },
-                {
-                    text: ":",
-                },
-                {
-                    text: "float32",
-                },
-                {
-                    text: "=",
-                },
-                {
-                    text: "2",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "foos",
-                },
-                {
-                    text: ":",
-                },
-                {
-                    text: "[",
-                },
-                {
-                    text: "foo",
-                },
-                {
-                    text: ".",
-                },
-                {
-                    text: "Foo",
-                },
-                {
-                    text: "]",
-                },
-                {
-                    text: "?",
-                },
-                {
-                    text: "=",
-                },
-                {
-                    text: "4",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "bars",
-                },
-                {
-                    text: ":",
-                },
-                {
-                    text: "[",
-                },
-                {
-                    text: "Bar",
-                },
-                {
-                    text: "|",
-                },
-                {
-                    text: "key",
-                },
-                {
-                    text: "]",
-                },
-                {
-                    text: "=",
-                },
-                {
-                    text: "8",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "removed",
-                },
-                {
-                    text: "3",
-                },
-                {
-                    text: ",",
-                },
-                {
-                    text: "5",
-                },
-                {
-                    text: "..",
-                },
-                {
-                    text: "7",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "}",
-                },
-                {
-                    text: "const",
-                },
-                {
-                    text: "MINUS_ONE",
-                },
-                {
-                    text: ":",
-                },
-                {
-                    text: "int32",
-                },
-                {
-                    text: "=",
-                },
-                {
-                    text: "-1",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "const",
-                },
-                {
-                    text: "MINUS_ONE_AND_A_HALF",
-                },
-                {
-                    text: ":",
-                },
-                {
-                    text: "float32",
-                },
-                {
-                    text: "=",
-                },
-                {
-                    text: "-1.5",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "const",
-                },
-                {
-                    text: "STRUCT",
-                },
-                {
-                    text: ":",
-                },
-                {
-                    text: "Point2d",
-                },
-                {
-                    text: "=",
-                },
-                {
-                    text: "{|",
-                },
-                {
-                    text: "|}",
-                },
-                {
-                    text: ";",
-                },
-                {
-                    text: "",
-                },
-            ],
+                    {
+                        text: "*",
+                        position: 40,
+                        line: {
+                            lineNumber: 1,
+                            line: 'import * as bar from "path/to/bar";',
+                            position: 33,
+                            modulePath: "path/to/module",
+                        },
+                        colNumber: 7,
+                    },
+                    {
+                        text: "as",
+                        position: 42,
+                        colNumber: 9,
+                    },
+                    {
+                        text: "bar",
+                        position: 45,
+                        colNumber: 12,
+                    },
+                    {
+                        text: "from",
+                        position: 49,
+                        colNumber: 16,
+                    },
+                    {
+                        text: '"path/to/bar"',
+                        position: 54,
+                        colNumber: 21,
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "struct",
+                    },
+                    {
+                        text: "Point2d",
+                    },
+                    {
+                        text: "{",
+                    },
+                    {
+                        text: "x",
+                    },
+                    {
+                        text: ":",
+                    },
+                    {
+                        text: "float32",
+                    },
+                    {
+                        text: "=",
+                    },
+                    {
+                        text: "1",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "y",
+                    },
+                    {
+                        text: ":",
+                    },
+                    {
+                        text: "float32",
+                    },
+                    {
+                        text: "=",
+                    },
+                    {
+                        text: "2",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "foos",
+                    },
+                    {
+                        text: ":",
+                    },
+                    {
+                        text: "[",
+                    },
+                    {
+                        text: "foo",
+                    },
+                    {
+                        text: ".",
+                    },
+                    {
+                        text: "Foo",
+                    },
+                    {
+                        text: "]",
+                    },
+                    {
+                        text: "?",
+                    },
+                    {
+                        text: "=",
+                    },
+                    {
+                        text: "4",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "bars",
+                    },
+                    {
+                        text: ":",
+                    },
+                    {
+                        text: "[",
+                    },
+                    {
+                        text: "Bar",
+                    },
+                    {
+                        text: "|",
+                    },
+                    {
+                        text: "key",
+                    },
+                    {
+                        text: "]",
+                    },
+                    {
+                        text: "=",
+                    },
+                    {
+                        text: "8",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "removed",
+                    },
+                    {
+                        text: "3",
+                    },
+                    {
+                        text: ",",
+                    },
+                    {
+                        text: "5",
+                    },
+                    {
+                        text: "..",
+                    },
+                    {
+                        text: "7",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "}",
+                    },
+                    {
+                        text: "const",
+                    },
+                    {
+                        text: "MINUS_ONE",
+                    },
+                    {
+                        text: ":",
+                    },
+                    {
+                        text: "int32",
+                    },
+                    {
+                        text: "=",
+                    },
+                    {
+                        text: "-1",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "const",
+                    },
+                    {
+                        text: "MINUS_ONE_AND_A_HALF",
+                    },
+                    {
+                        text: ":",
+                    },
+                    {
+                        text: "float32",
+                    },
+                    {
+                        text: "=",
+                    },
+                    {
+                        text: "-1.5",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "/// Doc comment for STRUCT",
+                    },
+                    {
+                        text: "const",
+                    },
+                    {
+                        text: "STRUCT",
+                    },
+                    {
+                        text: ":",
+                    },
+                    {
+                        text: "Point2d",
+                    },
+                    {
+                        text: "=",
+                    },
+                    {
+                        text: "{|",
+                    },
+                    {
+                        text: "|}",
+                    },
+                    {
+                        text: ";",
+                    },
+                    {
+                        text: "",
+                    },
+                ],
+            },
         });
+        expect(actual.result.tokensWithComments.filter((t) => t.text.startsWith("/*") ||
+            (t.text.startsWith("//") && !t.text.startsWith("///")))).toMatch([
+            {
+                text: "// Single-line comment",
+                originalText: "// Single-line comment",
+            },
+            {
+                text: ["/* Multi-line", "  comment */"].join("\n"),
+            },
+        ]);
     });
     it("tokenizes module with single-quoted string", () => {
         const code = ['const FOO: string = \'"\\\\\\""\''].join("\n");

package/dist/tokenizer.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.test.js","sourceRoot":"","sources":["../src/tokenizer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,IAAI,GAAG;YACX,kCAAkC;YAClC,qCAAqC;YACrC,EAAE;YACF,wBAAwB;YACxB,gCAAgC;YAChC,cAAc;YACd,mBAAmB;YACnB,mBAAmB;YACnB,yBAAyB;YACzB,wBAAwB;YACxB,oBAAoB;YACpB,GAAG;YACH,EAAE;YACF,8BAA8B;YAC9B,6CAA6C;YAC7C,gCAAgC;SACjC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,EAAE;YACV,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,CAAC;oBACX,IAAI,EAAE;wBACJ,UAAU,EAAE,CAAC;wBACb,IAAI,EAAE,kCAAkC;wBACxC,QAAQ,EAAE,CAAC;wBACX,UAAU,EAAE,gBAAgB;qBAC7B;oBACD,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,CAAC;oBACX,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,iBAAiB;oBACvB,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,GAAG;oBACT,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,QAAQ;oBACd,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;oBACT,QAAQ,EAAE,EAAE;oBACZ,IAAI,EAAE;wBACJ,UAAU,EAAE,CAAC;wBACb,IAAI,EAAE,qCAAqC;wBAC3C,QAAQ,EAAE,EAAE;wBACZ,UAAU,EAAE,gBAAgB;qBAC7B;oBACD,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,CAAC;iBACb;gBACD;oBACE,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,eAAe;oBACrB,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,EAAE;iBACd;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,QAAQ;iBACf;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,MAAM;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,MAAM;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,KAAK;iBACZ;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,WAAW;iBAClB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,sBAAsB;iBAC7B;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,MAAM;iBACb;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,OAAO;iBACd;gBACD;oBACE,IAAI,EAAE,QAAQ;iBACf;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,SAAS;iBAChB;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,IAAI;iBACX;gBACD;oBACE,IAAI,EAAE,GAAG;iBACV;gBACD;oBACE,IAAI,EAAE,EAAE;iBACT;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,IAAI,GAAG,CAAC,mCAAmC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,EAAE;SACX,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,IAAI,GAAG,MAAM,CAAC;QAEpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,IAAI;wBACV,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,iCAAiC;iBAC3C;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,IAAI,GAAG,aAAa,CAAC;QAE3B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,MAAM;wBACZ,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,6BAA6B;iBACvC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,IAAI,GAAG,MAAM,CAAC;QAEpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,IAAI;wBACV,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,gCAAgC;iBAC1C;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,IAAI,GAAG,sBAAsB,CAAC;QAEpC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,cAAc;wBACpB,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,iDAAiD;iBAC3D;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,IAAI,GAAG,WAAW,CAAC;QAEzB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,yCAAyC;iBACnD;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,IAAI,GAAG,UAAU,CAAC;QAExB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;gBACD;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;gBACD;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,IAAI,GAAG,gBAAgB,CAAC;QAE9B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,OAAO,EAAE,gCAAgC;iBAC1C;gBACD;oBACE,OAAO,EAAE,8BAA8B;iBACxC;gBACD;oBACE,OAAO,EAAE,uCAAuC;iBACjD;gBACD;oBACE,OAAO,EAAE,uBAAuB;iBACjC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"tokenizer.test.js","sourceRoot":"","sources":["../src/tokenizer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,IAAI,GAAG;YACX,kCAAkC;YAClC,qCAAqC;YACrC,EAAE;YACF,wBAAwB;YACxB,gCAAgC;YAChC,cAAc;YACd,mBAAmB;YACnB,mBAAmB;YACnB,yBAAyB;YACzB,wBAAwB;YACxB,oBAAoB;YACpB,GAAG;YACH,EAAE;YACF,8BAA8B;YAC9B,6CAA6C;YAC7C,4BAA4B;YAC5B,gCAAgC;SACjC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,EAAE;YACV,MAAM,EAAE;gBACN,MAAM,EAAE;oBACN;wBACE,IAAI,EAAE,QAAQ;wBACd,QAAQ,EAAE,CAAC;wBACX,IAAI,EAAE;4BACJ,UAAU,EAAE,CAAC;4BACb,IAAI,EAAE,kCAAkC;4BACxC,QAAQ,EAAE,CAAC;4BACX,UAAU,EAAE,gBAAgB;yBAC7B;wBACD,SAAS,EAAE,CAAC;qBACb;oBACD;wBACE,IAAI,EAAE,KAAK;wBACX,QAAQ,EAAE,CAAC;wBACX,SAAS,EAAE,CAAC;qBACb;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,EAAE;qBACd;oBACD;wBACE,IAAI,EAAE,iBAAiB;wBACvB,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,EAAE;qBACd;oBACD;wBACE,IAAI,EAAE,GAAG;wBACT,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,EAAE;qBACd;oBACD;wBACE,IAAI,EAAE,QAAQ;wBACd,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,CAAC;qBACb;oBACD;wBACE,IAAI,EAAE,GAAG;wBACT,QAAQ,EAAE,EAAE;wBACZ,IAAI,EAAE;4BACJ,UAAU,EAAE,CAAC;4BACb,IAAI,EAAE,qCAAqC;4BAC3C,QAAQ,EAAE,EAAE;4BACZ,UAAU,EAAE,gBAAgB;yBAC7B;wBACD,SAAS,EAAE,CAAC;qBACb;oBACD;wBACE,IAAI,EAAE,IAAI;wBACV,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,CAAC;qBACb;oBACD;wBACE,IAAI,EAAE,KAAK;wBACX,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,EAAE;qBACd;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,EAAE;qBACd;oBACD;wBACE,IAAI,EAAE,eAAe;wBACrB,QAAQ,EAAE,EAAE;wBACZ,SAAS,EAAE,EAAE;qBACd;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,QAAQ;qBACf;oBACD;wBACE,IAAI,EAAE,SAAS;qBAChB;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,SAAS;qBAChB;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,SAAS;qBAChB;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,MAAM;qBACb;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,KAAK;qBACZ;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,KAAK;qBACZ;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,MAAM;qBACb;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,KAAK;qBACZ;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,KAAK;qBACZ;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,SAAS;qBAChB;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,IAAI;qBACX;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,OAAO;qBACd;oBACD;wBACE,IAAI,EAAE,WAAW;qBAClB;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,OAAO;qBACd;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,IAAI;qBACX;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,OAAO;qBACd;oBACD;wBACE,IAAI,EAAE,sBAAsB;qBAC7B;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,SAAS;qBAChB;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,MAAM;qBACb;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,4BAA4B;qBACnC;oBACD;wBACE,IAAI,EAAE,OAAO;qBACd;oBACD;wBACE,IAAI,EAAE,QAAQ;qBACf;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,SAAS;qBAChB;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,IAAI;qBACX;oBACD;wBACE,IAAI,EAAE,IAAI;qBACX;oBACD;wBACE,IAAI,EAAE,GAAG;qBACV;oBACD;wBACE,IAAI,EAAE,EAAE;qBACT;iBACF;aACF;SACF,CAAC,CAAC;QAEH,MAAM,CACJ,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACvB,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CACzD,CACF,CAAC,OAAO,CAAC;YACR;gBACE,IAAI,EAAE,wBAAwB;gBAC9B,YAAY,EAAE,wBAAwB;aACvC;YACD;gBACE,IAAI,EAAE,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;aACnD;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,IAAI,GAAG,CAAC,mCAAmC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,EAAE;SACX,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,IAAI,GAAG,MAAM,CAAC;QAEpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,IAAI;wBACV,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,iCAAiC;iBAC3C;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,IAAI,GAAG,aAAa,CAAC;QAE3B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,MAAM;wBACZ,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,6BAA6B;iBACvC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,IAAI,GAAG,MAAM,CAAC;QAEpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,IAAI;wBACV,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,gCAAgC;iBAC1C;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,IAAI,GAAG,sBAAsB,CAAC;QAEpC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,cAAc;wBACpB,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,iDAAiD;iBAC3D;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,IAAI,GAAG,WAAW,CAAC;QAEzB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE;wBACL,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,CAAC;qBACZ;oBACD,OAAO,EAAE,yCAAyC;iBACnD;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,IAAI,GAAG,UAAU,CAAC;QAExB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;gBACD;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;gBACD;oBACE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;oBACrB,OAAO,EAAE,gBAAgB;iBAC1B;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,IAAI,GAAG,gBAAgB,CAAC;QAE9B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE;gBACN;oBACE,OAAO,EAAE,gCAAgC;iBAC1C;gBACD;oBACE,OAAO,EAAE,8BAA8B;iBACxC;gBACD;oBACE,OAAO,EAAE,uCAAuC;iBACjD;gBACD;oBACE,OAAO,EAAE,uBAAuB;iBACjC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skir",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "",
   "homepage": "https://github.com/gepheum/skir#readme",
   "bugs": {
@@ -42,22 +42,24 @@
   },
   "dependencies": {
     "glob": "^11.0.1",
+    "skir-internal": "^0.0.3",
     "watcher": "^2.3.1",
     "yaml": "^2.7.1",
-    "zod": "^3.24.3",
-    "zod-validation-error": "^3.4.0"
+    "zod": "^4.1.13",
+    "zod-validation-error": "^5.0.0"
   },
   "devDependencies": {
     "@types/mocha": "^10.0.1",
     "@types/node": "^20.6.0",
     "@typescript-eslint/eslint-plugin": "^6.13.0",
     "@typescript-eslint/parser": "^6.13.0",
-    "eslint": "^8.54.0",
-    "prettier-plugin-organize-imports": "^4.2.0",
     "buckwheat": "^1.1.2",
+    "eslint": "^8.54.0",
     "mocha": "^10.2.0",
     "prettier": "^3.2.4",
+    "prettier-plugin-organize-imports": "^4.2.0",
     "ts-node": "^10.9.2",
+    "tsx": "^4.21.0",
     "typescript": "^5.2.2"
   }
 }