rawsql-ts 0.1.0-beta.1 → 0.1.0-beta.11

package/dist/utils/stringUtils.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Utilities for string operations during tokenization
+ */
+export declare class StringUtils {
+    /**
+     * Creates a visual representation of an error position in text
+     * @param input The input text
+     * @param errPosition The error position
+     * @returns A string with a caret pointing to the error position
+     */
+    static getDebugPositionInfo(input: string, errPosition: number): string;
+    /**
+     * Skip white space characters.
+     */
+    private static skipWhiteSpace;
+    /**
+     * Skip line comment.
+     */
+    private static readLineComment;
+    /**
+     * Skip block comment.
+     */
+    private static readBlockComment;
+    /**
+     * Skip white space characters and SQL comments.
+     * @returns Object containing the new position and an array of skipped comments
+     */
+    static readWhiteSpaceAndComment(input: string, position: number): {
+        position: number;
+        lines: string[];
+    };
+    /**
+     * Read a regular identifier.
+     */
+    static readRegularIdentifier(input: string, position: number): {
+        identifier: string;
+        newPosition: number;
+    };
+    static tryReadRegularIdentifier(input: string, position: number): {
+        identifier: string;
+        newPosition: number;
+    } | null;
+}

package/dist/utils/stringUtils.js

@@ -0,0 +1,168 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StringUtils = void 0;
+const charLookupTable_1 = require("./charLookupTable");
+/**
+ * Utilities for string operations during tokenization
+ */
+class StringUtils {
+    /**
+     * Creates a visual representation of an error position in text
+     * @param input The input text
+     * @param errPosition The error position
+     * @returns A string with a caret pointing to the error position
+     */
+    static getDebugPositionInfo(input, errPosition) {
+        // Get 5 characters before and after the error
+        // If the start and end points are out of the string range, keep them within the range
+        // Display ^ at the error position on the next line
+        const start = Math.max(0, errPosition - 5);
+        const end = Math.min(input.length, errPosition + 5);
+        const debugInfo = input.slice(start, end);
+        const caret = ' '.repeat(errPosition - start) + '^';
+        return `${debugInfo}\n${caret}`;
+    }
+    /**
+     * Skip white space characters.
+     */
+    static skipWhiteSpace(input, position) {
+        const length = input.length;
+        while (position < length) {
+            const charCode = input.charCodeAt(position);
+            // ' '=32, '\t'=9, '\n'=10, '\r'=13
+            if (charCode !== 32 && charCode !== 9 && charCode !== 10 && charCode !== 13) {
+                break;
+            }
+            position++;
+        }
+        return position;
+    }
+    /**
+     * Skip line comment.
+     */
+    static readLineComment(input, position) {
+        if (position + 1 >= input.length) {
+            return { newPosition: position, comment: null };
+        }
+        // '-'=45
+        if (input.charCodeAt(position) === 45 && input.charCodeAt(position + 1) === 45) {
+            const start = position;
+            position += 2;
+            // '\n'=10
+            while (position < input.length && input.charCodeAt(position) !== 10) {
+                position++;
+            }
+            // Return the trimmed comment content (excluding -- tokens)
+            const comment = input.slice(start + 2, position).trim();
+            return { newPosition: position, comment };
+        }
+        return { newPosition: position, comment: null };
+    }
+    /**
+     * Skip block comment.
+     */
+    static readBlockComment(input, position) {
+        if (position + 3 >= input.length) {
+            return { newPosition: position, comments: null };
+        }
+        // '/'=47, '*'=42, '+'=43
+        if (input.charCodeAt(position) === 47 && input.charCodeAt(position + 1) === 42 && input.charCodeAt(position + 2) !== 43) {
+            const start = position;
+            position += 2;
+            while (position + 1 < input.length) {
+                // '*'=42, '/'=47
+                if (input.charCodeAt(position) === 42 && input.charCodeAt(position + 1) === 47) {
+                    position += 2;
+                    // Process the comment content
+                    const lines = input.slice(start + 2, position - 2).replace(/\r/g, '').split('\n');
+                    for (let i = 0; i < lines.length; i++) {
+                        lines[i] = lines[i].trim();
+                    }
+                    // Remove empty lines, but only at the beginning and end
+                    while (lines.length > 0 && lines[0] === '') {
+                        lines.shift();
+                    }
+                    while (lines.length > 0 && lines[lines.length - 1] === '') {
+                        lines.pop();
+                    }
+                    return { newPosition: position, comments: lines };
+                }
+                position++;
+            }
+            throw new Error(`Block comment is not closed. position: ${position}`);
+        }
+        return { newPosition: position, comments: null };
+    }
+    /**
+     * Skip white space characters and SQL comments.
+     * @returns Object containing the new position and an array of skipped comments
+     */
+    static readWhiteSpaceAndComment(input, position) {
+        const lines = [];
+        const length = input.length;
+        while (position < length) {
+            // Store current position
+            const oldPosition = position;
+            // Skip whitespace first
+            position = StringUtils.skipWhiteSpace(input, position);
+            if (position !== oldPosition) {
+                continue;
+            }
+            // Fast character code check
+            const charCode = input.charCodeAt(position);
+            // '-'=45 (Line comment)
+            if (charCode === 45) {
+                const lineCommentResult = StringUtils.readLineComment(input, position);
+                if (lineCommentResult.newPosition !== position) {
+                    position = lineCommentResult.newPosition;
+                    if (lineCommentResult.comment) {
+                        lines.push(lineCommentResult.comment.trim());
+                    }
+                    continue;
+                }
+            }
+            // '/'=47 (Block comment)
+            else if (charCode === 47) {
+                const blockCommentResult = StringUtils.readBlockComment(input, position);
+                if (blockCommentResult.newPosition !== position) {
+                    position = blockCommentResult.newPosition;
+                    if (blockCommentResult.comments) {
+                        lines.push(...blockCommentResult.comments);
+                    }
+                    continue;
+                }
+            }
+            // No more whitespace or comments found
+            break;
+        }
+        return { position, lines: lines };
+    }
+    /**
+     * Read a regular identifier.
+     */
+    static readRegularIdentifier(input, position) {
+        const result = this.tryReadRegularIdentifier(input, position);
+        if (!result) {
+            throw new Error(`Unexpected character. position: ${position}\n${StringUtils.getDebugPositionInfo(input, position)}`);
+        }
+        return result;
+    }
+    static tryReadRegularIdentifier(input, position) {
+        const start = position;
+        while (position < input.length) {
+            if (charLookupTable_1.CharLookupTable.isDelimiter(input[position])) {
+                break;
+            }
+            position++;
+        }
+        if (start === position) {
+            return null;
+        }
+        return {
+            identifier: input.slice(start, position),
+            newPosition: position
+        };
+    }
+}
+exports.StringUtils = StringUtils;
+//# sourceMappingURL=stringUtils.js.map

package/dist/utils/stringUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stringUtils.js","sourceRoot":"","sources":["../../src/utils/stringUtils.ts"],"names":[],"mappings":";;;AAAA,uDAAoD;AAEpD;;GAEG;AACH,MAAa,WAAW;IACpB;;;;;OAKG;IACI,MAAM,CAAC,oBAAoB,CAAC,KAAa,EAAE,WAAmB;QACjE,8CAA8C;QAC9C,sFAAsF;QACtF,mDAAmD;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC;QACpD,OAAO,GAAG,SAAS,KAAK,KAAK,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,cAAc,CAAC,KAAa,EAAE,QAAgB;QACzD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAE5B,OAAO,QAAQ,GAAG,MAAM,EAAE,CAAC;YACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC5C,mCAAmC;YACnC,IAAI,QAAQ,KAAK,EAAE,IAAI,QAAQ,KAAK,CAAC,IAAI,QAAQ,KAAK,EAAE,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;gBAC1E,MAAM;YACV,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,eAAe,CAAC,KAAa,EAAE,QAAgB;QAC1D,IAAI,QAAQ,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACpD,CAAC;QAED,SAAS;QACT,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YAC7E,MAAM,KAAK,GAAG,QAAQ,CAAC;YACvB,QAAQ,IAAI,CAAC,CAAC;YAEd,UAAU;YACV,OAAO,QAAQ,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;gBAClE,QAAQ,EAAE,CAAC;YACf,CAAC;YAED,2DAA2D;YAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YACxD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC9C,CAAC;QACD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACpD,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,gBAAgB,CAAC,KAAa,EAAE,QAAgB;QAC3D,IAAI,QAAQ,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACrD,CAAC;QAED,yBAAyB;QACzB,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YACtH,MAAM,KAAK,GAAG,QAAQ,CAAC;YACvB,QAAQ,IAAI,CAAC,CAAC;YAEd,OAAO,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjC,iBAAiB;gBACjB,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;oBAC7E,QAAQ,IAAI,CAAC,CAAC;oBAEd,8BAA8B;oBAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBACpC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC/B,CAAC;oBAED,wDAAwD;oBACxD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;wBACzC,KAAK,CAAC,KAAK,EAAE,CAAC;oBAClB,CAAC;oBACD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;wBACxD,KAAK,CAAC,GAAG,EAAE,CAAC;oBAChB,CAAC;oBAED,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;gBACtD,CAAC;gBACD,QAAQ,EAAE,CAAC;YACf,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACrD,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,wBAAwB,CAAC,KAAa,EAAE,QAAgB;QAClE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAE5B,OAAO,QAAQ,GAAG,MAAM,EAAE,CAAC;YACvB,yBAAyB;YACzB,MAAM,WAAW,GAAG,QAAQ,CAAC;YAE7B,wBAAwB;YACxB,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACvD,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC3B,SAAS;YACb,CAAC;YAED,4BAA4B;YAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAE5C,wBAAwB;YACxB,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;gBAClB,MAAM,iBAAiB,GAAG,WAAW,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACvE,IAAI,iBAAiB,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;oBAC7C,QAAQ,GAAG,iBAAiB,CAAC,WAAW,CAAC;oBACzC,IAAI,iBAAiB,CAAC,OAAO,EAAE,CAAC;wBAC5B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;oBACjD,CAAC;oBACD,SAAS;gBACb,CAAC;YACL,CAAC;YACD,yBAAyB;iBACpB,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;gBACvB,MAAM,kBAAkB,GAAG,WAAW,CAAC,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACzE,IAAI,kBAAkB,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;oBAC9C,QAAQ,GAAG,kBAAkB,CAAC,WAAW,CAAC;oBAC1C,IAAI,kBAAkB,CAAC,QAAQ,EAAE,CAAC;wBAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;oBAC/C,CAAC;oBACD,SAAS;gBACb,CAAC;YACL,CAAC;YAED,uCAAuC;YACvC,MAAM;QACV,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,qBAAqB,CAAC,KAAa,EAAE,QAAgB;QAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE9D,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,KAAK,WAAW,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzH,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEM,MAAM,CAAC,wBAAwB,CAAC,KAAa,EAAE,QAAgB;QAClE,MAAM,KAAK,GAAG,QAAQ,CAAC;QAEvB,OAAO,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,iCAAe,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/C,MAAM;YACV,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;QAED,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO;YACH,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC;YACxC,WAAW,EAAE,QAAQ;SACxB,CAAC;IACN,CAAC;CACJ;AAzLD,kCAyLC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "rawsql-ts",
-    "version": "0.1.0-beta.1",
+    "version": "0.1.0-beta.11",
     "description": "[beta]High-performance SQL parser and AST analyzer written in TypeScript. Provides fast parsing and advanced transformation capabilities.",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -13,9 +13,18 @@
         "lint": "eslint . --ext .ts,.tsx --fix",
         "benchmark": "ts-node benchmarks/parse-benchmark.ts"
     },
-    "keywords": [],
-    "author": "",
-    "license": "ISC",
+    "keywords": [
+        "sql",
+        "sql-parser",
+        "query-translator",
+        "ast-analysis"
+    ],
+    "author": "msugiura",
+    "license": "MIT",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/mk3008/rawsql-ts.git"
+    },
     "devDependencies": {
         "@types/benchmark": "^2.1.5",
         "@types/node": "^22.13.10",
@@ -28,11 +37,13 @@
         "eslint-plugin-prettier": "^5.2.3",
         "microtime": "^3.1.1",
         "node-sql-parser": "^5.3.8",
+        "rawsql-ts": "^0.1.0-beta.10",
+        "sql-formatter": "^15.5.2",
         "ts-node": "^10.9.2",
         "typescript": "^5.8.2",
         "vitest": "^1.5.2"
     },
-    "dependencies": {
-        "sql-formatter": "^15.5.2"
-    }
+    "files": [
+        "dist"
+    ]
 }

package/.vs/CopilotSnapshots/02E79E48DFF5234787CB618DE990598A/48814CCF3E4E4B44A66AB8BC7D6CF6B6/904140CF7EDF0C98AEA45F0D36062FE1

@@ -1 +0,0 @@
-

package/.vs/CopilotSnapshots/02E79E48DFF5234787CB618DE990598A/48814CCF3E4E4B44A66AB8BC7D6CF6B6/E621434FB1D5415B14FBBCC11967E420

@@ -1,34 +0,0 @@
-/**
- * Fast character classification utilities for SQL tokenization
- */
-export class CharLookupTable {
-    private static readonly WHITESPACE = new Set([' ', '\t', '\r', '\n']);
-    private static readonly DIGITS = new Set(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']);
-    private static readonly HEX_CHARS = new Set([
-        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-        'a', 'b', 'c', 'd', 'e', 'f',
-        'A', 'B', 'C', 'D', 'E', 'F'
-    ]);
-    private static readonly DELIMITERS = new Set([' ', '.', ',', '(', ')', '+', '-', '*', '/']);
-    private static readonly NAMED_PARAMETER_PREFIX = new Set(['@', ':', '$']);
-    
-    public static isWhitespace(char: string): boolean {
-        return CharLookupTable.WHITESPACE.has(char);
-    }
-    
-    public static isDigit(char: string): boolean {
-        return CharLookupTable.DIGITS.has(char);
-    }
-
-    public static isHexChar(char: string): boolean {
-        return CharLookupTable.HEX_CHARS.has(char);
-    }
-    
-    public static isDelimiter(char: string): boolean {
-        return CharLookupTable.DELIMITERS.has(char);
-    }
-    
-    public static isNamedParameterPrefix(char: string): boolean {
-        return CharLookupTable.NAMED_PARAMETER_PREFIX.has(char);
-    }
-}

package/.vs/CopilotSnapshots/02E79E48DFF5234787CB618DE990598A/48814CCF3E4E4B44A66AB8BC7D6CF6B6/EF71788BE39818EEA3F0164DC15E1BD7

@@ -1,170 +0,0 @@
-import { Lexeme } from './models/Lexeme';
-import { IdentifierTokenReader } from './tokenReaders/IdentifierTokenReader';
-import { LiteralTokenReader } from './tokenReaders/LiteralTokenReader';
-import { ParameterTokenReader } from './tokenReaders/ParameterTokenReader';
-import { SymbolTokenReader } from './tokenReaders/SymbolTokenReader';
-
-export class SqlTokenizer {
-    /// <summary>
-    /// Input string.
-    /// </summary>
-    private input: string;
-
-    /// <summary>
-    /// Current position in the input string.
-    /// </summary>
-    private position: number;
-
-    constructor(input: string) {
-        this.input = input;
-        this.position = 0;
-    }
-
-    private isEndOfInput(shift: number = 0): boolean {
-        return this.position + shift >= this.input.length;
-    }
-
-    private canRead(shift: number = 0): boolean {
-        return !this.isEndOfInput(shift);
-    }
-
-    public readLexmes(): Lexeme[] {
-        const lexemes: Lexeme[] = [];
-
-        // Skip whitespace and comments at the start
-        this.skipWhiteSpacesAndComments();
-
-        // Initialize each token reader
-        const symbolReader = new SymbolTokenReader(this.input, this.position);
-        const identifierReader = new IdentifierTokenReader(this.input, this.position);
-        const literalReader = new LiteralTokenReader(this.input, this.position);
-        const parameterReader = new ParameterTokenReader(this.input, this.position);
-
-        // Track the previous token
-        let previous: Lexeme | null = null;
-
-        // Read tokens until the end of the input is reached
-        while (this.canRead()) {
-            let lexeme: Lexeme | null = null;
-
-            // Attempt to read with each reader in order
-            parameterReader.setPosition(this.position);
-            lexeme = parameterReader.tryRead(previous);
-
-            if (!lexeme) {
-                literalReader.setPosition(this.position);
-                lexeme = literalReader.tryRead(previous);
-            }
-
-            if (!lexeme) {
-                identifierReader.setPosition(this.position);
-                lexeme = identifierReader.tryRead(previous);
-            }
-
-            if (!lexeme) {
-                symbolReader.setPosition(this.position);
-                lexeme = symbolReader.tryRead(previous);
-            }
-
-            // If a token is read by any reader
-            if (lexeme) {
-                lexemes.push(lexeme);
-                previous = lexeme;
-
-                // Update position
-                const newPos = Math.max(
-                    symbolReader.getPosition(),
-                    literalReader.getPosition(),
-                    identifierReader.getPosition(),
-                    parameterReader.getPosition()
-                );
-                this.position = newPos;
-
-                // Skip whitespace and comments after the token
-                this.skipWhiteSpacesAndComments();
-            } else {
-                // Exception
-                throw new Error(`Unexpected character. actual: ${this.input[this.position]}, position: ${this.position}`);
-            }
-        }
-
-        return lexemes;
-    }
-
-    /// <summary>
-    /// Skip white space characters and sql comments.
-    /// </summary>
-    private skipWhiteSpacesAndComments(): void {
-        while (true) {
-            if (this.skipWhiteSpace()) {
-                continue;
-            }
-            if (this.skipLineComment()) {
-                continue;
-            }
-            if (this.skipBlockComment()) {
-                continue;
-            }
-            break;
-        }
-    }
-
-    private skipWhiteSpace(): boolean {
-        const start = this.position;
-
-        // Skip tab, newline, and space characters
-        const whitespace = new Set([' ', '\r', '\n', '\t']);
-
-        while (this.canRead()) {
-            if (!whitespace.has(this.input[this.position])) {
-                break;
-            }
-            this.position++;
-        }
-        return start !== this.position;
-    }
-
-    private skipLineComment(): boolean {
-        // At least 2 characters are needed. '--'
-        if (this.isEndOfInput(1)) {
-            return false;
-        }
-
-        if (this.input[this.position] === '-' && this.input[this.position + 1] === '-') {
-            this.position += 2;
-
-            while (this.canRead() && this.input[this.position] !== '\n') {
-                this.position++;
-            }
-            return true;
-        }
-
-        return false;
-    }
-
-    private skipBlockComment(): boolean {
-        // At least 4 characters are needed. '/**/'
-        if (this.isEndOfInput(3)) {
-            return false;
-        }
-
-        // Record the start position of the comment to track error location
-        const start = this.position;
-
-        if (this.input[this.position] === '/' && this.input[this.position + 1] === '*') {
-            this.position += 2;
-
-            while (this.canRead(1)) {
-                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
-                    this.position += 2;
-                    return true;
-                }
-                this.position++;
-            }
-
-            throw new Error(`Block comment is not closed. position: ${start}`);
-        }
-
-        return false;
-    }
-}

package/.vs/CopilotSnapshots/02E79E48DFF5234787CB618DE990598A/state.mpack

@@ -1 +0,0 @@
-��SessionId��$489ee702-f5df-4723-87cb-618de990598a��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$cf4c8148-4e3e-444b-a66a-b8bc7d6cf6b6�Snapshots�� E621434FB1D5415B14FBBCC11967E420��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\02E79E48DFF5234787CB618DE990598A\48814CCF3E4E4B44A66AB8BC7D6CF6B6\E621434FB1D5415B14FBBCC11967E420�OriginalFilePath�FC:\Users\mssgm\Documents\GitHub\ts-sample\src\utils\charLookupTable.ts�NewFileMarker�� 904140CF7EDF0C98AEA45F0D36062FE1��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\02E79E48DFF5234787CB618DE990598A\48814CCF3E4E4B44A66AB8BC7D6CF6B6\904140CF7EDF0C98AEA45F0D36062FE1�OriginalFilePath�JC:\Users\mssgm\Documents\GitHub\ts-sample\src\utils\tokenReaderRegistry.ts�NewFileMarker�� A32D1DB906C73FA74EF270AAEE165FC0��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\02E79E48DFF5234787CB618DE990598A\48814CCF3E4E4B44A66AB8BC7D6CF6B6\A32D1DB906C73FA74EF270AAEE165FC0�OriginalFilePath�MC:\Users\mssgm\Documents\GitHub\ts-sample\src\tokenReaders\BaseTokenReader.ts�NewFileMarker�� EF71788BE39818EEA3F0164DC15E1BD7��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\02E79E48DFF5234787CB618DE990598A\48814CCF3E4E4B44A66AB8BC7D6CF6B6\EF71788BE39818EEA3F0164DC15E1BD7�OriginalFilePath�=C:\Users\mssgm\Documents\GitHub\ts-sample\src\sqlTokenizer.ts�NewFileMarker�