rawsql-ts 0.1.0-beta.1

package/.vs/CopilotSnapshots/3BACC9346120824DB30006E353477163/87C831E9EA1AA249A25A44D08CF8E0B5/EF71788BE39818EEA3F0164DC15E1BD7

@@ -0,0 +1,328 @@
+import { TokenType } from './enums/tokenType';
+
+interface Lexeme {
+    type: TokenType;
+    value: string;
+    command?: string; // Add the command property
+}
+
+export class SqlTokenizer {
+    /// <summary>
+    /// Input string.
+    /// </summary>
+    private input: string;
+
+    /// <summary>
+    /// Current position in the input string.
+    /// </summary>
+    private position: number;
+
+    constructor(input: string) {
+        this.input = input;
+        this.position = 0;
+    }
+
+    private isEndOfInput(shift: number = 0): boolean {
+        return this.position + shift >= this.input.length;
+    }
+
+    private canRead(shift: number = 0): boolean {
+        return !this.isEndOfInput(shift);
+    }
+
+    private read(expectChar: string): string {
+        if (this.isEndOfInput()) {
+            throw new Error(`Unexpected character. expect: ${expectChar}, actual: EndOfInut, position: ${this.position}`);
+        }
+
+        if (this.input[this.position] !== expectChar) {
+            throw new Error(`Unexpected character. expect: ${expectChar}, actual: ${this.input[this.position]}, position: ${this.position}`);
+        }
+
+        var char = this.input[this.position];
+        this.position++;
+        return char;
+    }
+
+    private static readonly SYMBOL_TOKENS: Record<string, TokenType> = {
+        '.': TokenType.Dot,
+        ',': TokenType.Comma,
+        '(': TokenType.OpenParen,
+        ')': TokenType.CloseParen,
+    };
+
+
+    public readLexmes(): Lexeme[] {
+        const lexemes: Lexeme[] = [];
+
+        let lexeme: Lexeme | null = null;
+        while ((lexeme = this.readLexme(lexeme)) !== null) {
+            lexemes.push(lexeme);
+        }
+        return lexemes;
+    }
+
+    public readLexme(previous: Lexeme | null = null): Lexeme | null {
+        // end of input
+        if (this.isEndOfInput()) {
+            return null;
+        }
+
+        this.skipWhiteSpacesAndComments();
+
+        // current character
+        const char = this.input[this.position];
+
+        // digit tokens (prioritize dot for decimal point and delimiter)
+        if (char === '.' && this.canRead(1) && /[0-9]/.test(this.input[this.position + 1])) {
+            return { type: TokenType.Literal, value: this.readDigit() };
+        }
+
+        // symbol tokens
+        if (char in SqlTokenizer.SYMBOL_TOKENS) {
+            this.position++;
+            return { type: SqlTokenizer.SYMBOL_TOKENS[char], value: char, command: char.toLowerCase() };
+        }
+
+        // MySQL escaped identifier (escape character is backtick)
+        if (char === '`') {
+            this.position++;
+            const identifier = this.readIdentifier();
+            this.read('`');
+            return { type: TokenType.Identifier, value: identifier, command: identifier.toLowerCase() };
+        }
+
+        // Postgres escaped identifier (escape character is double quote)
+        if (char === '"') {
+            this.position++;
+            const identifier = this.readIdentifier();
+            this.read('"');
+            return { type: TokenType.Identifier, value: identifier, command: identifier.toLowerCase() };
+        }
+
+        // SQLServer escaped identifier (escape character is square bracket)
+        if (char === '[' && (previous === null || previous.command !== "array")) {
+            this.position++;
+            const identifier = this.readIdentifier();
+            this.read(']');
+            return { type: TokenType.Identifier, value: identifier, command: identifier.toLowerCase() };
+        }
+
+        // named parameter
+        const namedParameterPrefix = new Set(['@', ':', '$']);
+        if (namedParameterPrefix.has(char)) {
+            this.position++;
+            const value = char + this.readIdentifier();
+            return { type: TokenType.Parameter, value, command: value.toLowerCase() };
+        }
+
+        // nameless parameter
+        if (char === '?') {
+            this.position++;
+            return { type: TokenType.Parameter, value: char, command: char.toLowerCase() };
+        }
+
+        // string literal
+        if (char === '\'') {
+            const value = this.readSingleQuotedString();
+            return { type: TokenType.Literal, value, command: value.toLowerCase() };
+        }
+
+        // digit tokens
+        if (/[0-9]/.test(char)){
+            return { type: TokenType.Literal, value: this.readDigit() };
+        }
+
+        // positive number
+        if (char === '+' && (previous === null || (previous.type !== TokenType.Literal && previous.type !== TokenType.Identifier))) {
+            this.position++;
+            this.skipWhiteSpacesAndComments();
+            // removed positive sign
+            return { type: TokenType.Literal, value: this.readDigit(), command: char.toLowerCase() };
+        }
+
+        // negative number
+        if (char === '-' && (previous === null || (previous.type !== TokenType.Literal && previous.type !== TokenType.Identifier))) {
+            this.position++;
+            this.skipWhiteSpacesAndComments();
+            // include negative sign
+            return { type: TokenType.Literal, value: char + this.readDigit(), command: char.toLowerCase() };
+        }
+
+        const identifier = this.readIdentifier();
+        return { type: TokenType.Identifier, value: identifier };
+    }
+
+    private readDigit(): string {
+        const start = this.position;
+        let hasDot = false;
+        let hasExponent = false;
+
+        // Consider 0x, 0b, 0o
+        if (this.input[this.position] === '0' && this.canRead(1)) {
+            const nextChar = this.input[this.position + 1].toLowerCase();
+            if (nextChar === 'x' || nextChar === 'b' || nextChar === 'o') {
+                this.position += 2;
+
+                // Continue to get numeric and hexadecimal notation strings
+                while (this.canRead() && /[0-9a-f]/i.test(this.input[this.position])) {
+                    this.position++;
+                }
+
+                return this.input.slice(start, this.position);
+            }
+            // If it is not 0x, 0b, 0o notation, treat it as a normal number
+        }
+
+        // Consider decimal point and exponential notation
+        while (this.canRead()) {
+            const char = this.input[this.position];
+
+            if (char === '.' && !hasDot) {
+                hasDot = true;
+            } else if ((char === 'e' || char === 'E') && !hasExponent) {
+                hasExponent = true;
+                if (this.canRead(1) && (this.input[this.position + 1] === '+' || this.input[this.position + 1] === '-')) {
+                    this.position++;
+                }
+            } else if (!/[0-9]/.test(char)) {
+                break;
+            }
+
+            this.position++;
+        }
+
+        if (start === this.position) {
+            throw new Error(`Unexpected character. position: ${start}`);
+        }
+
+        return this.input.slice(start, this.position);
+    }
+
+    private readSingleQuotedString(): string {
+        let start = this.position;
+
+        this.read('\'');
+
+        while (this.canRead()) {
+            var char = this.input[this.position];
+
+            // escape character check
+            if (char === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
+                this.position += 2;
+                continue;
+            }
+            else if (char === '\'') {
+                break;
+            }
+            this.position++;
+        }
+
+        if (this.isEndOfInput()) {
+            throw new Error(`Single quote is not closed. position: ${start}`);
+        }
+
+        const value = this.input.slice(start, this.position);
+        this.position++;
+        return value;
+    }
+
+    private readIdentifier(): string {
+        const start = this.position;
+
+        // Read until a delimiter appears
+        // Space, dot, comma, parentheses, arithmetic operators
+        // Note that underscores are treated as part of the word
+        const delimiters = new Set([' ', '.', ',', '(', ')', '+', '-', '*', '/']);
+
+        while (this.canRead()) {
+            if (delimiters.has(this.input[this.position])) {
+                break;
+            }
+            this.position++;
+        }
+
+        if (start === this.position) {
+            throw new Error(`Unexpected character. position: ${start}`);
+        }
+
+        return this.input.slice(start, this.position);
+    }
+
+    /// <summary>
+    /// Skip white space characters and sql comments.
+    /// </summary>
+    private skipWhiteSpacesAndComments(): void {
+        while (true) {
+            if (this.skipWhiteSpace()) {
+                continue;
+            }
+            if (this.skipLineComment()) {
+                continue;
+            }
+            if (this.skipBlockComment()) {
+                continue;
+            }
+            break;
+        }
+    }
+
+    private skipWhiteSpace(): boolean {
+        const start = this.position;
+
+        // Skip tab, newline, and space characters
+        const whitespace = new Set([' ', '\r', '\n', '\t']);
+
+        while (this.canRead()) {
+            if (!whitespace.has(this.input[this.position])) {
+                break;
+            }
+            this.position++;
+        }
+        return start !== this.position;
+    }
+
+    private skipLineComment(): boolean {
+        // At least 2 characters are needed. '--'
+        if (this.isEndOfInput(1)) {
+            return false;
+        }
+
+        if (this.input[this.position] === '-' && this.input[this.position + 1] === '-') {
+            this.position += 2;
+
+            while (this.canRead() && this.input[this.position] !== '\n') {
+                this.position++;
+            }
+            return true;
+        }
+
+        return false;
+    }
+
+    private skipBlockComment(): boolean {
+        // At least 4 characters are needed. '/**/'
+        if (this.isEndOfInput(3)) {
+            return false;
+        }
+
+        // Record the start position of the comment to track error location
+        const start = this.position;
+
+        if (this.input[this.position] === '/' && this.input[this.position + 1] === '*') {
+            this.position += 2;
+
+            while (this.canRead(1)) {
+                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
+                    this.position += 2;
+                    return true;
+                }
+                this.position++;
+            }
+
+            throw new Error(`Block comment is not closed. position: ${start}`);
+        }
+
+        return false;
+    }
+}

package/.vs/CopilotSnapshots/3BACC9346120824DB30006E353477163/state.mpack

@@ -0,0 +1 @@
+��SessionId��$34c9ac3b-2061-4d82-b300-06e353477163��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$e931c887-1aea-49a2-a25a-44d08cf8e0b5�Snapshots�� EF71788BE39818EEA3F0164DC15E1BD7��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\3BACC9346120824DB30006E353477163\87C831E9EA1AA249A25A44D08CF8E0B5\EF71788BE39818EEA3F0164DC15E1BD7�OriginalFilePath�=C:\Users\mssgm\Documents\GitHub\ts-sample\src\sqlTokenizer.ts�NewFileMarker�

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/14FE66AD0B834E488EECD9594AE72472/23E7151F0483C5BB5E663F83D4962FD4

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/14FE66AD0B834E488EECD9594AE72472/2D4DCEBC89D52B097AEEEABA23C0EB59

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/14FE66AD0B834E488EECD9594AE72472/7C951A8A572EA5C371FDE657AF968B9C

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/14FE66AD0B834E488EECD9594AE72472/A32D1DB906C73FA74EF270AAEE165FC0

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/14FE66AD0B834E488EECD9594AE72472/B692C85119711C76235AA5D78F9F3818

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/14FE66AD0B834E488EECD9594AE72472/E602628EF6D764C2C267810C92906DBA

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/14FE66AD0B834E488EECD9594AE72472/E621434FB1D5415B14FBBCC11967E420

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/9C5A45FA50D5344FBB275F406599FF4E/23E7151F0483C5BB5E663F83D4962FD4

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/9C5A45FA50D5344FBB275F406599FF4E/2D4DCEBC89D52B097AEEEABA23C0EB59

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/9C5A45FA50D5344FBB275F406599FF4E/7C951A8A572EA5C371FDE657AF968B9C

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/9C5A45FA50D5344FBB275F406599FF4E/A32D1DB906C73FA74EF270AAEE165FC0

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/9C5A45FA50D5344FBB275F406599FF4E/B692C85119711C76235AA5D78F9F3818

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/9C5A45FA50D5344FBB275F406599FF4E/E602628EF6D764C2C267810C92906DBA

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/3EFABA5D1530804CA04C606F5EFDDC38/9C5A45FA50D5344FBB275F406599FF4E/E621434FB1D5415B14FBBCC11967E420

@@ -0,0 +1 @@
+