rawsql-ts 0.1.0-beta.1

package/.vs/CopilotSnapshots/50E0417FAD2E524D887CB5763C2FB7B4/63A4C250D9DCE04B9F61D8833B89B5BF/EF71788BE39818EEA3F0164DC15E1BD7

@@ -0,0 +1,174 @@
+import { Lexeme } from './models/Lexeme';
+import { IdentifierTokenReader } from './tokenReaders/IdentifierTokenReader';
+import { LiteralTokenReader } from './tokenReaders/LiteralTokenReader';
+import { ParameterTokenReader } from './tokenReaders/ParameterTokenReader';
+import { SymbolTokenReader } from './tokenReaders/SymbolTokenReader';
+
+export class SqlTokenizer {
+    /// <summary>
+    /// Input string.
+    /// </summary>
+    private input: string;
+
+    /// <summary>
+    /// Current position in the input string.
+    /// </summary>
+    private position: number;
+
+    constructor(input: string) {
+        this.input = input;
+        this.position = 0;
+    }
+
+    private isEndOfInput(shift: number = 0): boolean {
+        return this.position + shift >= this.input.length;
+    }
+
+    private canRead(shift: number = 0): boolean {
+        return !this.isEndOfInput(shift);
+    }
+
+    public readLexmes(): Lexeme[] {
+        const lexemes: Lexeme[] = [];
+
+        // Pre-allocate array based on the estimated number of tokens (on average, 1 token every 5 characters)
+        const estimatedCapacity = Math.max(10, Math.ceil(this.input.length / 5));
+        lexemes.length = 0;
+
+        // Skip whitespace and comments at the start
+        this.skipWhiteSpacesAndComments();
+
+        // Initialize each token reader
+        const symbolReader = new SymbolTokenReader(this.input, this.position);
+        const identifierReader = new IdentifierTokenReader(this.input, this.position);
+        const literalReader = new LiteralTokenReader(this.input, this.position);
+        const parameterReader = new ParameterTokenReader(this.input, this.position);
+
+        // Track the previous token
+        let previous: Lexeme | null = null;
+
+        // Read tokens until the end of the input is reached
+        while (this.canRead()) {
+            let lexeme: Lexeme | null = null;
+
+            // Attempt to read with each reader in order
+            parameterReader.setPosition(this.position);
+            lexeme = parameterReader.tryRead(previous);
+
+            if (!lexeme) {
+                literalReader.setPosition(this.position);
+                lexeme = literalReader.tryRead(previous);
+            }
+
+            if (!lexeme) {
+                identifierReader.setPosition(this.position);
+                lexeme = identifierReader.tryRead(previous);
+            }
+
+            if (!lexeme) {
+                symbolReader.setPosition(this.position);
+                lexeme = symbolReader.tryRead(previous);
+            }
+
+            // If a token is read by any reader
+            if (lexeme) {
+                lexemes.push(lexeme);
+                previous = lexeme;
+
+                // Update position
+                const newPos = Math.max(
+                    symbolReader.getPosition(),
+                    literalReader.getPosition(),
+                    identifierReader.getPosition(),
+                    parameterReader.getPosition()
+                );
+                this.position = newPos;
+
+                // Skip whitespace and comments after the token
+                this.skipWhiteSpacesAndComments();
+            } else {
+                // Exception
+                throw new Error(`Unexpected character. actual: ${this.input[this.position]}, position: ${this.position}`);
+            }
+        }
+
+        return lexemes;
+    }
+
+    /// <summary>
+    /// Skip white space characters and sql comments.
+    /// </summary>
+    private skipWhiteSpacesAndComments(): void {
+        while (true) {
+            if (this.skipWhiteSpace()) {
+                continue;
+            }
+            if (this.skipLineComment()) {
+                continue;
+            }
+            if (this.skipBlockComment()) {
+                continue;
+            }
+            break;
+        }
+    }
+
+    private skipWhiteSpace(): boolean {
+        const start = this.position;
+
+        // Skip tab, newline, and space characters
+        const whitespace = new Set([' ', '\r', '\n', '\t']);
+
+        while (this.canRead()) {
+            if (!whitespace.has(this.input[this.position])) {
+                break;
+            }
+            this.position++;
+        }
+        return start !== this.position;
+    }
+
+    private skipLineComment(): boolean {
+        // At least 2 characters are needed. '--'
+        if (this.isEndOfInput(1)) {
+            return false;
+        }
+
+        if (this.input[this.position] === '-' && this.input[this.position + 1] === '-') {
+            this.position += 2;
+
+            while (this.canRead() && this.input[this.position] !== '\n') {
+                this.position++;
+            }
+            return true;
+        }
+
+        return false;
+    }
+
+    private skipBlockComment(): boolean {
+        // At least 4 characters are needed. '/**/'
+        if (this.isEndOfInput(3)) {
+            return false;
+        }
+
+        // Record the start position of the comment to track error location
+        const start = this.position;
+
+        if (this.input[this.position] === '/' && this.input[this.position + 1] === '*') {
+            this.position += 2;
+
+            while (this.canRead(1)) {
+                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
+                    this.position += 2;
+                    return true;
+                }
+                this.position++;
+            }
+
+            throw new Error(`Block comment is not closed. position: ${start}`);
+        }
+
+        return false;
+    }
+}

package/.vs/CopilotSnapshots/50E0417FAD2E524D887CB5763C2FB7B4/state.mpack

@@ -0,0 +1 @@
+��SessionId��$7f41e050-2ead-4d52-887c-b5763c2fb7b4��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$50c2a463-dcd9-4be0-9f61-d8833b89b5bf�Snapshots�� EF71788BE39818EEA3F0164DC15E1BD7��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\50E0417FAD2E524D887CB5763C2FB7B4\63A4C250D9DCE04B9F61D8833B89B5BF\EF71788BE39818EEA3F0164DC15E1BD7�OriginalFilePath�=C:\Users\mssgm\Documents\GitHub\ts-sample\src\sqlTokenizer.ts�NewFileMarker�

package/.vs/CopilotSnapshots/5EF2507B002AC14EA991043C37C36551/60FF68DE489DAD4FBCBBB54AA3F6A552/EF71788BE39818EEA3F0164DC15E1BD7

@@ -0,0 +1,496 @@
+import { TokenType } from './enums/tokenType';
+
+interface Lexeme {
+    type: TokenType;
+    value: string;
+    command?: string; // Add the command property
+}
+
+export class SqlTokenizer {
+    /// <summary>
+    /// Input string.
+    /// </summary>
+    private input: string;
+
+    /// <summary>
+    /// Current position in the input string.
+    /// </summary>
+    private position: number;
+
+    constructor(input: string) {
+        this.input = input;
+        this.position = 0;
+    }
+
+    private isEndOfInput(shift: number = 0): boolean {
+        return this.position + shift >= this.input.length;
+    }
+
+    private canRead(shift: number = 0): boolean {
+        return !this.isEndOfInput(shift);
+    }
+
+    private read(expectChar: string): string {
+        if (this.isEndOfInput()) {
+            throw new Error(`Unexpected character. expect: ${expectChar}, actual: EndOfInut, position: ${this.position}`);
+        }
+
+        if (this.input[this.position] !== expectChar) {
+            throw new Error(`Unexpected character. expect: ${expectChar}, actual: ${this.input[this.position]}, position: ${this.position}`);
+        }
+
+        var char = this.input[this.position];
+        this.position++;
+        return char;
+    }
+
+    private static readonly SYMBOL_TOKENS: Record<string, TokenType> = {
+        '.': TokenType.Dot,
+        ',': TokenType.Comma,
+        '(': TokenType.OpenParen,
+        ')': TokenType.CloseParen,
+    };
+
+    private getDebugPositionInfo(errPosition: number): string {
+        // Get 5 characters before and after the error
+        // If the start and end points are out of the string range, keep them within the range
+        // Display ^ at the error position on the next line
+        const start = Math.max(0, errPosition - 5);
+        const end = Math.min(this.input.length, errPosition + 5);
+        const debugInfo = this.input.slice(start, end);
+        const caret = ' '.repeat(errPosition - start) + '^';
+        return `${debugInfo}\n${caret}`;
+    }
+
+    public readLexmes(): Lexeme[] {
+        const lexemes: Lexeme[] = [];
+
+        let lexeme: Lexeme | null = null;
+        while ((lexeme = this.readLexme(lexeme)) !== null) {
+            lexemes.push(lexeme);
+        }
+        return lexemes;
+    }
+
+    public readLexme(previous: Lexeme | null = null): Lexeme | null {
+        // end of input
+        if (this.isEndOfInput()) {
+            return null;
+        }
+
+        this.skipWhiteSpacesAndComments();
+
+        // current character
+        const char = this.input[this.position];
+
+        // Try each token type handler in order
+        const lexeme = this.tryReadDecimalToken(char) ||
+            this.tryReadSymbolToken(char) ||
+            this.tryReadEscapedIdentifier(char, previous) ||
+            this.tryReadParameter(char) ||
+            this.tryReadStringLiteral(char) ||
+            this.tryReadDigit(char) ||
+            this.tryReadSignedNumber(char, previous) ||
+            this.tryReadEscapedLiteral(char);
+
+        if (lexeme) {
+            return lexeme;
+        }
+
+        // Default to reading an identifier if no other token type matches
+        const identifier = this.readIdentifier();
+        return {
+            type: TokenType.Identifier,
+            value: identifier
+        };
+    }
+
+    private tryReadDecimalToken(char: string): Lexeme | null {
+        // digit tokens (prioritize dot for decimal point and delimiter)
+        if (char === '.' && this.canRead(1) && /[0-9]/.test(this.input[this.position + 1])) {
+            return {
+                type: TokenType.Literal,
+                value: this.readDigit()
+            };
+        }
+        return null;
+    }
+
+    private tryReadSymbolToken(char: string): Lexeme | null {
+        // symbol tokens
+        if (char in SqlTokenizer.SYMBOL_TOKENS) {
+            this.position++;
+            return {
+                type: SqlTokenizer.SYMBOL_TOKENS[char],
+                value: char,
+                command: char.toLowerCase()
+            };
+        }
+        return null;
+    }
+
+    private tryReadEscapedIdentifier(char: string, previous: Lexeme | null): Lexeme | null {
+        // MySQL escaped identifier (escape character is backtick)
+        if (char === '`') {
+            const identifier = this.readEscapedIdentifier('`');
+            return {
+                type: TokenType.Identifier,
+                value: identifier
+            };
+        }
+
+        // Postgres escaped identifier (escape character is double quote)
+        if (char === '"') {
+            const identifier = this.readEscapedIdentifier('"');
+            return {
+                type: TokenType.Identifier,
+                value: identifier
+            };
+        }
+
+        // SQLServer escaped identifier (escape character is square bracket)
+        if (char === '[' && (previous === null || previous.command !== "array")) {
+            const identifier = this.readEscapedIdentifier(']');
+            return {
+                type: TokenType.Identifier,
+                value: identifier,
+            };
+        }
+
+        return null;
+    }
+
+    private tryReadParameter(char: string): Lexeme | null {
+        // named parameter
+        const namedParameterPrefix = new Set(['@', ':', '$']);
+        if (namedParameterPrefix.has(char)) {
+            this.position++;
+            const value = char + this.readIdentifier();
+            return {
+                type: TokenType.Parameter,
+                value
+            };
+        }
+
+        // nameless parameter
+        if (char === '?') {
+            this.position++;
+            return {
+                type: TokenType.Parameter,
+                value: char
+            };
+        }
+
+        return null;
+    }
+
+    private tryReadStringLiteral(char: string): Lexeme | null {
+        // string literal
+        if (char === '\'') {
+            const value = this.readSingleQuotedString();
+            return {
+                type: TokenType.Literal,
+                value
+            };
+        }
+
+        return null;
+    }
+
+    private tryReadDigit(char: string): Lexeme | null {
+        // digit tokens
+        if (/[0-9]/.test(char)) {
+            return {
+                type: TokenType.Literal,
+                value: this.readDigit()
+            };
+        }
+
+        return null;
+    }
+
+    private tryReadSignedNumber(char: string, previous: Lexeme | null): Lexeme | null {
+        // Check if this could be a signed number (not after a literal/identifier)
+        const isValidNumericPrefix = previous === null ||
+            (previous.type !== TokenType.Literal &&
+                previous.type !== TokenType.Identifier);
+
+        // positive number
+        if (char === '+' && isValidNumericPrefix) {
+            this.position++;
+            this.skipWhiteSpacesAndComments();
+            // removed positive sign
+            return {
+                type: TokenType.Literal,
+                value: this.readDigit()
+            };
+        }
+
+        // negative number
+        if (char === '-' && isValidNumericPrefix) {
+            this.position++;
+            this.skipWhiteSpacesAndComments();
+            // include negative sign
+            return {
+                type: TokenType.Literal,
+                value: char + this.readDigit()
+            };
+        }
+
+        return null;
+    }
+
+    private tryReadEscapedLiteral(char: string): Lexeme | null {
+        const start = this.position;
+
+        const prefix = new Set(['e\'', 'x\'', 'b\'']);
+        if (this.canRead(1) && prefix.has(this.input.slice(start, 2).toLocaleLowerCase())) {
+            this.position += 2;
+
+            while (this.canRead()) {
+                if (this.input[this.position] === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
+                    this.position += 2;
+                    continue;
+                }
+                else if (this.input[this.position] === '\'') {
+                    this.position++;
+                    break;
+                }
+                this.position++;
+            }
+
+            if (this.position - 2 === start) {
+                throw new Error(`Closing delimiter is not found. position: ${start}`);
+            };
+
+            return {
+                type: TokenType.Literal,
+                value: this.input.slice(start, this.position)
+            };
+        }
+
+        // u&' 
+        if (this.canRead(2) && this.input.slice(start, 3).toLocaleLowerCase() === 'u&\'') {
+            this.position += 3;
+
+            while (this.canRead()) {
+                if (this.input[this.position] === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
+                    this.position += 2;
+                    continue;
+                }
+                else if (this.input[this.position] === '\'') {
+                    this.position++;
+                    break;
+                }
+                this.position++;
+            }
+            if (this.position - 2 === start) {
+                throw new Error(`Closing delimiter is not found. position: ${start}`);
+            };
+            return {
+                type: TokenType.Literal,
+                value: this.input.slice(start, this.position)
+            }
+        }
+
+        return null;
+    }
+
+    private readDigit(): string {
+        const start = this.position;
+        let hasDot = false;
+        let hasExponent = false;
+
+        // Consider 0x, 0b, 0o
+        const prefix = new Set(['0x', '0b', '0o']);
+        if (this.canRead(1) && prefix.has(this.input.slice(start, 2).toLocaleLowerCase())) {
+            this.position += 2;
+
+            // Continue to get numeric and hexadecimal notation strings
+            while (this.canRead() && /[0-9a-f]/i.test(this.input[this.position])) {
+                this.position++;
+            }
+
+            return this.input.slice(start, this.position);
+        }
+
+        // Consider decimal point and exponential notation
+        while (this.canRead()) {
+            const char = this.input[this.position];
+
+            if (char === '.' && !hasDot) {
+                hasDot = true;
+            } else if ((char === 'e' || char === 'E') && !hasExponent) {
+                hasExponent = true;
+                if (this.canRead(1) && (this.input[this.position + 1] === '+' || this.input[this.position + 1] === '-')) {
+                    this.position++;
+                }
+            } else if (!/[0-9]/.test(char)) {
+                break;
+            }
+
+            this.position++;
+        }
+
+        if (start === this.position) {
+            throw new Error(`Unexpected character. position: ${start}`);
+        }
+
+        if (this.input[start] === '.') {
+            // If the number starts with a dot, add 0 to the front
+            return '0' + this.input.slice(start, this.position);
+        }
+
+        return this.input.slice(start, this.position);
+    }
+
+    private readEscapedIdentifier(delimiter: string): string {
+        const start = this.position;
+
+        // Skip the opening delimiter
+        this.position++;
+
+        while (this.canRead()) {
+            if (this.input[this.position] === delimiter) {
+                break;
+            }
+            this.position++;
+        }
+        if (start === this.position) {
+            throw new Error(`Closing delimiter is not found. position: ${start}, delimiter: ${delimiter}`);
+        }
+
+        // Skip the closing delimiter
+        this.position++;
+
+        // exclude the delimiter
+        return this.input.slice(start + 1, this.position - 1);
+    }
+
+    private readSingleQuotedString(): string {
+        const start = this.position;
+
+        this.read('\'');
+
+        while (this.canRead()) {
+            var char = this.input[this.position];
+
+            // escape character check
+            if (char === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
+                this.position += 2;
+                continue;
+            }
+            else if (char === '\'') {
+                break;
+            }
+            this.position++;
+        }
+
+        if (this.isEndOfInput()) {
+            throw new Error(`Single quote is not closed. position: ${start}`);
+        }
+
+        const value = this.input.slice(start, this.position);
+        this.position++;
+        return value;
+    }
+
+    private readIdentifier(): string {
+        const start = this.position;
+
+        // Read until a delimiter appears
+        // Space, dot, comma, parentheses, arithmetic operators
+        // Note that underscores are treated as part of the word
+        const delimiters = new Set([' ', '.', ',', '(', ')', '+', '-', '*', '/']);
+
+        while (this.canRead()) {
+            if (delimiters.has(this.input[this.position])) {
+                break;
+            }
+            this.position++;
+        }
+
+        if (start === this.position) {
+            throw new Error(`Unexpected character. position: ${start}\n${this.getDebugPositionInfo(start)}`);
+        }
+
+        return this.input.slice(start, this.position);
+    }
+
+    /// <summary>
+    /// Skip white space characters and sql comments.
+    /// </summary>
+    private skipWhiteSpacesAndComments(): void {
+        while (true) {
+            if (this.skipWhiteSpace()) {
+                continue;
+            }
+            if (this.skipLineComment()) {
+                continue;
+            }
+            if (this.skipBlockComment()) {
+                continue;
+            }
+            break;
+        }
+    }
+
+    private skipWhiteSpace(): boolean {
+        const start = this.position;
+
+        // Skip tab, newline, and space characters
+        const whitespace = new Set([' ', '\r', '\n', '\t']);
+
+        while (this.canRead()) {
+            if (!whitespace.has(this.input[this.position])) {
+                break;
+            }
+            this.position++;
+        }
+        return start !== this.position;
+    }
+
+    private skipLineComment(): boolean {
+        // At least 2 characters are needed. '--'
+        if (this.isEndOfInput(1)) {
+            return false;
+        }
+
+        if (this.input[this.position] === '-' && this.input[this.position + 1] === '-') {
+            this.position += 2;
+
+            while (this.canRead() && this.input[this.position] !== '\n') {
+                this.position++;
+            }
+            return true;
+        }
+
+        return false;
+    }
+
+    private skipBlockComment(): boolean {
+        // At least 4 characters are needed. '/**/'
+        if (this.isEndOfInput(3)) {
+            return false;
+        }
+
+        // Record the start position of the comment to track error location
+        const start = this.position;
+
+        if (this.input[this.position] === '/' && this.input[this.position + 1] === '*') {
+            this.position += 2;
+
+            while (this.canRead(1)) {
+                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
+                    this.position += 2;
+                    return true;
+                }
+                this.position++;
+            }
+
+            throw new Error(`Block comment is not closed. position: ${start}`);
+        }
+
+        return false;
+    }
+}

package/.vs/CopilotSnapshots/5EF2507B002AC14EA991043C37C36551/state.mpack

@@ -0,0 +1 @@
+��SessionId��$7b50f25e-2a00-4ec1-a991-043c37c36551��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$de68ff60-9d48-4fad-bcbb-b54aa3f6a552�Snapshots�� EF71788BE39818EEA3F0164DC15E1BD7��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\5EF2507B002AC14EA991043C37C36551\60FF68DE489DAD4FBCBBB54AA3F6A552\EF71788BE39818EEA3F0164DC15E1BD7�OriginalFilePath�=C:\Users\mssgm\Documents\GitHub\ts-sample\src\sqlTokenizer.ts�NewFileMarker�

package/.vs/CopilotSnapshots/6AA7610F1FB2B746821E54C28A8D109B/7FB78C0F8CB04F4AA6AF3315277B80F8/FB35DD13E708043D4CD8ACD54E14708F

@@ -0,0 +1 @@
+

package/.vs/CopilotSnapshots/6AA7610F1FB2B746821E54C28A8D109B/state.mpack

@@ -0,0 +1 @@
+��SessionId��$0f61a76a-b21f-46b7-821e-54c28a8d109b��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$0f8cb77f-b08c-4a4f-a6af-3315277b80f8�Snapshots�� FB35DD13E708043D4CD8ACD54E14708F��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\6AA7610F1FB2B746821E54C28A8D109B\7FB78C0F8CB04F4AA6AF3315277B80F8\FB35DD13E708043D4CD8ACD54E14708F�OriginalFilePath�>C:\Users\mssgm\Documents\GitHub\ts-sample\ts-sample\.gitignore�NewFileMarker�

package/.vs/CopilotSnapshots/7F905F8BAC0DAD49BD507CA45C569AFD/5782E13CAB03A04E9E1EC5692F892170/BA5D2B145A59265D63FA3B3584344B02

@@ -0,0 +1,5 @@
+import { TokenType } from "../src/enums/tokenType";
+import { SqlTokenizer } from "../src/sqlTokenizer";
+
+// e.g 1+1, 1*2, 1/3, 1%4
+