rawsql-ts 0.1.0-beta.1 → 0.1.0-beta.11

package/.vs/CopilotSnapshots/50E0417FAD2E524D887CB5763C2FB7B4/63A4C250D9DCE04B9F61D8833B89B5BF/EF71788BE39818EEA3F0164DC15E1BD7

@@ -1,174 +0,0 @@
-import { Lexeme } from './models/Lexeme';
-import { IdentifierTokenReader } from './tokenReaders/IdentifierTokenReader';
-import { LiteralTokenReader } from './tokenReaders/LiteralTokenReader';
-import { ParameterTokenReader } from './tokenReaders/ParameterTokenReader';
-import { SymbolTokenReader } from './tokenReaders/SymbolTokenReader';
-
-export class SqlTokenizer {
-    /// <summary>
-    /// Input string.
-    /// </summary>
-    private input: string;
-
-    /// <summary>
-    /// Current position in the input string.
-    /// </summary>
-    private position: number;
-
-    constructor(input: string) {
-        this.input = input;
-        this.position = 0;
-    }
-
-    private isEndOfInput(shift: number = 0): boolean {
-        return this.position + shift >= this.input.length;
-    }
-
-    private canRead(shift: number = 0): boolean {
-        return !this.isEndOfInput(shift);
-    }
-
-    public readLexmes(): Lexeme[] {
-        const lexemes: Lexeme[] = [];
-
-        // Pre-allocate array based on the estimated number of tokens (on average, 1 token every 5 characters)
-        const estimatedCapacity = Math.max(10, Math.ceil(this.input.length / 5));
-        lexemes.length = 0;
-
-        // Skip whitespace and comments at the start
-        this.skipWhiteSpacesAndComments();
-
-        // Initialize each token reader
-        const symbolReader = new SymbolTokenReader(this.input, this.position);
-        const identifierReader = new IdentifierTokenReader(this.input, this.position);
-        const literalReader = new LiteralTokenReader(this.input, this.position);
-        const parameterReader = new ParameterTokenReader(this.input, this.position);
-
-        // Track the previous token
-        let previous: Lexeme | null = null;
-
-        // Read tokens until the end of the input is reached
-        while (this.canRead()) {
-            let lexeme: Lexeme | null = null;
-
-            // Attempt to read with each reader in order
-            parameterReader.setPosition(this.position);
-            lexeme = parameterReader.tryRead(previous);
-
-            if (!lexeme) {
-                literalReader.setPosition(this.position);
-                lexeme = literalReader.tryRead(previous);
-            }
-
-            if (!lexeme) {
-                identifierReader.setPosition(this.position);
-                lexeme = identifierReader.tryRead(previous);
-            }
-
-            if (!lexeme) {
-                symbolReader.setPosition(this.position);
-                lexeme = symbolReader.tryRead(previous);
-            }
-
-            // If a token is read by any reader
-            if (lexeme) {
-                lexemes.push(lexeme);
-                previous = lexeme;
-
-                // Update position
-                const newPos = Math.max(
-                    symbolReader.getPosition(),
-                    literalReader.getPosition(),
-                    identifierReader.getPosition(),
-                    parameterReader.getPosition()
-                );
-                this.position = newPos;
-
-                // Skip whitespace and comments after the token
-                this.skipWhiteSpacesAndComments();
-            } else {
-                // Exception
-                throw new Error(`Unexpected character. actual: ${this.input[this.position]}, position: ${this.position}`);
-            }
-        }
-
-        return lexemes;
-    }
-
-    /// <summary>
-    /// Skip white space characters and sql comments.
-    /// </summary>
-    private skipWhiteSpacesAndComments(): void {
-        while (true) {
-            if (this.skipWhiteSpace()) {
-                continue;
-            }
-            if (this.skipLineComment()) {
-                continue;
-            }
-            if (this.skipBlockComment()) {
-                continue;
-            }
-            break;
-        }
-    }
-
-    private skipWhiteSpace(): boolean {
-        const start = this.position;
-
-        // Skip tab, newline, and space characters
-        const whitespace = new Set([' ', '\r', '\n', '\t']);
-
-        while (this.canRead()) {
-            if (!whitespace.has(this.input[this.position])) {
-                break;
-            }
-            this.position++;
-        }
-        return start !== this.position;
-    }
-
-    private skipLineComment(): boolean {
-        // At least 2 characters are needed. '--'
-        if (this.isEndOfInput(1)) {
-            return false;
-        }
-
-        if (this.input[this.position] === '-' && this.input[this.position + 1] === '-') {
-            this.position += 2;
-
-            while (this.canRead() && this.input[this.position] !== '\n') {
-                this.position++;
-            }
-            return true;
-        }
-
-        return false;
-    }
-
-    private skipBlockComment(): boolean {
-        // At least 4 characters are needed. '/**/'
-        if (this.isEndOfInput(3)) {
-            return false;
-        }
-
-        // Record the start position of the comment to track error location
-        const start = this.position;
-
-        if (this.input[this.position] === '/' && this.input[this.position + 1] === '*') {
-            this.position += 2;
-
-            while (this.canRead(1)) {
-                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
-                    this.position += 2;
-                    return true;
-                }
-                this.position++;
-            }
-
-            throw new Error(`Block comment is not closed. position: ${start}`);
-        }
-
-        return false;
-    }
-}

package/.vs/CopilotSnapshots/50E0417FAD2E524D887CB5763C2FB7B4/state.mpack

@@ -1 +0,0 @@
-��SessionId��$7f41e050-2ead-4d52-887c-b5763c2fb7b4��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$50c2a463-dcd9-4be0-9f61-d8833b89b5bf�Snapshots�� EF71788BE39818EEA3F0164DC15E1BD7��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\50E0417FAD2E524D887CB5763C2FB7B4\63A4C250D9DCE04B9F61D8833B89B5BF\EF71788BE39818EEA3F0164DC15E1BD7�OriginalFilePath�=C:\Users\mssgm\Documents\GitHub\ts-sample\src\sqlTokenizer.ts�NewFileMarker�

package/.vs/CopilotSnapshots/5EF2507B002AC14EA991043C37C36551/60FF68DE489DAD4FBCBBB54AA3F6A552/EF71788BE39818EEA3F0164DC15E1BD7

@@ -1,496 +0,0 @@
-import { TokenType } from './enums/tokenType';
-
-interface Lexeme {
-    type: TokenType;
-    value: string;
-    command?: string; // Add the command property
-}
-
-export class SqlTokenizer {
-    /// <summary>
-    /// Input string.
-    /// </summary>
-    private input: string;
-
-    /// <summary>
-    /// Current position in the input string.
-    /// </summary>
-    private position: number;
-
-    constructor(input: string) {
-        this.input = input;
-        this.position = 0;
-    }
-
-    private isEndOfInput(shift: number = 0): boolean {
-        return this.position + shift >= this.input.length;
-    }
-
-    private canRead(shift: number = 0): boolean {
-        return !this.isEndOfInput(shift);
-    }
-
-    private read(expectChar: string): string {
-        if (this.isEndOfInput()) {
-            throw new Error(`Unexpected character. expect: ${expectChar}, actual: EndOfInut, position: ${this.position}`);
-        }
-
-        if (this.input[this.position] !== expectChar) {
-            throw new Error(`Unexpected character. expect: ${expectChar}, actual: ${this.input[this.position]}, position: ${this.position}`);
-        }
-
-        var char = this.input[this.position];
-        this.position++;
-        return char;
-    }
-
-    private static readonly SYMBOL_TOKENS: Record<string, TokenType> = {
-        '.': TokenType.Dot,
-        ',': TokenType.Comma,
-        '(': TokenType.OpenParen,
-        ')': TokenType.CloseParen,
-    };
-
-    private getDebugPositionInfo(errPosition: number): string {
-        // Get 5 characters before and after the error
-        // If the start and end points are out of the string range, keep them within the range
-        // Display ^ at the error position on the next line
-        const start = Math.max(0, errPosition - 5);
-        const end = Math.min(this.input.length, errPosition + 5);
-        const debugInfo = this.input.slice(start, end);
-        const caret = ' '.repeat(errPosition - start) + '^';
-        return `${debugInfo}\n${caret}`;
-    }
-
-    public readLexmes(): Lexeme[] {
-        const lexemes: Lexeme[] = [];
-
-        let lexeme: Lexeme | null = null;
-        while ((lexeme = this.readLexme(lexeme)) !== null) {
-            lexemes.push(lexeme);
-        }
-        return lexemes;
-    }
-
-    public readLexme(previous: Lexeme | null = null): Lexeme | null {
-        // end of input
-        if (this.isEndOfInput()) {
-            return null;
-        }
-
-        this.skipWhiteSpacesAndComments();
-
-        // current character
-        const char = this.input[this.position];
-
-        // Try each token type handler in order
-        const lexeme = this.tryReadDecimalToken(char) ||
-            this.tryReadSymbolToken(char) ||
-            this.tryReadEscapedIdentifier(char, previous) ||
-            this.tryReadParameter(char) ||
-            this.tryReadStringLiteral(char) ||
-            this.tryReadDigit(char) ||
-            this.tryReadSignedNumber(char, previous) ||
-            this.tryReadEscapedLiteral(char);
-
-        if (lexeme) {
-            return lexeme;
-        }
-
-        // Default to reading an identifier if no other token type matches
-        const identifier = this.readIdentifier();
-        return {
-            type: TokenType.Identifier,
-            value: identifier
-        };
-    }
-
-    private tryReadDecimalToken(char: string): Lexeme | null {
-        // digit tokens (prioritize dot for decimal point and delimiter)
-        if (char === '.' && this.canRead(1) && /[0-9]/.test(this.input[this.position + 1])) {
-            return {
-                type: TokenType.Literal,
-                value: this.readDigit()
-            };
-        }
-        return null;
-    }
-
-    private tryReadSymbolToken(char: string): Lexeme | null {
-        // symbol tokens
-        if (char in SqlTokenizer.SYMBOL_TOKENS) {
-            this.position++;
-            return {
-                type: SqlTokenizer.SYMBOL_TOKENS[char],
-                value: char,
-                command: char.toLowerCase()
-            };
-        }
-        return null;
-    }
-
-    private tryReadEscapedIdentifier(char: string, previous: Lexeme | null): Lexeme | null {
-        // MySQL escaped identifier (escape character is backtick)
-        if (char === '`') {
-            const identifier = this.readEscapedIdentifier('`');
-            return {
-                type: TokenType.Identifier,
-                value: identifier
-            };
-        }
-
-        // Postgres escaped identifier (escape character is double quote)
-        if (char === '"') {
-            const identifier = this.readEscapedIdentifier('"');
-            return {
-                type: TokenType.Identifier,
-                value: identifier
-            };
-        }
-
-        // SQLServer escaped identifier (escape character is square bracket)
-        if (char === '[' && (previous === null || previous.command !== "array")) {
-            const identifier = this.readEscapedIdentifier(']');
-            return {
-                type: TokenType.Identifier,
-                value: identifier,
-            };
-        }
-
-        return null;
-    }
-
-    private tryReadParameter(char: string): Lexeme | null {
-        // named parameter
-        const namedParameterPrefix = new Set(['@', ':', '$']);
-        if (namedParameterPrefix.has(char)) {
-            this.position++;
-            const value = char + this.readIdentifier();
-            return {
-                type: TokenType.Parameter,
-                value
-            };
-        }
-
-        // nameless parameter
-        if (char === '?') {
-            this.position++;
-            return {
-                type: TokenType.Parameter,
-                value: char
-            };
-        }
-
-        return null;
-    }
-
-    private tryReadStringLiteral(char: string): Lexeme | null {
-        // string literal
-        if (char === '\'') {
-            const value = this.readSingleQuotedString();
-            return {
-                type: TokenType.Literal,
-                value
-            };
-        }
-
-        return null;
-    }
-
-    private tryReadDigit(char: string): Lexeme | null {
-        // digit tokens
-        if (/[0-9]/.test(char)) {
-            return {
-                type: TokenType.Literal,
-                value: this.readDigit()
-            };
-        }
-
-        return null;
-    }
-
-    private tryReadSignedNumber(char: string, previous: Lexeme | null): Lexeme | null {
-        // Check if this could be a signed number (not after a literal/identifier)
-        const isValidNumericPrefix = previous === null ||
-            (previous.type !== TokenType.Literal &&
-                previous.type !== TokenType.Identifier);
-
-        // positive number
-        if (char === '+' && isValidNumericPrefix) {
-            this.position++;
-            this.skipWhiteSpacesAndComments();
-            // removed positive sign
-            return {
-                type: TokenType.Literal,
-                value: this.readDigit()
-            };
-        }
-
-        // negative number
-        if (char === '-' && isValidNumericPrefix) {
-            this.position++;
-            this.skipWhiteSpacesAndComments();
-            // include negative sign
-            return {
-                type: TokenType.Literal,
-                value: char + this.readDigit()
-            };
-        }
-
-        return null;
-    }
-
-    private tryReadEscapedLiteral(char: string): Lexeme | null {
-        const start = this.position;
-
-        const prefix = new Set(['e\'', 'x\'', 'b\'']);
-        if (this.canRead(1) && prefix.has(this.input.slice(start, 2).toLocaleLowerCase())) {
-            this.position += 2;
-
-            while (this.canRead()) {
-                if (this.input[this.position] === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
-                    this.position += 2;
-                    continue;
-                }
-                else if (this.input[this.position] === '\'') {
-                    this.position++;
-                    break;
-                }
-                this.position++;
-            }
-
-            if (this.position - 2 === start) {
-                throw new Error(`Closing delimiter is not found. position: ${start}`);
-            };
-
-            return {
-                type: TokenType.Literal,
-                value: this.input.slice(start, this.position)
-            };
-        }
-
-        // u&' 
-        if (this.canRead(2) && this.input.slice(start, 3).toLocaleLowerCase() === 'u&\'') {
-            this.position += 3;
-
-            while (this.canRead()) {
-                if (this.input[this.position] === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
-                    this.position += 2;
-                    continue;
-                }
-                else if (this.input[this.position] === '\'') {
-                    this.position++;
-                    break;
-                }
-                this.position++;
-            }
-            if (this.position - 2 === start) {
-                throw new Error(`Closing delimiter is not found. position: ${start}`);
-            };
-            return {
-                type: TokenType.Literal,
-                value: this.input.slice(start, this.position)
-            }
-        }
-
-        return null;
-    }
-
-    private readDigit(): string {
-        const start = this.position;
-        let hasDot = false;
-        let hasExponent = false;
-
-        // Consider 0x, 0b, 0o
-        const prefix = new Set(['0x', '0b', '0o']);
-        if (this.canRead(1) && prefix.has(this.input.slice(start, 2).toLocaleLowerCase())) {
-            this.position += 2;
-
-            // Continue to get numeric and hexadecimal notation strings
-            while (this.canRead() && /[0-9a-f]/i.test(this.input[this.position])) {
-                this.position++;
-            }
-
-            return this.input.slice(start, this.position);
-        }
-
-        // Consider decimal point and exponential notation
-        while (this.canRead()) {
-            const char = this.input[this.position];
-
-            if (char === '.' && !hasDot) {
-                hasDot = true;
-            } else if ((char === 'e' || char === 'E') && !hasExponent) {
-                hasExponent = true;
-                if (this.canRead(1) && (this.input[this.position + 1] === '+' || this.input[this.position + 1] === '-')) {
-                    this.position++;
-                }
-            } else if (!/[0-9]/.test(char)) {
-                break;
-            }
-
-            this.position++;
-        }
-
-        if (start === this.position) {
-            throw new Error(`Unexpected character. position: ${start}`);
-        }
-
-        if (this.input[start] === '.') {
-            // If the number starts with a dot, add 0 to the front
-            return '0' + this.input.slice(start, this.position);
-        }
-
-        return this.input.slice(start, this.position);
-    }
-
-    private readEscapedIdentifier(delimiter: string): string {
-        const start = this.position;
-
-        // Skip the opening delimiter
-        this.position++;
-
-        while (this.canRead()) {
-            if (this.input[this.position] === delimiter) {
-                break;
-            }
-            this.position++;
-        }
-        if (start === this.position) {
-            throw new Error(`Closing delimiter is not found. position: ${start}, delimiter: ${delimiter}`);
-        }
-
-        // Skip the closing delimiter
-        this.position++;
-
-        // exclude the delimiter
-        return this.input.slice(start + 1, this.position - 1);
-    }
-
-    private readSingleQuotedString(): string {
-        const start = this.position;
-
-        this.read('\'');
-
-        while (this.canRead()) {
-            var char = this.input[this.position];
-
-            // escape character check
-            if (char === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
-                this.position += 2;
-                continue;
-            }
-            else if (char === '\'') {
-                break;
-            }
-            this.position++;
-        }
-
-        if (this.isEndOfInput()) {
-            throw new Error(`Single quote is not closed. position: ${start}`);
-        }
-
-        const value = this.input.slice(start, this.position);
-        this.position++;
-        return value;
-    }
-
-    private readIdentifier(): string {
-        const start = this.position;
-
-        // Read until a delimiter appears
-        // Space, dot, comma, parentheses, arithmetic operators
-        // Note that underscores are treated as part of the word
-        const delimiters = new Set([' ', '.', ',', '(', ')', '+', '-', '*', '/']);
-
-        while (this.canRead()) {
-            if (delimiters.has(this.input[this.position])) {
-                break;
-            }
-            this.position++;
-        }
-
-        if (start === this.position) {
-            throw new Error(`Unexpected character. position: ${start}\n${this.getDebugPositionInfo(start)}`);
-        }
-
-        return this.input.slice(start, this.position);
-    }
-
-    /// <summary>
-    /// Skip white space characters and sql comments.
-    /// </summary>
-    private skipWhiteSpacesAndComments(): void {
-        while (true) {
-            if (this.skipWhiteSpace()) {
-                continue;
-            }
-            if (this.skipLineComment()) {
-                continue;
-            }
-            if (this.skipBlockComment()) {
-                continue;
-            }
-            break;
-        }
-    }
-
-    private skipWhiteSpace(): boolean {
-        const start = this.position;
-
-        // Skip tab, newline, and space characters
-        const whitespace = new Set([' ', '\r', '\n', '\t']);
-
-        while (this.canRead()) {
-            if (!whitespace.has(this.input[this.position])) {
-                break;
-            }
-            this.position++;
-        }
-        return start !== this.position;
-    }
-
-    private skipLineComment(): boolean {
-        // At least 2 characters are needed. '--'
-        if (this.isEndOfInput(1)) {
-            return false;
-        }
-
-        if (this.input[this.position] === '-' && this.input[this.position + 1] === '-') {
-            this.position += 2;
-
-            while (this.canRead() && this.input[this.position] !== '\n') {
-                this.position++;
-            }
-            return true;
-        }
-
-        return false;
-    }
-
-    private skipBlockComment(): boolean {
-        // At least 4 characters are needed. '/**/'
-        if (this.isEndOfInput(3)) {
-            return false;
-        }
-
-        // Record the start position of the comment to track error location
-        const start = this.position;
-
-        if (this.input[this.position] === '/' && this.input[this.position + 1] === '*') {
-            this.position += 2;
-
-            while (this.canRead(1)) {
-                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
-                    this.position += 2;
-                    return true;
-                }
-                this.position++;
-            }
-
-            throw new Error(`Block comment is not closed. position: ${start}`);
-        }
-
-        return false;
-    }
-}

package/.vs/CopilotSnapshots/5EF2507B002AC14EA991043C37C36551/state.mpack

@@ -1 +0,0 @@
-��SessionId��$7b50f25e-2a00-4ec1-a991-043c37c36551��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$de68ff60-9d48-4fad-bcbb-b54aa3f6a552�Snapshots�� EF71788BE39818EEA3F0164DC15E1BD7��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\5EF2507B002AC14EA991043C37C36551\60FF68DE489DAD4FBCBBB54AA3F6A552\EF71788BE39818EEA3F0164DC15E1BD7�OriginalFilePath�=C:\Users\mssgm\Documents\GitHub\ts-sample\src\sqlTokenizer.ts�NewFileMarker�

package/.vs/CopilotSnapshots/6AA7610F1FB2B746821E54C28A8D109B/7FB78C0F8CB04F4AA6AF3315277B80F8/FB35DD13E708043D4CD8ACD54E14708F

@@ -1 +0,0 @@
-

package/.vs/CopilotSnapshots/6AA7610F1FB2B746821E54C28A8D109B/state.mpack

@@ -1 +0,0 @@
-��SessionId��$0f61a76a-b21f-46b7-821e-54c28a8d109b��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$0f8cb77f-b08c-4a4f-a6af-3315277b80f8�Snapshots�� FB35DD13E708043D4CD8ACD54E14708F��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\6AA7610F1FB2B746821E54C28A8D109B\7FB78C0F8CB04F4AA6AF3315277B80F8\FB35DD13E708043D4CD8ACD54E14708F�OriginalFilePath�>C:\Users\mssgm\Documents\GitHub\ts-sample\ts-sample\.gitignore�NewFileMarker�

package/.vs/CopilotSnapshots/7F905F8BAC0DAD49BD507CA45C569AFD/5782E13CAB03A04E9E1EC5692F892170/BA5D2B145A59265D63FA3B3584344B02

@@ -1,5 +0,0 @@
-import { TokenType } from "../src/enums/tokenType";
-import { SqlTokenizer } from "../src/sqlTokenizer";
-
-// e.g 1+1, 1*2, 1/3, 1%4
-