rawsql-ts 0.1.0-beta.1

package/.vs/CopilotSnapshots/AF947096A34634478C5D084B8442CD81/1FCDCC65B992954DB78BD1F46892312B/E602628EF6D764C2C267810C92906DBA

@@ -0,0 +1,253 @@
+import { BaseTokenReader } from './BaseTokenReader';
+import { TokenType } from '../enums/tokenType';
+import { Lexeme } from '../models/Lexeme';
+import { CharLookupTable } from '../utils/charLookupTable';
+import { KeywordTrie, KeywordTrieReader } from '../utils/KeywordTrie';
+
+/**
+ * Reads SQL literal tokens (numbers, strings)
+ */
+
+const keywords = [
+    ["null"],
+    ["true"],
+    ["false"],
+    ["current_date"],
+    ["current_time"],
+    ["current_timestamp"],
+    ["localtime"],
+    ["localtimestamp"],
+    ["unbounded"],
+    ["normalized"],
+    ["nfc", "normalized"],
+    ["nfd", "normalized"],
+    ["nfkc", "normalized"],
+    ["nfkd", "normalized"],
+    ["nfc"],
+    ["nfd"],
+    ["nfkc"],
+    ["nfkd"]
+];
+
+export class LiteralTokenReader extends BaseTokenReader {
+    /**
+     * Try to read a literal token
+     */
+    public tryRead(previous: Lexeme | null): Lexeme | null {
+        if (this.isEndOfInput()) {
+            return null;
+        }
+
+        const char = this.input[this.position];
+
+        // Check for keyword literals
+        const keywordTrie = new KeywordTrie(keywords);
+        const keywordReader = new KeywordTrieReader(this.input, this.position, keywordTrie);
+        const keywordResult = keywordReader.readKeyword();
+        if (keywordResult) {
+            this.position = keywordResult.newPosition;
+            return this.createLexeme(TokenType.Literal, keywordResult.keyword);
+        }
+
+        // Decimal token starting with a dot
+        if (char === '.' && this.canRead(1) && CharLookupTable.isDigit(this.input[this.position + 1])) {
+            return this.createLexeme(TokenType.Literal, this.readDigit());
+        }
+
+        // String literal
+        if (char === '\'') {
+            const value = this.readSingleQuotedString();
+            return this.createLexeme(TokenType.Literal, value);
+        }
+
+        // Digit tokens
+        if (CharLookupTable.isDigit(char)) {
+            return this.createLexeme(TokenType.Literal, this.readDigit());
+        }
+
+        // Signed number
+        if ((char === '+' || char === '-') && this.isValidNumericPrefix(previous)) {
+            const sign = char;
+            this.position++;
+
+            // Skip whitespace after sign
+            let pos = this.position;
+            while (this.canRead() && CharLookupTable.isWhitespace(this.input[this.position])) {
+                this.position++;
+            }
+
+            if (this.canRead() && (
+                CharLookupTable.isDigit(this.input[this.position]) ||
+                (this.input[this.position] === '.' &&
+                    this.canRead(1) &&
+                    CharLookupTable.isDigit(this.input[this.position + 1]))
+            )) {
+                return this.createLexeme(
+                    TokenType.Literal,
+                    sign === '-' ? sign + this.readDigit() : this.readDigit()
+                );
+            }
+
+            // Not a number, restore position
+            this.position = pos - 1; // Adjust for the increment at the beginning
+        }
+
+        // Check for prefixed literals
+        return this.tryReadEscapedLiteral();
+    }
+
+    /**
+     * Check if the current context allows for a signed number
+     */
+    private isValidNumericPrefix(previous: Lexeme | null): boolean {
+        return previous === null ||
+            (previous.type !== TokenType.Literal &&
+                previous.type !== TokenType.Identifier);
+    }
+
+    /**
+     * Read a numeric value
+     */
+    private readDigit(): string {
+        const start = this.position;
+        let hasDot = false;
+        let hasExponent = false;
+
+        // Consider 0x, 0b, 0o
+        if (this.canRead(1) &&
+            this.input[this.position] === '0' &&
+            "xbo".includes(this.input[this.position + 1].toLowerCase())) {
+
+            const prefixType = this.input[this.position + 1].toLowerCase();
+            this.position += 2;
+
+            // Continue to get numeric and hexadecimal notation strings
+            const isHex = prefixType === 'x';
+            while (this.canRead()) {
+                const c = this.input[this.position];
+                if (CharLookupTable.isDigit(c) || (isHex && CharLookupTable.isHexChar(c))) {
+                    this.position++;
+                } else {
+                    break;
+                }
+            }
+
+            return this.input.slice(start, this.position);
+        }
+
+        // If starting with dot, note it
+        if (this.input[start] === '.') {
+            hasDot = true;
+            this.position++;
+        }
+
+        // Consider decimal point and exponential notation
+        while (this.canRead()) {
+            const char = this.input[this.position];
+
+            if (char === '.' && !hasDot) {
+                hasDot = true;
+            } else if ((char === 'e' || char === 'E') && !hasExponent) {
+                hasExponent = true;
+                if (this.canRead(1) && (this.input[this.position + 1] === '+' || this.input[this.position + 1] === '-')) {
+                    this.position++;
+                }
+            } else if (!CharLookupTable.isDigit(char)) {
+                break;
+            }
+
+            this.position++;
+        }
+
+        if (start === this.position) {
+            throw new Error(`Unexpected character. position: ${start}\n${this.getDebugPositionInfo(start)}`);
+        }
+
+        if (this.input[start] === '.') {
+            // If the number starts with a dot, add 0 to the front
+            return '0' + this.input.slice(start, this.position);
+        }
+
+        return this.input.slice(start, this.position);
+    }
+
+    /**
+     * Read a string literal
+     */
+    private readSingleQuotedString(): string {
+        const start = this.position;
+
+        this.read('\'');
+
+        while (this.canRead()) {
+            const char = this.input[this.position];
+
+            // escape character check
+            if (char === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
+                this.position += 2;
+                continue;
+            }
+            else if (char === '\'') {
+                break;
+            }
+            this.position++;
+        }
+
+        if (this.isEndOfInput()) {
+            throw new Error(`Single quote is not closed. position: ${start}\n${this.getDebugPositionInfo(start)}`);
+        }
+
+        const value = this.input.slice(start, this.position);
+        this.position++;
+        return value;
+    }
+
+    /**
+     * Try to read an escaped literal like e'...', x'...', etc.
+     */
+    private tryReadEscapedLiteral(): Lexeme | null {
+        const start = this.position;
+
+        // Check for prefixed literals: e', x', b'
+        const prefix = new Set(['e\'', 'x\'', 'b\'']);
+        if (this.canRead(1) && prefix.has(this.input.slice(start, start + 2).toLocaleLowerCase())) {
+            return this.readPrefixedLiteral(start, 2);
+        }
+
+        // Check for unicode literal: u&'
+        if (this.canRead(2) && this.input.slice(start, start + 3).toLocaleLowerCase() === 'u&\'') {
+            return this.readPrefixedLiteral(start, 3);
+        }
+
+        return null;
+    }
+
+    private readPrefixedLiteral(start: number, prefixLength: number): Lexeme {
+        // Skip the prefix
+        this.position += prefixLength;
+
+        // Read until the closing quote
+        while (this.canRead()) {
+            if (this.input[this.position] === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
+                // Skip escaped single quote
+                this.position += 2;
+                continue;
+            }
+            else if (this.input[this.position] === '\'') {
+                // Found closing quote
+                this.position++;
+                break;
+            }
+            this.position++;
+        }
+
+        if (this.position <= start + prefixLength) {
+            throw new Error(`Closing delimiter is not found. position: ${start}\n${this.getDebugPositionInfo(start)}`);
+        }
+
+        return {
+            type: TokenType.Literal,
+            value: this.input.slice(start, this.position)
+        };
+    }
+}

package/.vs/CopilotSnapshots/AF947096A34634478C5D084B8442CD81/state.mpack

@@ -0,0 +1 @@
+��SessionId��$967094af-46a3-4734-8c5d-084b8442cd81��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$65cccd1f-92b9-4d95-b78b-d1f46892312b�Snapshots�� 44EB81A1663981E3F52FFFECE3A2918E��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\AF947096A34634478C5D084B8442CD81\1FCDCC65B992954DB78BD1F46892312B\44EB81A1663981E3F52FFFECE3A2918E�OriginalFilePath�BC:\Users\mssgm\Documents\GitHub\ts-sample\src\utils\KeywordTrie.ts�NewFileMarker�� E602628EF6D764C2C267810C92906DBA��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\AF947096A34634478C5D084B8442CD81\1FCDCC65B992954DB78BD1F46892312B\E602628EF6D764C2C267810C92906DBA�OriginalFilePath�PC:\Users\mssgm\Documents\GitHub\ts-sample\src\tokenReaders\LiteralTokenReader.ts�NewFileMarker�

package/.vs/CopilotSnapshots/B1DED422A45D5740BA4EE745C04B2B0B/226F822A22D6BD44BCF56BAA47FA274C/EF71788BE39818EEA3F0164DC15E1BD7

@@ -0,0 +1,174 @@
+import { Lexeme } from './models/Lexeme';
+import { IdentifierTokenReader } from './tokenReaders/IdentifierTokenReader';
+import { LiteralTokenReader } from './tokenReaders/LiteralTokenReader';
+import { ParameterTokenReader } from './tokenReaders/ParameterTokenReader';
+import { SymbolTokenReader } from './tokenReaders/SymbolTokenReader';
+
+export class SqlTokenizer {
+    /// <summary>
+    /// Input string.
+    /// </summary>
+    private input: string;
+
+    /// <summary>
+    /// Current position in the input string.
+    /// </summary>
+    private position: number;
+
+    constructor(input: string) {
+        this.input = input;
+        this.position = 0;
+    }
+
+    private isEndOfInput(shift: number = 0): boolean {
+        return this.position + shift >= this.input.length;
+    }
+
+    private canRead(shift: number = 0): boolean {
+        return !this.isEndOfInput(shift);
+    }
+
+    public readLexmes(): Lexeme[] {
+        const lexemes: Lexeme[] = [];
+
+        // 推定されるトークン数に基づいて配列を事前に確保 (平均的に5文字ごとに1トークン)
+        const estimatedCapacity = Math.max(10, Math.ceil(this.input.length / 5));
+        lexemes.length = 0;
+
+        // 開始時に空白とコメントをスキップ
+        this.skipWhiteSpacesAndComments();
+
+        // 各トークンリーダーを初期化
+        const symbolReader = new SymbolTokenReader(this.input, this.position);
+        const identifierReader = new IdentifierTokenReader(this.input, this.position);
+        const literalReader = new LiteralTokenReader(this.input, this.position);
+        const parameterReader = new ParameterTokenReader(this.input, this.position);
+
+        // 前のトークンを追跡
+        let previous: Lexeme | null = null;
+
+        // 入力の終わりに達するまでトークンを読み取る
+        while (this.canRead()) {
+            let lexeme: Lexeme | null = null;
+
+            // 各リーダーで順番に読み取りを試みる
+            parameterReader.setPosition(this.position);
+            lexeme = parameterReader.tryRead(previous);
+
+            if (!lexeme) {
+                literalReader.setPosition(this.position);
+                lexeme = literalReader.tryRead(previous);
+            }
+
+            if (!lexeme) {
+                identifierReader.setPosition(this.position);
+                lexeme = identifierReader.tryRead(previous);
+            }
+
+            if (!lexeme) {
+                symbolReader.setPosition(this.position);
+                lexeme = symbolReader.tryRead(previous);
+            }
+
+            // いずれかのリーダーでトークンが読み取れた場合
+            if (lexeme) {
+                lexemes.push(lexeme);
+                previous = lexeme;
+
+                // 位置を更新
+                const newPos = Math.max(
+                    symbolReader.getPosition(),
+                    literalReader.getPosition(),
+                    identifierReader.getPosition(),
+                    parameterReader.getPosition()
+                );
+                this.position = newPos;
+
+                // トークンの後の空白とコメントをスキップ
+                this.skipWhiteSpacesAndComments();
+            } else {
+                // 例外
+                throw new Error(`Unexpected character. actual: ${this.input[this.position]}, position: ${this.position}`);
+            }
+        }
+
+        return lexemes;
+    }
+
+    /// <summary>
+    /// Skip white space characters and sql comments.
+    /// </summary>
+    private skipWhiteSpacesAndComments(): void {
+        while (true) {
+            if (this.skipWhiteSpace()) {
+                continue;
+            }
+            if (this.skipLineComment()) {
+                continue;
+            }
+            if (this.skipBlockComment()) {
+                continue;
+            }
+            break;
+        }
+    }
+
+    private skipWhiteSpace(): boolean {
+        const start = this.position;
+
+        // Skip tab, newline, and space characters
+        const whitespace = new Set([' ', '\r', '\n', '\t']);
+
+        while (this.canRead()) {
+            if (!whitespace.has(this.input[this.position])) {
+                break;
+            }
+            this.position++;
+        }
+        return start !== this.position;
+    }
+
+    private skipLineComment(): boolean {
+        // At least 2 characters are needed. '--'
+        if (this.isEndOfInput(1)) {
+            return false;
+        }
+
+        if (this.input[this.position] === '-' && this.input[this.position + 1] === '-') {
+            this.position += 2;
+
+            while (this.canRead() && this.input[this.position] !== '\n') {
+                this.position++;
+            }
+            return true;
+        }
+
+        return false;
+    }
+
+    private skipBlockComment(): boolean {
+        // At least 4 characters are needed. '/**/'
+        if (this.isEndOfInput(3)) {
+            return false;
+        }
+
+        // Record the start position of the comment to track error location
+        const start = this.position;
+
+        if (this.input[this.position] === '/' && this.input[this.position + 1] === '*') {
+            this.position += 2;
+
+            while (this.canRead(1)) {
+                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
+                    this.position += 2;
+                    return true;
+                }
+                this.position++;
+            }
+
+            throw new Error(`Block comment is not closed. position: ${start}`);
+        }
+
+        return false;
+    }
+}

package/.vs/CopilotSnapshots/B1DED422A45D5740BA4EE745C04B2B0B/state.mpack

@@ -0,0 +1 @@
+��SessionId��$22d4deb1-5da4-4057-ba4e-e745c04b2b0b��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$2a826f22-d622-44bd-bcf5-6baa47fa274c�Snapshots�� EF71788BE39818EEA3F0164DC15E1BD7��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\B1DED422A45D5740BA4EE745C04B2B0B\226F822A22D6BD44BCF56BAA47FA274C\EF71788BE39818EEA3F0164DC15E1BD7�OriginalFilePath�=C:\Users\mssgm\Documents\GitHub\ts-sample\src\sqlTokenizer.ts�NewFileMarker�

package/.vs/CopilotSnapshots/B5DD5E50ACD0FD4998564F0FDE7FAEE0/97908D1D83AD2D4EA7082388DE07CBB4/44EB81A1663981E3F52FFFECE3A2918E

@@ -0,0 +1,148 @@
+import { StringUtils } from "./stringUtils";
+
+export enum KeywordMatchResult {
+    NotAKeyword,     // "キーワードではない"
+    PartialOnly,     // "途中（ここで終わることはない）"
+    PartialOrFinal,  // "途中または終端（ここで止まることはある）"
+    Final            // "終端（これ以上長いキーワードはない）"
+}
+
+export class KeywordTrieReader {
+    private input: string;
+    private trie: KeywordTrie;
+
+    constructor(input: string, trie: KeywordTrie) {
+        this.input = input;
+        this.trie = trie;
+        this.trie.reset();
+    }
+
+    private isEndOfInput(position: number, shift: number = 0): boolean {
+        return position + shift >= this.input.length;
+    }
+
+    private canRead(position: number, shift: number = 0): boolean {
+        return !this.isEndOfInput(position, shift);
+    }
+
+    public readKeyword(position: number): { keyword: string, newPosition: number } | null {
+        if (this.isEndOfInput(position)) {
+            return null;
+        }
+
+        // reset trie node
+        this.trie.reset();
+        const result = StringUtils.tryReadRegularIdentifier(this.input, position);
+
+        if (result === null) {
+            return null;
+        }
+
+        let matchResult = this.trie.pushLexeme(result.identifier);
+
+        if (matchResult === KeywordMatchResult.NotAKeyword) {
+            return null;
+        }
+
+        if (matchResult === KeywordMatchResult.Final) {
+            return {
+                keyword: result.identifier,
+                newPosition: result.newPosition
+            };
+        }
+
+        // multi-word keyword
+        let lexeme = result.identifier;
+        position = StringUtils.skipWhiteSpacesAndComments(this.input, result.newPosition);
+
+        while (this.canRead(position)) {
+            const previousMatchResult = matchResult;
+
+            const result = StringUtils.tryReadRegularIdentifier(this.input, position);
+
+            if (result !== null) {
+                matchResult = this.trie.pushLexeme(result.identifier);
+
+                if (matchResult === KeywordMatchResult.NotAKeyword) {
+                    if (previousMatchResult === KeywordMatchResult.PartialOrFinal) {
+                        break;
+                    } else {
+                        return null;
+                    }
+                }
+
+                lexeme += ' ' + result.identifier;
+                position = StringUtils.skipWhiteSpacesAndComments(this.input, result.newPosition);
+
+                if (matchResult === KeywordMatchResult.Final) {
+                    break;
+                }
+            } else if (previousMatchResult === KeywordMatchResult.PartialOrFinal) {
+                break;
+            } else {
+                return null;
+            }
+        }
+        return {
+            keyword: lexeme,
+            newPosition: position
+        };
+    }
+}
+
+export class KeywordTrie {
+    private root: Map<string, any> = new Map();
+    private currentNode: Map<string, any>;
+    
+    // Add a caching property to avoid regex pattern creation for each check
+    private hasEndProperty: boolean = false;
+    private hasMoreProperties: boolean = false;
+
+    constructor(keywords: string[][]) {
+        // initialize root node
+        for (const keyword of keywords) {
+            this.addKeyword(keyword);
+        }
+        // set current node to root
+        this.currentNode = this.root;
+    }
+
+    private addKeyword(keyword: string[]) {
+        let node = this.root;
+        for (const word of keyword) {
+            if (!node.has(word)) {
+                node.set(word, new Map());
+            }
+            node = node.get(word);
+        }
+        node.set("__end__", true);
+    }
+
+    public reset(): void {
+        this.currentNode = this.root;
+        this.hasEndProperty = false;
+        this.hasMoreProperties = false;
+    }
+
+    public pushLexeme(lexeme: string): KeywordMatchResult {
+        if (!this.currentNode.has(lexeme)) {
+            return KeywordMatchResult.NotAKeyword;
+        }
+
+        // move to next node
+        this.currentNode = this.currentNode.get(lexeme);
+        
+        // Cache property checks to avoid repeated operations
+        this.hasEndProperty = this.currentNode.has("__end__");
+        this.hasMoreProperties = this.currentNode.size > (this.hasEndProperty ? 1 : 0);
+
+        if (this.hasEndProperty && !this.hasMoreProperties) {
+            return KeywordMatchResult.Final;
+        }
+        if (this.hasEndProperty && this.hasMoreProperties) {
+            return KeywordMatchResult.PartialOrFinal;
+        }
+
+        return KeywordMatchResult.PartialOnly;
+    }
+}

package/.vs/CopilotSnapshots/B5DD5E50ACD0FD4998564F0FDE7FAEE0/state.mpack

@@ -0,0 +1 @@
+��SessionId��$505eddb5-d0ac-49fd-9856-4f0fde7faee0��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$1d8d9097-ad83-4e2d-a708-2388de07cbb4�Snapshots�� 44EB81A1663981E3F52FFFECE3A2918E��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\B5DD5E50ACD0FD4998564F0FDE7FAEE0\97908D1D83AD2D4EA7082388DE07CBB4\44EB81A1663981E3F52FFFECE3A2918E�OriginalFilePath�BC:\Users\mssgm\Documents\GitHub\ts-sample\src\utils\KeywordTrie.ts�NewFileMarker�