rawsql-ts 0.1.0-beta.5 → 0.1.0-beta.7

package/.vs/CopilotSnapshots/C1E2C1A9C1DCFA46B1D5F1C6F36A152D/BC91C0FD38D8694BB992A8DEC1833670/148399016C7971F5592DAADBB9523118

@@ -1,120 +0,0 @@
-import { SqlTokenizer } from "../src/sqlTokenizer";
-
-test('tokenizes integer number', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('123');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('123');
-});
-
-test('tokenizes positive number', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('+123');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('123');
-});
-
-test('tokenizes decimal number', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('123.456');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('123.456');
-});
-
-test('tokenizes number starting with dot', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('.456');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('0.456');
-});
-
-test('tokenizes hexadecimal number', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('0x1A3F');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('0x1A3F');
-});
-
-test('tokenizes exponential notation', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('1.23e+10');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-});
-
-test('tokenizes negative integer number', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('-123');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('-123');
-});
-
-test('tokenizes negative decimal number', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('-123.456');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('-123.456');
-});
-
-test('tokenizes negative number starting with dot', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('-.456');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('-0.456');
-});
-
-test('tokenizes negative exponential notation', () => {
-    // Arrange
-    const tokenizer = new SqlTokenizer('-1.23e+10');
-
-    // Act
-    const tokens = tokenizer.readLexmes();
-
-    // Assert
-    expect(tokens.length).toBe(1);
-    expect(tokens[0].value).toBe('-1.23e+10');
-});

package/.vs/CopilotSnapshots/C1E2C1A9C1DCFA46B1D5F1C6F36A152D/state.mpack

@@ -1 +0,0 @@
-��SessionId��$a9c1e2c1-dcc1-46fa-b1d5-f1c6f36a152d��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$fdc091bc-d838-4b69-b992-a8dec1833670�Snapshots�� 148399016C7971F5592DAADBB9523118��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\C1E2C1A9C1DCFA46B1D5F1C6F36A152D\BC91C0FD38D8694BB992A8DEC1833670\148399016C7971F5592DAADBB9523118�OriginalFilePath�?C:\Users\mssgm\Documents\GitHub\ts-sample\tests\literal.test.ts�NewFileMarker�

package/.vs/CopilotSnapshots/C4D8E85893425E4F86F18850F5CA5C08/6C0950E97063C547A72D737E618436B0/1CF9939611B0D194AC81951A171DF046

@@ -1,9 +0,0 @@
-
-
-/*
-以下の記号で始まる文字列は演算子として扱う
-また、以下の記号が続く場合、それらをまとめて演算子として扱う
-        '+', '-', '*', '/', '%', // Arithmetic operators
-        '~', '@', '#', '$', '^', '&', // Special symbols
-        '!', '<', '>', '=', '|', '\\', // Other symbols
-  */

package/.vs/CopilotSnapshots/C4D8E85893425E4F86F18850F5CA5C08/6C0950E97063C547A72D737E618436B0/DFF70C60F68ECD9A4E4C3C74E4CC4DEE

@@ -1,84 +0,0 @@
-import { BaseTokenReader } from './BaseTokenReader';
-import { Lexeme } from '../models/Lexeme';
-
-/**
- * Manages and coordinates multiple token readers
- */
-export class TokenReaderManager {
-    private readers: BaseTokenReader[];
-    private input: string;
-    private position: number;
-
-    constructor(input: string, position: number = 0) {
-        this.input = input;
-        this.position = position;
-        this.readers = [];
-    }
-
-    /**
-     * Register a token reader
-     * @param reader The reader to register
-     * @returns This manager instance for chaining
-     */
-    public register(reader: BaseTokenReader): TokenReaderManager {
-        this.readers.push(reader);
-        return this;
-    }
-
-    /**
-     * Register multiple token readers
-     * @param readers The readers to register
-     * @returns This manager instance for chaining
-     */
-    public registerAll(readers: BaseTokenReader[]): TokenReaderManager {
-        readers.forEach(reader => this.register(reader));
-        return this;
-    }
-
-    /**
-     * Update the position for all readers
-     */
-    private setPosition(position: number): void {
-        this.position = position;
-        for (const reader of this.readers) {
-            reader.setPosition(position);
-        }
-    }
-
-    /**
-     * Try to read a token using all registered readers
-     * @param previous The previous token, if any
-     * @returns The lexeme if a reader could read it, null otherwise
-     */
-    public tryRead(position: number, previous: Lexeme | null): Lexeme | null {
-        this.setPosition(position);
-        for (const reader of this.readers) {
-            const lexeme = reader.tryRead(previous);
-            if (lexeme) {
-                return lexeme;
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Get the maximum position among all readers
-     */
-    public getMaxPosition(): number {
-        let maxPosition = this.position;
-        for (const reader of this.readers) {
-            const position = reader.getPosition();
-            if (position > maxPosition) {
-                maxPosition = position;
-            }
-        }
-        return maxPosition;
-    }
-
-    /**
-     * Get the input string
-     */
-    public getInput(): string {
-        return this.input;
-    }
-}

package/.vs/CopilotSnapshots/C4D8E85893425E4F86F18850F5CA5C08/state.mpack

@@ -1 +0,0 @@
-��SessionId��$58e8d8c4-4293-4f5e-86f1-8850f5ca5c08��Id�4Microsoft.VisualStudio.Conversations.Chat.HelpWindow�CorrelationStates���CorrelationId�$e950096c-6370-47c5-a72d-737e618436b0�Snapshots�� DFF70C60F68ECD9A4E4C3C74E4CC4DEE��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\C4D8E85893425E4F86F18850F5CA5C08\6C0950E97063C547A72D737E618436B0\DFF70C60F68ECD9A4E4C3C74E4CC4DEE�OriginalFilePath�PC:\Users\mssgm\Documents\GitHub\ts-sample\src\tokenReaders\TokenReaderManager.ts�NewFileMarker�� 1CF9939611B0D194AC81951A171DF046��SavedFileName١C:\Users\mssgm\Documents\GitHub\ts-sample\.vs\CopilotSnapshots\C4D8E85893425E4F86F18850F5CA5C08\6C0950E97063C547A72D737E618436B0\1CF9939611B0D194AC81951A171DF046�OriginalFilePath�QC:\Users\mssgm\Documents\GitHub\ts-sample\src\tokenReaders\OperatorTokenReader.ts�NewFileMarker�

package/.vs/CopilotSnapshots/C5C3CA4C37B14A47B6BC753D09F69117/03EA1376F04C0D468FD0851119159D34/44EB81A1663981E3F52FFFECE3A2918E

@@ -1,148 +0,0 @@
-import { StringUtils } from "./stringUtils";
-
-export enum KeywordMatchResult {
-    NotAKeyword,     // "キーワードではない"
-    PartialOnly,     // "途中（ここで終わることはない）"
-    PartialOrFinal,  // "途中または終端（ここで止まることはある）"
-    Final            // "終端（これ以上長いキーワードはない）"
-}
-
-export class KeywordTrieReader {
-    private input: string;
-    private trie: KeywordTrie;
-
-    constructor(input: string, trie: KeywordTrie) {
-        this.input = input;
-        this.trie = trie;
-        this.trie.reset();
-    }
-
-    private isEndOfInput(position: number, shift: number = 0): boolean {
-        return position + shift >= this.input.length;
-    }
-
-    private canRead(position: number, shift: number = 0): boolean {
-        return !this.isEndOfInput(position, shift);
-    }
-
-    public readKeyword(position: number): { keyword: string, newPosition: number } | null {
-        if (this.isEndOfInput(position)) {
-            return null;
-        }
-
-        // reset trie node
-        this.trie.reset();
-        const result = StringUtils.tryReadRegularIdentifier(this.input, position);
-
-        if (result === null) {
-            return null;
-        }
-
-        let matchResult = this.trie.pushLexeme(result.identifier);
-
-        if (matchResult === KeywordMatchResult.NotAKeyword) {
-            return null;
-        }
-
-        if (matchResult === KeywordMatchResult.Final) {
-            return {
-                keyword: result.identifier,
-                newPosition: result.newPosition
-            };
-        }
-
-        // multi-word keyword
-        let lexeme = result.identifier;
-        position = StringUtils.skipWhiteSpacesAndComments(this.input, result.newPosition);
-
-        while (this.canRead(position)) {
-            const previousMatchResult = matchResult;
-
-            const result = StringUtils.tryReadRegularIdentifier(this.input, position);
-
-            if (result !== null) {
-                matchResult = this.trie.pushLexeme(result.identifier);
-
-                if (matchResult === KeywordMatchResult.NotAKeyword) {
-                    if (previousMatchResult === KeywordMatchResult.PartialOrFinal) {
-                        break;
-                    } else {
-                        return null;
-                    }
-                }
-
-                lexeme += ' ' + result.identifier;
-                position = StringUtils.skipWhiteSpacesAndComments(this.input, result.newPosition);
-
-                if (matchResult === KeywordMatchResult.Final) {
-                    break;
-                }
-            } else if (previousMatchResult === KeywordMatchResult.PartialOrFinal) {
-                break;
-            } else {
-                return null;
-            }
-        }
-        return {
-            keyword: lexeme,
-            newPosition: position
-        };
-    }
-}
-
-export class KeywordTrie {
-    private root: Map<string, any> = new Map();
-    private currentNode: Map<string, any>;
-    
-    // Add a caching property to avoid regex pattern creation for each check
-    private hasEndProperty: boolean = false;
-    private hasMoreProperties: boolean = false;
-
-    constructor(keywords: string[][]) {
-        // initialize root node
-        for (const keyword of keywords) {
-            this.addKeyword(keyword);
-        }
-        // set current node to root
-        this.currentNode = this.root;
-    }
-
-    private addKeyword(keyword: string[]) {
-        let node = this.root;
-        for (const word of keyword) {
-            if (!node.has(word)) {
-                node.set(word, new Map());
-            }
-            node = node.get(word);
-        }
-        node.set("__end__", true);
-    }
-
-    public reset(): void {
-        this.currentNode = this.root;
-        this.hasEndProperty = false;
-        this.hasMoreProperties = false;
-    }
-
-    public pushLexeme(lexeme: string): KeywordMatchResult {
-        if (!this.currentNode.has(lexeme)) {
-            return KeywordMatchResult.NotAKeyword;
-        }
-
-        // move to next node
-        this.currentNode = this.currentNode.get(lexeme);
-        
-        // Cache property checks to avoid repeated operations
-        this.hasEndProperty = this.currentNode.has("__end__");
-        this.hasMoreProperties = this.currentNode.size > (this.hasEndProperty ? 1 : 0);
-
-        if (this.hasEndProperty && !this.hasMoreProperties) {
-            return KeywordMatchResult.Final;
-        }
-        if (this.hasEndProperty && this.hasMoreProperties) {
-            return KeywordMatchResult.PartialOrFinal;
-        }
-
-        return KeywordMatchResult.PartialOnly;
-    }
-}

package/.vs/CopilotSnapshots/C5C3CA4C37B14A47B6BC753D09F69117/03EA1376F04C0D468FD0851119159D34/E602628EF6D764C2C267810C92906DBA

@@ -1,265 +0,0 @@
-import { BaseTokenReader } from './BaseTokenReader';
-import { TokenType } from '../enums/tokenType';
-import { Lexeme } from '../models/Lexeme';
-import { CharLookupTable } from '../utils/charLookupTable';
-import { KeywordTrie, KeywordTrieReader } from '../utils/KeywordTrie';
-
-/**
- * Reads SQL literal tokens (numbers, strings)
- */
-
-const keywords = [
-    ["null"],
-    ["true"],
-    ["false"],
-    ["current_date"],
-    ["current_time"],
-    ["current_timestamp"],
-    ["localtime"],
-    ["localtimestamp"],
-    ["unbounded"],
-    ["normalized"],
-    ["nfc", "normalized"],
-    ["nfd", "normalized"],
-    ["nfkc", "normalized"],
-    ["nfkd", "normalized"],
-    ["nfc"],
-    ["nfd"],
-    ["nfkc"],
-    ["nfkd"]
-];
-const trie = new KeywordTrie(keywords);
-
-// Prefix sets for quick checks
-const SINGLE_CHAR_ESCAPED_PREFIX = new Set(['e\'', 'E\'', 'x\'', 'X\'', 'b\'',  'B\'']);
-const UNICODE_ESCAPED_PREFIX = new Set(['u&\'', 'U&\'']);
-
-export class LiteralTokenReader extends BaseTokenReader {
-    /**
-     * Try to read a literal token
-     */
-    public tryRead(previous: Lexeme | null): Lexeme | null {
-        if (this.isEndOfInput()) {
-            return null;
-        }
-
-        const char = this.input[this.position];
-
-        // Check for keyword literals    
-        const keyword = this.tryReadKeyword();
-        if (keyword) {
-            return keyword;
-        }
-
-        // Decimal token starting with a dot
-        if (char === '.' && this.canRead(1) && CharLookupTable.isDigit(this.input[this.position + 1])) {
-            return this.createLexeme(TokenType.Literal, this.readDigit());
-        }
-
-        // String literal
-        if (char === '\'') {
-            const value = this.readSingleQuotedString();
-            return this.createLexeme(TokenType.Literal, value);
-        }
-
-        // Digit tokens
-        if (CharLookupTable.isDigit(char)) {
-            return this.createLexeme(TokenType.Literal, this.readDigit());
-        }
-
-        // Signed number
-        if ((char === '+' || char === '-') && this.isValidNumericPrefix(previous)) {
-            const sign = char;
-            this.position++;
-
-            // Skip whitespace after sign
-            let pos = this.position;
-            while (this.canRead() && CharLookupTable.isWhitespace(this.input[this.position])) {
-                this.position++;
-            }
-
-            if (this.canRead() && (
-                CharLookupTable.isDigit(this.input[this.position]) ||
-                (this.input[this.position] === '.' &&
-                    this.canRead(1) &&
-                    CharLookupTable.isDigit(this.input[this.position + 1]))
-            )) {
-                return this.createLexeme(
-                    TokenType.Literal,
-                    sign === '-' ? sign + this.readDigit() : this.readDigit()
-                );
-            }
-
-            // Not a number, restore position
-            this.position = pos - 1; // Adjust for the increment at the beginning
-        }
-
-        // Check for prefixed literals
-        return this.tryReadEscapedLiteral();
-    }
-
-    private tryReadKeyword(): Lexeme | null {
-        // Check for keyword literals    
-        const reader = new KeywordTrieReader(this.input, trie);
-        const result = reader.readKeyword(this.position);
-        if (result) {
-            this.position = result.newPosition;
-            return this.createLexeme(TokenType.Literal, result.keyword);
-        }
-        return null;
-    }
-
-    /**
-     * Check if the current context allows for a signed number
-     */
-    private isValidNumericPrefix(previous: Lexeme | null): boolean {
-        return previous === null ||
-            (previous.type !== TokenType.Literal &&
-                previous.type !== TokenType.Identifier);
-    }
-
-    /**
-     * Read a numeric value
-     */
-    private readDigit(): string {
-        const start = this.position;
-        let hasDot = false;
-        let hasExponent = false;
-
-        // Consider 0x, 0b, 0o
-        if (this.canRead(1) &&
-            this.input[this.position] === '0' &&
-            "xbo".includes(this.input[this.position + 1].toLowerCase())) {
-
-            const prefixType = this.input[this.position + 1].toLowerCase();
-            this.position += 2;
-
-            // Continue to get numeric and hexadecimal notation strings
-            const isHex = prefixType === 'x';
-            while (this.canRead()) {
-                const c = this.input[this.position];
-                if (CharLookupTable.isDigit(c) || (isHex && CharLookupTable.isHexChar(c))) {
-                    this.position++;
-                } else {
-                    break;
-                }
-            }
-
-            return this.input.slice(start, this.position);
-        }
-
-        // If starting with dot, note it
-        if (this.input[start] === '.') {
-            hasDot = true;
-            this.position++;
-        }
-
-        // Consider decimal point and exponential notation
-        while (this.canRead()) {
-            const char = this.input[this.position];
-
-            if (char === '.' && !hasDot) {
-                hasDot = true;
-            } else if ((char === 'e' || char === 'E') && !hasExponent) {
-                hasExponent = true;
-                if (this.canRead(1) && (this.input[this.position + 1] === '+' || this.input[this.position + 1] === '-')) {
-                    this.position++;
-                }
-            } else if (!CharLookupTable.isDigit(char)) {
-                break;
-            }
-
-            this.position++;
-        }
-
-        if (start === this.position) {
-            throw new Error(`Unexpected character. position: ${start}\n${this.getDebugPositionInfo(start)}`);
-        }
-
-        if (this.input[start] === '.') {
-            // If the number starts with a dot, add 0 to the front
-            return '0' + this.input.slice(start, this.position);
-        }
-
-        return this.input.slice(start, this.position);
-    }
-
-    /**
-     * Read a string literal
-     */
-    private readSingleQuotedString(): string {
-        const start = this.position;
-
-        this.read('\'');
-
-        while (this.canRead()) {
-            const char = this.input[this.position];
-
-            // escape character check
-            if (char === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
-                this.position += 2;
-                continue;
-            }
-            else if (char === '\'') {
-                break;
-            }
-            this.position++;
-        }
-
-        if (this.isEndOfInput()) {
-            throw new Error(`Single quote is not closed. position: ${start}\n${this.getDebugPositionInfo(start)}`);
-        }
-
-        const value = this.input.slice(start, this.position);
-        this.position++;
-        return value;
-    }
-
-    /**
-     * Try to read an escaped literal like e'...', x'...', etc.
-     */
-    private tryReadEscapedLiteral(): Lexeme | null {
-        const start = this.position;
-
-        // Check for prefixed literals: e', x', b'
-        if (this.canRead(1) && SINGLE_CHAR_ESCAPED_PREFIX.has(this.input.slice(start, start + 2))) {
-            return this.readPrefixedLiteral(start, 2);
-        }
-
-        // Check for unicode literal: u&'
-        if (this.canRead(2) && UNICODE_ESCAPED_PREFIX.has( this.input.slice(start, start + 3))) {
-            return this.readPrefixedLiteral(start, 3);
-        }
-
-        return null;
-    }
-
-    private readPrefixedLiteral(start: number, prefixLength: number): Lexeme {
-        // Skip the prefix
-        this.position += prefixLength;
-
-        // Read until the closing quote
-        while (this.canRead()) {
-            if (this.input[this.position] === '\\' && this.canRead(1) && this.input[this.position + 1] === '\'') {
-                // Skip escaped single quote
-                this.position += 2;
-                continue;
-            }
-            else if (this.input[this.position] === '\'') {
-                // Found closing quote
-                this.position++;
-                break;
-            }
-            this.position++;
-        }
-
-        if (this.position <= start + prefixLength) {
-            throw new Error(`Closing delimiter is not found. position: ${start}\n${this.getDebugPositionInfo(start)}`);
-        }
-
-        return {
-            type: TokenType.Literal,
-            value: this.input.slice(start, this.position)
-        };
-    }
-}