rawsql-ts 0.1.0-beta.9 → 0.1.1-beta.2

package/dist/esm/tokenReaders/CommandTokenReader.js

@@ -0,0 +1,141 @@
+import { BaseTokenReader } from "./BaseTokenReader";
+import { TokenType } from '../models/Lexeme';
+import { KeywordTrie } from "../models/KeywordTrie";
+import { KeywordParser } from "../parsers/KeywordParser";
+// Commands are those that require a dedicated parser.
+// Keywords composed of multiple words are also considered commands.
+// The exception is "type". Since types can be user-defined and cannot be accurately identified, they are treated as Identifiers.
+const joinTrie = new KeywordTrie([
+    ["join"],
+    ["inner", "join"],
+    ["cross", "join"],
+    ["left", "join"],
+    ["left", "outer", "join"],
+    ["right", "join"],
+    ["right", "outer", "join"],
+    ["full", "join"],
+    ["full", "outer", "join"],
+    ["natural", "join"],
+    ["natural", "inner", "join"],
+    ["natural", "left", "join"],
+    ["natural", "left", "outer", "join"],
+    ["natural", "right", "join"],
+    ["natural", "right", "outer", "join"],
+    ["natural", "full", "join"],
+    ["natural", "full", "outer", "join"],
+]);
+const keywordTrie = new KeywordTrie([
+    ["with"],
+    ["recursive"],
+    ["materialized"],
+    ["not", "materialized"],
+    ["select"],
+    ["from"],
+    ["distinct"],
+    ["distinct", "on"],
+    ["where"],
+    ["group", "by"],
+    ["having"],
+    ["order", "by"],
+    ["limit"],
+    ["offset"],
+    // for
+    ["for"],
+    ["update"],
+    ["share"],
+    ["key", "share"],
+    ["no", "key", "update"],
+    // set operations
+    ["union"],
+    ["union", "all"],
+    ["intersect"],
+    ["intersect", "all"],
+    ["except"],
+    ["except", "all"],
+    // between and
+    ["beteen"],
+    // window functions
+    ["window"],
+    ["over"],
+    ["partition", "by"],
+    ["range"],
+    ["range"],
+    ["rows"],
+    ["groups"],
+    ["groups"],
+    // window frame
+    ["current", "row"],
+    ["unbounded", "preceding"],
+    ["unbounded", "following"],
+    ["preceding"],
+    ["following"],
+    // table join commands
+    ["on"],
+    ["using"],
+    ["lateral"],
+    // case 
+    ["case"],
+    ["case", "when"],
+    ["when"],
+    ["then"],
+    ["else"],
+    ["end"],
+    // others
+    ["insert", "into"],
+    ["update"],
+    ["delete", "from"],
+    ["merge", "into"],
+    ["matched"],
+    ["not", "matched"],
+    ["update", "set"],
+    ["do", "nothing"],
+    ["values"],
+    ["set"],
+    ["returning"],
+    ["create", "table"],
+    ["create", "temporary", "table"],
+    ["tablesample"],
+    ["array"],
+    // cast
+    ["as"],
+    // odrder
+    ["asc"],
+    ["desc"],
+    ["nulls", "first"],
+    ["nulls", "last"],
+]);
+const keywordParser = new KeywordParser(keywordTrie);
+export const joinkeywordParser = new KeywordParser(joinTrie);
+export class CommandTokenReader extends BaseTokenReader {
+    tryRead(previous) {
+        if (this.isEndOfInput()) {
+            return null;
+        }
+        const keywordJoin = joinkeywordParser.parse(this.input, this.position);
+        if (keywordJoin !== null) {
+            this.position = keywordJoin.newPosition;
+            return this.createLexeme(TokenType.Command, keywordJoin.keyword);
+        }
+        // Check for keyword identifiers
+        const keyword = keywordParser.parse(this.input, this.position);
+        if (keyword !== null) {
+            this.position = keyword.newPosition;
+            return this.createLexeme(TokenType.Command, keyword.keyword);
+        }
+        // check hint clause
+        if (this.canRead(2) && this.input[this.position] === '/' && this.input[this.position + 1] === '*' && this.input[this.position + 2] === '+') {
+            this.position += 3;
+            const start = this.position;
+            while (this.position + 1 < this.input.length) {
+                if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
+                    this.position += 2;
+                    return this.createLexeme(TokenType.Command, '/*+ ' + this.input.slice(start, this.position - 2).trim() + ' */');
+                }
+                this.position++;
+            }
+            throw new Error(`Block comment is not closed. position: ${this.position}`);
+        }
+        return null;
+    }
+}
+//# sourceMappingURL=CommandTokenReader.js.map

package/dist/esm/tokenReaders/CommandTokenReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CommandTokenReader.js","sourceRoot":"","sources":["../../../src/tokenReaders/CommandTokenReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAU,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEzD,sDAAsD;AACtD,oEAAoE;AACpE,iIAAiI;AAEjI,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC;IAC7B,CAAC,MAAM,CAAC;IACR,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;IACzB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC;IAC1B,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;IAEzB,CAAC,SAAS,EAAE,MAAM,CAAC;IACnB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;IAC5B,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC;IAC3B,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;IACpC,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;IAC5B,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC;IACrC,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC;IAC3B,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;CACvC,CAAC,CAAC;AACH,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC;IAChC,CAAC,MAAM,CAAC;IACR,CAAC,WAAW,CAAC;IACb,CAAC,cAAc,CAAC;IAChB,CAAC,KAAK,EAAE,cAAc,CAAC;IACvB,CAAC,QAAQ,CAAC;IACV,CAAC,MAAM,CAAC;IACR,CAAC,UAAU,CAAC;IACZ,CAAC,UAAU,EAAE,IAAI,CAAC;IAClB,CAAC,OAAO,CAAC;IACT,CAAC,OAAO,EAAE,IAAI,CAAC;IACf,CAAC,QAAQ,CAAC;IACV,CAAC,OAAO,EAAE,IAAI,CAAC;IACf,CAAC,OAAO,CAAC;IACT,CAAC,QAAQ,CAAC;IACV,MAAM;IACN,CAAC,KAAK,CAAC;IACP,CAAC,QAAQ,CAAC;IACV,CAAC,OAAO,CAAC;IACT,CAAC,KAAK,EAAE,OAAO,CAAC;IAChB,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC;IACvB,iBAAiB;IACjB,CAAC,OAAO,CAAC;IACT,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,WAAW,CAAC;IACb,CAAC,WAAW,EAAE,KAAK,CAAC;IACpB,CAAC,QAAQ,CAAC;IACV,CAAC,QAAQ,EAAE,KAAK,CAAC;IACjB,cAAc;IACd,CAAC,QAAQ,CAAC;IACV,mBAAmB;IACnB,CAAC,QAAQ,CAAC;IACV,CAAC,MAAM,CAAC;IACR,CAAC,WAAW,EAAE,IAAI,CAAC;IACnB,CAAC,OAAO,CAAC;IACT,CAAC,OAAO,CAAC;IACT,CAAC,MAAM,CAAC;IACR,CAAC,QAAQ,CAAC;IACV,CAAC,QAAQ,CAAC;IACV,eAAe;IACf,CAAC,SAAS,EAAE,KAAK,CAAC;IAClB,CAAC,WAAW,EAAE,WAAW,CAAC;IAC1B,CAAC,WAAW,EAAE,WAAW,CAAC;IAC1B,CAAC,WAAW,CAAC;IACb,CAAC,WAAW,CAAC;IACb,sBAAsB;IACtB,CAAC,IAAI,CAAC;IACN,CAAC,OAAO,CAAC;IACT,CAAC,SAAS,CAAC;IACX,QAAQ;IACR,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;IACR,CAAC,KAAK,CAAC;IACP,SAAS;IACT,CAAC,QAAQ,EAAE,MAAM,CAAC;IAClB,CAAC,QAAQ,CAAC;IACV,CAAC,QAAQ,EAAE,MAAM,CAAC;IAClB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,SAAS,CAAC;IACX,CAAC,KAAK,EAAE,SAAS,CAAC;IAClB,CAAC,QAAQ,EAAE,KAAK,CAAC;IACjB,CAAC,IAAI,EAAE,SAAS,CAAC;IACjB,CAAC,QAAQ,CAAC;IACV,CAAC,KAAK,CAAC;IACP,CAAC,WAAW,CAAC;IACb,CAAC,QAAQ,EAAE,OAAO,CAAC;IACnB,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC;IAChC,CAAC,aAAa,CAAC;IACf,CAAC,OAAO,CAAC;IACT,OAAO;IACP,CAAC,IAAI,CAAC;IACN,SAAS;IACT,CAAC,KAAK,CAAC;IACP,CAAC,MAAM,CAAC;IACR,CAAC,OAAO,EAAE,OAAO,CAAC;IAClB,CAAC,OAAO,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AACH,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,WAAW,CAAC,CAAC;AACrD,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC;AAE7D,MAAM,OAAO,kBAAmB,SAAQ,eAAe;IAC5C,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvE,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,WAAW,CAAC;YACxC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;QACrE,CAAC;QAED,gCAAgC;QAChC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;YACpC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACjE,CAAC;QAED,oBAAoB;QACpB,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACzI,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC5B,OAAO,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC3C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBAC7E,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;oBACnB,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC;gBACpH,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}

package/dist/esm/tokenReaders/FunctionTokenReader.js

@@ -0,0 +1,41 @@
+import { BaseTokenReader } from './BaseTokenReader';
+import { TokenType } from '../models/Lexeme';
+import { StringUtils } from '../utils/stringUtils';
+import { KeywordTrie } from '../models/KeywordTrie';
+import { KeywordParser } from '../parsers/KeywordParser';
+const trie = new KeywordTrie([
+    ["grouping", "sets"],
+]);
+const keywordParser = new KeywordParser(trie);
+/**
+ * Reads SQL identifier tokens
+ */
+export class FunctionTokenReader extends BaseTokenReader {
+    /**
+     * Try to read an identifier token
+     */
+    tryRead(previous) {
+        if (this.isEndOfInput()) {
+            return null;
+        }
+        // Check for keyword identifiers
+        const keyword = keywordParser.parse(this.input, this.position);
+        if (keyword !== null) {
+            this.position = keyword.newPosition;
+            return this.createLexeme(TokenType.Function, keyword.keyword);
+        }
+        // Regular identifier
+        const result = StringUtils.tryReadRegularIdentifier(this.input, this.position);
+        if (!result) {
+            return null;
+        }
+        this.position = result.newPosition;
+        // peek next token 
+        var shift = StringUtils.readWhiteSpaceAndComment(this.input, this.position).position - this.position;
+        if (this.canRead(shift) && this.input[this.position + shift] === '(') {
+            return this.createLexeme(TokenType.Function, result.identifier);
+        }
+        return null;
+    }
+}
+//# sourceMappingURL=FunctionTokenReader.js.map

package/dist/esm/tokenReaders/FunctionTokenReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FunctionTokenReader.js","sourceRoot":"","sources":["../../../src/tokenReaders/FunctionTokenReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAU,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEzD,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC;IACzB,CAAC,UAAU,EAAE,MAAM,CAAC;CACvB,CAAC,CAAC;AACH,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC;AAE9C;;GAEG;AACH,MAAM,OAAO,mBAAoB,SAAQ,eAAe;IACpD;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,gCAAgC;QAChC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;YACpC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAClE,CAAC;QAED,qBAAqB;QACrB,MAAM,MAAM,GAAG,WAAW,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;QAEnC,mBAAmB;QACnB,IAAI,KAAK,GAAG,WAAW,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAErG,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YACnE,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}

package/dist/esm/tokenReaders/IdentifierTokenReader.js

@@ -0,0 +1,66 @@
+import { BaseTokenReader } from './BaseTokenReader';
+import { TokenType } from '../models/Lexeme';
+import { StringUtils } from '../utils/stringUtils';
+/**
+ * Reads SQL identifier tokens
+ */
+export class IdentifierTokenReader extends BaseTokenReader {
+    /**
+     * Try to read an identifier token
+     */
+    tryRead(previous) {
+        if (this.isEndOfInput()) {
+            return null;
+        }
+        const char = this.input[this.position];
+        // wildcard identifier
+        if (char === '*') {
+            // Assume that the OperatorTokenReader is executed before the IdentifierTokenReader.
+            // Since we have determined that the OperatorTokenReader is not an Operator,
+            // we treat '*' here as a wildcard identifier.
+            this.position++;
+            return this.createLexeme(TokenType.Identifier, char);
+        }
+        // MySQL escaped identifier (escape character is backtick)
+        if (char === '`') {
+            const identifier = this.readEscapedIdentifier('`');
+            return this.createLexeme(TokenType.Identifier, identifier);
+        }
+        // Postgres escaped identifier (escape character is double quote)
+        if (char === '"') {
+            const identifier = this.readEscapedIdentifier('"');
+            return this.createLexeme(TokenType.Identifier, identifier);
+        }
+        // SQLServer escaped identifier (escape character is square bracket)
+        if (char === '[' && (previous === null || previous.value !== "array")) {
+            const identifier = this.readEscapedIdentifier(']');
+            return this.createLexeme(TokenType.Identifier, identifier);
+        }
+        // Regular identifier
+        const result = StringUtils.readRegularIdentifier(this.input, this.position);
+        this.position = result.newPosition;
+        return this.createLexeme(TokenType.Identifier, result.identifier);
+    }
+    /**
+     * Read an escaped identifier (surrounded by delimiters)
+     */
+    readEscapedIdentifier(delimiter) {
+        const start = this.position;
+        // Skip the opening delimiter
+        this.position++;
+        while (this.canRead()) {
+            if (this.input[this.position] === delimiter) {
+                break;
+            }
+            this.position++;
+        }
+        if (start === this.position) {
+            throw new Error(`Closing delimiter is not found. position: ${start}, delimiter: ${delimiter}\n${this.getDebugPositionInfo(start)}}`);
+        }
+        // Skip the closing delimiter
+        this.position++;
+        // exclude the delimiter
+        return this.input.slice(start + 1, this.position - 1);
+    }
+}
+//# sourceMappingURL=IdentifierTokenReader.js.map

package/dist/esm/tokenReaders/IdentifierTokenReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IdentifierTokenReader.js","sourceRoot":"","sources":["../../../src/tokenReaders/IdentifierTokenReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAU,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,eAAe;IACtD;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,sBAAsB;QACtB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,oFAAoF;YACpF,4EAA4E;YAC5E,8CAA8C;YAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACzD,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAED,oEAAoE;QACpE,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,CAAC,EAAE,CAAC;YACpE,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAED,qBAAqB;QACrB,MAAM,MAAM,GAAG,WAAW,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5E,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;QACnC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,SAAiB;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE5B,6BAA6B;QAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC1C,MAAM;YACV,CAAC;YACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,6CAA6C,KAAK,gBAAgB,SAAS,KAAK,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzI,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,wBAAwB;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;CACJ"}

package/dist/esm/tokenReaders/LiteralTokenReader.js

@@ -0,0 +1,185 @@
+import { BaseTokenReader } from './BaseTokenReader';
+import { TokenType } from '../models/Lexeme';
+import { CharLookupTable } from '../utils/charLookupTable';
+import { KeywordParser } from '../parsers/KeywordParser';
+import { KeywordTrie } from '../models/KeywordTrie';
+/**
+ * Reads SQL literal tokens (numbers, strings)
+ */
+const keywords = [
+    ["null"],
+    ["true"],
+    ["false"],
+    ["current_date"],
+    ["current_time"],
+    ["current_timestamp"],
+    ["localtime"],
+    ["localtimestamp"],
+    ["unbounded"],
+    ["normalized"],
+    ["nfc", "normalized"],
+    ["nfd", "normalized"],
+    ["nfkc", "normalized"],
+    ["nfkd", "normalized"],
+    ["nfc"],
+    ["nfd"],
+    ["nfkc"],
+    ["nfkd"],
+];
+const trie = new KeywordTrie(keywords);
+export const literalKeywordParser = new KeywordParser(trie);
+export class LiteralTokenReader extends BaseTokenReader {
+    /**
+     * Try to read a literal token
+     */
+    tryRead(previous) {
+        if (this.isEndOfInput()) {
+            return null;
+        }
+        const char = this.input[this.position];
+        // Check for keyword literals    
+        const keyword = this.tryReadKeyword();
+        if (keyword) {
+            return keyword;
+        }
+        // Decimal token starting with a dot
+        if (char === '.' && this.canRead(1) && CharLookupTable.isDigit(this.input[this.position + 1])) {
+            return this.createLexeme(TokenType.Literal, this.readDigit());
+        }
+        // String literal
+        if (char === '\'') {
+            const value = this.readSingleQuotedString(false);
+            return this.createLexeme(TokenType.Literal, value);
+        }
+        // Digit tokens
+        if (CharLookupTable.isDigit(char)) {
+            return this.createLexeme(TokenType.Literal, this.readDigit());
+        }
+        // Signed number
+        if ((char === '+' || char === '-') && this.isValidNumericPrefix(previous)) {
+            const sign = char;
+            this.position++;
+            // Skip whitespace after sign
+            const pos = this.position;
+            while (this.canRead() && CharLookupTable.isWhitespace(this.input[this.position])) {
+                this.position++;
+            }
+            if (this.canRead() && (CharLookupTable.isDigit(this.input[this.position]) ||
+                (this.input[this.position] === '.' &&
+                    this.canRead(1) &&
+                    CharLookupTable.isDigit(this.input[this.position + 1])))) {
+                return this.createLexeme(TokenType.Literal, sign === '-' ? sign + this.readDigit() : this.readDigit());
+            }
+            // Not a number, restore position
+            this.position = pos - 1; // Adjust for the increment at the beginning
+        }
+        return null;
+    }
+    tryReadKeyword() {
+        // Check for keyword literals
+        const result = literalKeywordParser.parse(this.input, this.position);
+        if (result) {
+            this.position = result.newPosition;
+            return this.createLexeme(TokenType.Literal, result.keyword);
+        }
+        return null;
+    }
+    /**
+     * Check if the current context allows for a signed number
+     */
+    isValidNumericPrefix(previous) {
+        return previous === null ||
+            (previous.type !== TokenType.Literal &&
+                previous.type !== TokenType.Identifier);
+    }
+    /**
+     * Read a numeric value
+     */
+    readDigit() {
+        const start = this.position;
+        let hasDot = false;
+        let hasExponent = false;
+        // Consider 0x, 0b, 0o
+        if (this.canRead(1) &&
+            this.input[this.position] === '0' &&
+            "xbo".includes(this.input[this.position + 1].toLowerCase())) {
+            const prefixType = this.input[this.position + 1].toLowerCase();
+            this.position += 2;
+            // Continue to get numeric and hexadecimal notation strings
+            const isHex = prefixType === 'x';
+            while (this.canRead()) {
+                const c = this.input[this.position];
+                if (CharLookupTable.isDigit(c) || (isHex && CharLookupTable.isHexChar(c))) {
+                    this.position++;
+                }
+                else {
+                    break;
+                }
+            }
+            return this.input.slice(start, this.position);
+        }
+        // If starting with dot, note it
+        if (this.input[start] === '.') {
+            hasDot = true;
+            this.position++;
+        }
+        // Consider decimal point and exponential notation
+        while (this.canRead()) {
+            const char = this.input[this.position];
+            if (char === '.' && !hasDot) {
+                hasDot = true;
+            }
+            else if ((char === 'e' || char === 'E') && !hasExponent) {
+                hasExponent = true;
+                if (this.canRead(1) && (this.input[this.position + 1] === '+' || this.input[this.position + 1] === '-')) {
+                    this.position++;
+                }
+            }
+            else if (!CharLookupTable.isDigit(char)) {
+                break;
+            }
+            this.position++;
+        }
+        if (start === this.position) {
+            throw new Error(`Unexpected character. position: ${start}\n${this.getDebugPositionInfo(start)}`);
+        }
+        if (this.input[start] === '.') {
+            // If the number starts with a dot, add 0 to the front
+            return '0' + this.input.slice(start, this.position);
+        }
+        return this.input.slice(start, this.position);
+    }
+    /**
+     * Read a string literal
+     */
+    readSingleQuotedString(includeSingleQuote) {
+        const start = this.position;
+        let closed = false;
+        this.read("'");
+        while (this.canRead()) {
+            const char = this.input[this.position];
+            this.position++;
+            // escape character check
+            if (char === "\\" && this.canRead(1)) {
+                this.position++;
+                continue;
+            }
+            else if (char === '\'') {
+                closed = true;
+                break;
+            }
+        }
+        if (closed === false) {
+            throw new Error(`Single quote is not closed. position: ${start}\n${this.getDebugPositionInfo(start)}`);
+        }
+        if (includeSingleQuote) {
+            const value = this.input.slice(start, this.position);
+            return value;
+        }
+        else {
+            const value = this.input.slice(start + 1, this.position - 1);
+            return value;
+        }
+    }
+}
+//# sourceMappingURL=LiteralTokenReader.js.map

package/dist/esm/tokenReaders/LiteralTokenReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LiteralTokenReader.js","sourceRoot":"","sources":["../../../src/tokenReaders/LiteralTokenReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAU,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD;;GAEG;AAEH,MAAM,QAAQ,GAAG;IACb,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;IACR,CAAC,OAAO,CAAC;IACT,CAAC,cAAc,CAAC;IAChB,CAAC,cAAc,CAAC;IAChB,CAAC,mBAAmB,CAAC;IACrB,CAAC,WAAW,CAAC;IACb,CAAC,gBAAgB,CAAC;IAClB,CAAC,WAAW,CAAC;IACb,CAAC,YAAY,CAAC;IACd,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,MAAM,EAAE,YAAY,CAAC;IACtB,CAAC,MAAM,EAAE,YAAY,CAAC;IACtB,CAAC,KAAK,CAAC;IACP,CAAC,KAAK,CAAC;IACP,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;CACX,CAAC;AACF,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;AACvC,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAM,OAAO,kBAAmB,SAAQ,eAAe;IACnD;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,iCAAiC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACV,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5F,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,iBAAiB;QACjB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;YACjD,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;QAED,eAAe;QACf,IAAI,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,gBAAgB;QAChB,IAAI,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxE,MAAM,IAAI,GAAG,IAAI,CAAC;YAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,6BAA6B;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC1B,OAAO,IAAI,CAAC,OAAO,EAAE,IAAI,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/E,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,CAClB,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAClD,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG;oBAC9B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACf,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAC9D,EAAE,CAAC;gBACA,OAAO,IAAI,CAAC,YAAY,CACpB,SAAS,CAAC,OAAO,EACjB,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAC5D,CAAC;YACN,CAAC;YAED,iCAAiC;YACjC,IAAI,CAAC,QAAQ,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,4CAA4C;QACzE,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,cAAc;QAClB,6BAA6B;QAC7B,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrE,IAAI,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;YACnC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,QAAuB;QAChD,OAAO,QAAQ,KAAK,IAAI;YACpB,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO;gBAChC,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,UAAU,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACK,SAAS;QACb,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,WAAW,GAAG,KAAK,CAAC;QAExB,sBAAsB;QACtB,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG;YACjC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAE9D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/D,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YAEnB,2DAA2D;YAC3D,MAAM,KAAK,GAAG,UAAU,KAAK,GAAG,CAAC;YACjC,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBACpB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACpC,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACpB,CAAC;qBAAM,CAAC;oBACJ,MAAM;gBACV,CAAC;YACL,CAAC;YAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC;QAED,gCAAgC;QAChC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,GAAG,IAAI,CAAC;YACd,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;QAED,kDAAkD;QAClD,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEvC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,IAAI,CAAC;YAClB,CAAC;iBAAM,IAAI,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxD,WAAW,GAAG,IAAI,CAAC;gBACnB,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBACtG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACpB,CAAC;YACL,CAAC;iBAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,MAAM;YACV,CAAC;YAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,KAAK,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YAC5B,sDAAsD;YACtD,OAAO,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,kBAA2B;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,yBAAyB;YACzB,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,SAAS;YACb,CAAC;iBACI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,GAAG,IAAI,CAAC;gBACd,MAAM;YACV,CAAC;QACL,CAAC;QAED,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,KAAK,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3G,CAAC;QAED,IAAI,kBAAkB,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrD,OAAO,KAAK,CAAC;QACjB,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;CACJ"}

package/dist/esm/tokenReaders/OperatorTokenReader.js

@@ -0,0 +1,94 @@
+import { BaseTokenReader } from './BaseTokenReader';
+import { TokenType } from '../models/Lexeme';
+import { CharLookupTable } from '../utils/charLookupTable';
+import { KeywordParser } from '../parsers/KeywordParser';
+import { KeywordTrie } from '../models/KeywordTrie';
+const trie = new KeywordTrie([
+    // binary
+    ["and"],
+    ["or"],
+    ["is"],
+    ["is", "not"],
+    ["is", "distinct", "from"],
+    ["is", "not", "distinct", "from"],
+    ["like"],
+    ["in"],
+    ["exists"],
+    ["between"],
+    ["not", "like"],
+    ["not", "in"],
+    ["not", "exists"],
+    ["not", "between"],
+    ["escape"], // e.g. '10% OFF on all items' like '10\%%' escape '\'
+    ["uescape"], // e.g. U&'d!0061t!+000061' uescape '!'
+    ["similar"], // e.g. substring('abcdef' similar '%#"cd#"%' escape '#')
+    ["placing"], // e.g. overlay('abcdef' placing 'cd' from 3 for 2)
+    // unary
+    ["not"],
+    // unary - trim
+    ["both"],
+    ["leading"],
+    ["trailing"],
+    ["both", "from"], // Postgres
+    ["leading", "from"], // Postgres
+    ["trailing", "from"], // Postgres
+    // unary - extract
+    ["year", "from"],
+    ["month", "from"],
+    ["day", "from"],
+    ["hour", "from"],
+    ["minute", "from"],
+    ["second", "from"],
+    ["dow", "from"],
+    ["doy", "from"],
+    ["isodow", "from"],
+    ["quarter", "from"],
+    ["week", "from"],
+    ["epoch", "from"],
+    ["at", "time", "zone"],
+    ["interval"],
+    // The following are not considered operators.
+    // ["from"], can be used as an operator only within the substring function, but it cannot be distinguished from the Form Clause. This will be resolved with a dedicated substring parser.
+    // ["for"], can be used as an operator only within the substring function, but it cannot be distinguished from the For Clause. This will be resolved with a dedicated substring parser.
+]);
+const keywordParser = new KeywordParser(trie);
+export class OperatorTokenReader extends BaseTokenReader {
+    tryRead(previous) {
+        if (this.isEndOfInput()) {
+            return null;
+        }
+        /*
+            NOTE:
+            Asterisks could potentially be wildcard identifiers,
+            but since they're indistinguishable at this stage, they're treated as Operators at the token level.
+            The Parser needs to determine whether they are appropriate Operators or Identifiers.
+        */
+        const char = this.input[this.position];
+        if (CharLookupTable.isOperatorSymbol(char)) {
+            const start = this.position;
+            while (this.canRead() && CharLookupTable.isOperatorSymbol(this.input[this.position])) {
+                // check for `--` and `/*` comments
+                if (this.canRead(1)) {
+                    const current = this.input[this.position];
+                    if (current === '-' && this.input[this.position + 1] === '-') {
+                        break;
+                    }
+                    else if (current === '/' && this.input[this.position + 1] === '*') {
+                        break; // end of operator
+                    }
+                }
+                this.position++;
+            }
+            const resut = this.input.slice(start, this.position);
+            return this.createLexeme(TokenType.Operator, resut);
+        }
+        // Logical operators
+        const result = keywordParser.parse(this.input, this.position);
+        if (result !== null) {
+            this.position = result.newPosition;
+            return this.createLexeme(TokenType.Operator, result.keyword);
+        }
+        return null;
+    }
+}
+//# sourceMappingURL=OperatorTokenReader.js.map

package/dist/esm/tokenReaders/OperatorTokenReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OperatorTokenReader.js","sourceRoot":"","sources":["../../../src/tokenReaders/OperatorTokenReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAU,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC;IACzB,SAAS;IACT,CAAC,KAAK,CAAC;IACP,CAAC,IAAI,CAAC;IACN,CAAC,IAAI,CAAC;IACN,CAAC,IAAI,EAAE,KAAK,CAAC;IACb,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC;IAC1B,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;IACjC,CAAC,MAAM,CAAC;IACR,CAAC,IAAI,CAAC;IACN,CAAC,QAAQ,CAAC;IACV,CAAC,SAAS,CAAC;IACX,CAAC,KAAK,EAAE,MAAM,CAAC;IACf,CAAC,KAAK,EAAE,IAAI,CAAC;IACb,CAAC,KAAK,EAAE,QAAQ,CAAC;IACjB,CAAC,KAAK,EAAE,SAAS,CAAC;IAClB,CAAC,QAAQ,CAAC,EAAE,sDAAsD;IAClE,CAAC,SAAS,CAAC,EAAE,uCAAuC;IACpD,CAAC,SAAS,CAAC,EAAE,yDAAyD;IACtE,CAAC,SAAS,CAAC,EAAE,mDAAmD;IAChE,QAAQ;IACR,CAAC,KAAK,CAAC;IACP,eAAe;IACf,CAAC,MAAM,CAAC;IACR,CAAC,SAAS,CAAC;IACX,CAAC,UAAU,CAAC;IACZ,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW;IAC7B,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,WAAW;IAChC,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,WAAW;IACjC,kBAAkB;IAClB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,KAAK,EAAE,MAAM,CAAC;IACf,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,QAAQ,EAAE,MAAM,CAAC;IAClB,CAAC,QAAQ,EAAE,MAAM,CAAC;IAClB,CAAC,KAAK,EAAE,MAAM,CAAC;IACf,CAAC,KAAK,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,CAAC;IAClB,CAAC,SAAS,EAAE,MAAM,CAAC;IACnB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC;IACtB,CAAC,UAAU,CAAC;IACZ,8CAA8C;IAC9C,yLAAyL;IACzL,uLAAuL;CAC1L,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC;AAE9C,MAAM,OAAO,mBAAoB,SAAQ,eAAe;IAC7C,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED;;;;;UAKE;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,eAAe,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;YAE5B,OAAO,IAAI,CAAC,OAAO,EAAE,IAAI,eAAe,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBACnF,mCAAmC;gBACnC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC1C,IAAI,OAAO,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;wBAC3D,MAAM;oBACV,CAAC;yBAAM,IAAI,OAAO,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;wBAClE,MAAM,CAAC,kBAAkB;oBAC7B,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxD,CAAC;QAED,oBAAoB;QACpB,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9D,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;YACnC,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}

package/dist/esm/tokenReaders/ParameterTokenReader.js

@@ -0,0 +1,40 @@
+import { BaseTokenReader } from './BaseTokenReader';
+import { TokenType } from '../models/Lexeme';
+import { CharLookupTable } from '../utils/charLookupTable';
+/**
+ * Reads SQL parameter tokens (@param, :param, $param, ?)
+ */
+export class ParameterTokenReader extends BaseTokenReader {
+    /**
+     * Try to read a parameter token
+     */
+    tryRead(previous) {
+        if (this.isEndOfInput()) {
+            return null;
+        }
+        const char = this.input[this.position];
+        // named parameter (@param, :param, $param)
+        if (CharLookupTable.isNamedParameterPrefix(char)) {
+            // However, do not recognize as a parameter if the next character is an operator symbol
+            // To avoid postgres `::`
+            if (this.canRead(1) && CharLookupTable.isOperatorSymbol(this.input[this.position + 1])) {
+                return null;
+            }
+            this.position++;
+            // Read the identifier part after the prefix
+            const start = this.position;
+            while (this.canRead() && !CharLookupTable.isDelimiter(this.input[this.position])) {
+                this.position++;
+            }
+            const identifier = this.input.slice(start, this.position);
+            return this.createLexeme(TokenType.Parameter, char + identifier);
+        }
+        // nameless parameter (?)
+        if (char === '?') {
+            this.position++;
+            return this.createLexeme(TokenType.Parameter, char);
+        }
+        return null;
+    }
+}
+//# sourceMappingURL=ParameterTokenReader.js.map

package/dist/esm/tokenReaders/ParameterTokenReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ParameterTokenReader.js","sourceRoot":"","sources":["../../../src/tokenReaders/ParameterTokenReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAU,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAE3D;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,eAAe;IACrD;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,2CAA2C;QAC3C,IAAI,eAAe,CAAC,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;YAE/C,uFAAuF;YACvF,yBAAyB;YACzB,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrF,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,4CAA4C;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC5B,OAAO,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/E,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1D,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,GAAG,UAAU,CAAC,CAAC;QACrE,CAAC;QAED,yBAAyB;QACzB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}

package/dist/esm/tokenReaders/StringSpecifierTokenReader.js

@@ -0,0 +1,27 @@
+import { TokenType } from '../models/Lexeme';
+import { BaseTokenReader } from './BaseTokenReader';
+// Prefix sets for quick checks
+const STRING_SPECIFIERS = new Set(['e\'', 'E\'', 'x\'', 'X\'', 'b\'', 'B\'']);
+const UNICODE_STRING_SPECIFIERS = new Set(['u&\'', 'U&\'']);
+export class StringSpecifierTokenReader extends BaseTokenReader {
+    /**
+     * Try to read an escaped literal like e'...', x'...', etc.
+     */
+    tryRead(previous) {
+        const start = this.position;
+        // Check for prefixed literals: e', x', b'
+        if (this.canRead(1) && STRING_SPECIFIERS.has(this.input.slice(start, start + 2))) {
+            this.position += 1;
+            const result = this.createLexeme(TokenType.StringSpecifier, this.input.slice(start, this.position));
+            return result;
+        }
+        // Check for unicode literal: u&'
+        if (this.canRead(2) && UNICODE_STRING_SPECIFIERS.has(this.input.slice(start, start + 3))) {
+            this.position += 2;
+            const result = this.createLexeme(TokenType.StringSpecifier, this.input.slice(start, this.position));
+            return result;
+        }
+        return null;
+    }
+}
+//# sourceMappingURL=StringSpecifierTokenReader.js.map

package/dist/esm/tokenReaders/StringSpecifierTokenReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StringSpecifierTokenReader.js","sourceRoot":"","sources":["../../../src/tokenReaders/StringSpecifierTokenReader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAU,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,+BAA+B;AAC/B,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAC9E,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAE5D,MAAM,OAAO,0BAA2B,SAAQ,eAAe;IAE3D;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE5B,0CAA0C;QAC1C,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YACnB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpG,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACvF,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YACnB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpG,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}