rawsql-ts 0.1.0-beta.4 → 0.1.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +143 -199
- package/dist/index.js +32 -0
- package/dist/index.js.map +1 -0
- package/dist/models/BinarySelectQuery.js +140 -0
- package/dist/models/BinarySelectQuery.js.map +1 -0
- package/dist/models/Clause.js +318 -0
- package/dist/models/Clause.js.map +1 -0
- package/dist/models/KeywordTrie.js +52 -0
- package/dist/models/KeywordTrie.js.map +1 -0
- package/dist/models/Lexeme.js +21 -0
- package/dist/models/Lexeme.js.map +1 -0
- package/dist/models/SelectQuery.js +10 -0
- package/dist/models/SelectQuery.js.map +1 -0
- package/dist/models/SimpleSelectQuery.js +290 -0
- package/dist/models/SimpleSelectQuery.js.map +1 -0
- package/dist/models/SqlComponent.js +27 -0
- package/dist/models/SqlComponent.js.map +1 -0
- package/dist/models/ValueComponent.js +250 -0
- package/dist/models/ValueComponent.js.map +1 -0
- package/dist/models/ValuesQuery.js +16 -0
- package/dist/models/ValuesQuery.js.map +1 -0
- package/dist/parsers/CommandExpressionParser.js +124 -0
- package/dist/parsers/CommandExpressionParser.js.map +1 -0
- package/dist/parsers/CommonTableParser.js +60 -0
- package/dist/parsers/CommonTableParser.js.map +1 -0
- package/dist/parsers/ForClauseParser.js +56 -0
- package/dist/parsers/ForClauseParser.js.map +1 -0
- package/dist/parsers/FromClauseParser.js +45 -0
- package/dist/parsers/FromClauseParser.js.map +1 -0
- package/dist/parsers/FunctionExpressionParser.js +178 -0
- package/dist/parsers/FunctionExpressionParser.js.map +1 -0
- package/dist/parsers/GroupByParser.js +56 -0
- package/dist/parsers/GroupByParser.js.map +1 -0
- package/dist/parsers/HavingParser.js +34 -0
- package/dist/parsers/HavingParser.js.map +1 -0
- package/dist/parsers/IdentifierParser.js +39 -0
- package/dist/parsers/IdentifierParser.js.map +1 -0
- package/dist/parsers/JoinClauseParser.js +105 -0
- package/dist/parsers/JoinClauseParser.js.map +1 -0
- package/dist/parsers/KeywordParser.js +91 -0
- package/dist/parsers/KeywordParser.js.map +1 -0
- package/dist/parsers/LimitClauseParser.js +48 -0
- package/dist/parsers/LimitClauseParser.js.map +1 -0
- package/dist/parsers/LiteralParser.js +38 -0
- package/dist/parsers/LiteralParser.js.map +1 -0
- package/dist/parsers/OrderByClauseParser.js +75 -0
- package/dist/parsers/OrderByClauseParser.js.map +1 -0
- package/dist/parsers/OverExpressionParser.js +44 -0
- package/dist/parsers/OverExpressionParser.js.map +1 -0
- package/dist/parsers/ParameterExpressionParser.js +15 -0
- package/dist/parsers/ParameterExpressionParser.js.map +1 -0
- package/dist/parsers/ParenExpressionParser.js +33 -0
- package/dist/parsers/ParenExpressionParser.js.map +1 -0
- package/dist/parsers/PartitionByParser.js +51 -0
- package/dist/parsers/PartitionByParser.js.map +1 -0
- package/dist/parsers/SelectClauseParser.js +82 -0
- package/dist/parsers/SelectClauseParser.js.map +1 -0
- package/dist/parsers/SelectQueryParser.js +151 -0
- package/dist/parsers/SelectQueryParser.js.map +1 -0
- package/dist/parsers/SourceAliasExpressionParser.js +48 -0
- package/dist/parsers/SourceAliasExpressionParser.js.map +1 -0
- package/dist/parsers/SourceExpressionParser.js +34 -0
- package/dist/parsers/SourceExpressionParser.js.map +1 -0
- package/dist/parsers/SourceParser.js +116 -0
- package/dist/parsers/SourceParser.js.map +1 -0
- package/dist/parsers/SqlTokenizer.js +174 -0
- package/dist/parsers/SqlTokenizer.js.map +1 -0
- package/dist/parsers/StringSpecifierExpressionParser.js +22 -0
- package/dist/parsers/StringSpecifierExpressionParser.js.map +1 -0
- package/dist/parsers/UnaryExpressionParser.js +30 -0
- package/dist/parsers/UnaryExpressionParser.js.map +1 -0
- package/dist/parsers/ValueParser.js +134 -0
- package/dist/parsers/ValueParser.js.map +1 -0
- package/dist/parsers/ValuesQueryParser.js +86 -0
- package/dist/parsers/ValuesQueryParser.js.map +1 -0
- package/dist/parsers/WhereClauseParser.js +34 -0
- package/dist/parsers/WhereClauseParser.js.map +1 -0
- package/dist/parsers/WindowClauseParser.js +43 -0
- package/dist/parsers/WindowClauseParser.js.map +1 -0
- package/dist/parsers/WindowExpressionParser.js +151 -0
- package/dist/parsers/WindowExpressionParser.js.map +1 -0
- package/dist/parsers/WithClauseParser.js +55 -0
- package/dist/parsers/WithClauseParser.js.map +1 -0
- package/dist/tokenReaders/BaseTokenReader.js +82 -0
- package/dist/tokenReaders/BaseTokenReader.js.map +1 -0
- package/dist/tokenReaders/CommandTokenReader.js +145 -0
- package/dist/tokenReaders/CommandTokenReader.js.map +1 -0
- package/dist/tokenReaders/FunctionTokenReader.js +45 -0
- package/dist/tokenReaders/FunctionTokenReader.js.map +1 -0
- package/dist/tokenReaders/IdentifierTokenReader.js +70 -0
- package/dist/tokenReaders/IdentifierTokenReader.js.map +1 -0
- package/dist/tokenReaders/LiteralTokenReader.js +189 -0
- package/dist/tokenReaders/LiteralTokenReader.js.map +1 -0
- package/dist/tokenReaders/OperatorTokenReader.js +98 -0
- package/dist/tokenReaders/OperatorTokenReader.js.map +1 -0
- package/dist/tokenReaders/ParameterTokenReader.js +44 -0
- package/dist/tokenReaders/ParameterTokenReader.js.map +1 -0
- package/dist/tokenReaders/StringSpecifierTokenReader.js +31 -0
- package/dist/tokenReaders/StringSpecifierTokenReader.js.map +1 -0
- package/dist/tokenReaders/SymbolTokenReader.js +35 -0
- package/dist/tokenReaders/SymbolTokenReader.js.map +1 -0
- package/dist/tokenReaders/TokenReaderManager.js +110 -0
- package/dist/tokenReaders/TokenReaderManager.js.map +1 -0
- package/dist/tokenReaders/TypeTokenReader.js +59 -0
- package/dist/tokenReaders/TypeTokenReader.js.map +1 -0
- package/dist/transformers/CTEBuilder.js +188 -0
- package/dist/transformers/CTEBuilder.js.map +1 -0
- package/dist/transformers/CTECollector.js +384 -0
- package/dist/transformers/CTECollector.js.map +1 -0
- package/dist/transformers/CTEDisabler.js +325 -0
- package/dist/transformers/CTEDisabler.js.map +1 -0
- package/dist/transformers/CTEInjector.js +83 -0
- package/dist/transformers/CTEInjector.js.map +1 -0
- package/dist/transformers/CTENormalizer.js +42 -0
- package/dist/transformers/CTENormalizer.js.map +1 -0
- package/dist/transformers/Formatter.js +452 -0
- package/dist/transformers/Formatter.js.map +1 -0
- package/dist/transformers/QueryNormalizer.js +114 -0
- package/dist/transformers/QueryNormalizer.js.map +1 -0
- package/dist/transformers/SelectValueCollector.js +249 -0
- package/dist/transformers/SelectValueCollector.js.map +1 -0
- package/dist/transformers/SelectableColumnCollector.js +308 -0
- package/dist/transformers/SelectableColumnCollector.js.map +1 -0
- package/dist/transformers/TableSourceCollector.js +384 -0
- package/dist/transformers/TableSourceCollector.js.map +1 -0
- package/dist/transformers/UpstreamSelectQueryFinder.js +129 -0
- package/dist/transformers/UpstreamSelectQueryFinder.js.map +1 -0
- package/dist/utils/charLookupTable.js +73 -0
- package/dist/utils/charLookupTable.js.map +1 -0
- package/dist/utils/stringUtils.js +168 -0
- package/dist/utils/stringUtils.js.map +1 -0
- package/package.json +3 -3
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/dist/vitest.config.js +0 -15
- package/dist/vitest.config.js.map +0 -1
@@ -0,0 +1,82 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.BaseTokenReader = void 0;
|
4
|
+
const Lexeme_1 = require("../models/Lexeme");
|
5
|
+
const stringUtils_1 = require("../utils/stringUtils");
|
6
|
+
/**
|
7
|
+
* Base class for token readers
|
8
|
+
*/
|
9
|
+
class BaseTokenReader {
|
10
|
+
constructor(input, position = 0) {
|
11
|
+
this.input = input;
|
12
|
+
this.position = position;
|
13
|
+
}
|
14
|
+
/**
|
15
|
+
* Get the current position in the input
|
16
|
+
*/
|
17
|
+
getPosition() {
|
18
|
+
return this.position;
|
19
|
+
}
|
20
|
+
/**
|
21
|
+
* Set the position in the input
|
22
|
+
*/
|
23
|
+
setPosition(position) {
|
24
|
+
this.position = position;
|
25
|
+
}
|
26
|
+
/**
|
27
|
+
* Check if we've reached the end of input
|
28
|
+
*/
|
29
|
+
isEndOfInput(shift = 0) {
|
30
|
+
return this.position + shift >= this.input.length;
|
31
|
+
}
|
32
|
+
/**
|
33
|
+
* Check if we can read more characters
|
34
|
+
*/
|
35
|
+
canRead(shift = 0) {
|
36
|
+
return !this.isEndOfInput(shift);
|
37
|
+
}
|
38
|
+
/**
|
39
|
+
* Read an expected character
|
40
|
+
*/
|
41
|
+
read(expectChar) {
|
42
|
+
if (this.isEndOfInput()) {
|
43
|
+
throw new Error(`Unexpected character. expect: ${expectChar}, actual: EndOfInput, position: ${this.position}`);
|
44
|
+
}
|
45
|
+
if (this.input[this.position] !== expectChar) {
|
46
|
+
throw new Error(`Unexpected character. expect: ${expectChar}, actual: ${this.input[this.position]}, position: ${this.position}`);
|
47
|
+
}
|
48
|
+
const char = this.input[this.position];
|
49
|
+
this.position++;
|
50
|
+
return char;
|
51
|
+
}
|
52
|
+
/**
|
53
|
+
* Create a lexeme with the specified type and value
|
54
|
+
*/
|
55
|
+
createLexeme(type, value, maybeType = null, comments = null) {
|
56
|
+
if (type === Lexeme_1.TokenType.Command || type === Lexeme_1.TokenType.Operator || type === Lexeme_1.TokenType.Function) {
|
57
|
+
// Benchmark tests showed that directly calling toLowerCase() is ~5x faster
|
58
|
+
// than first checking if the string is already lowercase.
|
59
|
+
// See benchmarks/lowercase-benchmark.js for detailed performance analysis.
|
60
|
+
return {
|
61
|
+
type,
|
62
|
+
value: value.toLowerCase(),
|
63
|
+
comments: comments,
|
64
|
+
maybeType: maybeType,
|
65
|
+
};
|
66
|
+
}
|
67
|
+
return {
|
68
|
+
type,
|
69
|
+
value,
|
70
|
+
comments: comments,
|
71
|
+
maybeType: maybeType,
|
72
|
+
};
|
73
|
+
}
|
74
|
+
/**
|
75
|
+
* Get debug info for error reporting
|
76
|
+
*/
|
77
|
+
getDebugPositionInfo(errPosition) {
|
78
|
+
return stringUtils_1.StringUtils.getDebugPositionInfo(this.input, errPosition);
|
79
|
+
}
|
80
|
+
}
|
81
|
+
exports.BaseTokenReader = BaseTokenReader;
|
82
|
+
//# sourceMappingURL=BaseTokenReader.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"BaseTokenReader.js","sourceRoot":"","sources":["../../src/tokenReaders/BaseTokenReader.ts"],"names":[],"mappings":";;;AAAA,6CAAqD;AACrD,sDAAmD;AAEnD;;GAEG;AACH,MAAsB,eAAe;IAIjC,YAAY,KAAa,EAAE,WAAmB,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED;;OAEG;IACI,WAAW;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED;;OAEG;IACI,WAAW,CAAC,QAAgB;QAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,QAAgB,CAAC;QACpC,OAAO,IAAI,CAAC,QAAQ,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IACtD,CAAC;IAED;;OAEG;IACO,OAAO,CAAC,QAAgB,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACO,IAAI,CAAC,UAAkB;QAC7B,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iCAAiC,UAAU,mCAAmC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnH,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,UAAU,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,iCAAiC,UAAU,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACrI,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,IAAe,EAAE,KAAa,EAAE,YAA4B,IAAI,EAAE,WAA4B,IAAI;QACrH,IAAI,IAAI,KAAK,kBAAS,CAAC,OAAO,IAAI,IAAI,KAAK,kBAAS,CAAC,QAAQ,IAAI,IAAI,KAAK,kBAAS,CAAC,QAAQ,EAAE,CAAC;YAC3F,2EAA2E;YAC3E,0DAA0D;YAC1D,2EAA2E;YAC3E,OAAO;gBACH,IAAI;gBACJ,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;gBAC1B,QAAQ,EAAE,QAAQ;gBAClB,SAAS,EAAE,SAAS;aACvB,CAAC;QACN,CAAC;QACD,OAAO;YACH,IAAI;YACJ,KAAK;YACL,QAAQ,EAAE,QAAQ;YAClB,SAAS,EAAE,SAAS;SACvB,CAAC;IACN,CAAC;IAED;;OAEG;IACO,oBAAoB,CAAC,WAAmB;QAC9C,OAAO,yBAAW,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IACrE,CAAC;CAQJ;AA1FD,0CA0FC"}
|
@@ -0,0 +1,145 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.CommandTokenReader = exports.joinkeywordParser = void 0;
|
4
|
+
const BaseTokenReader_1 = require("./BaseTokenReader");
|
5
|
+
const Lexeme_1 = require("../models/Lexeme");
|
6
|
+
const KeywordTrie_1 = require("../models/KeywordTrie");
|
7
|
+
const KeywordParser_1 = require("../parsers/KeywordParser");
|
8
|
+
// Commands are those that require a dedicated parser.
|
9
|
+
// Keywords composed of multiple words are also considered commands.
|
10
|
+
// The exception is "type". Since types can be user-defined and cannot be accurately identified, they are treated as Identifiers.
|
11
|
+
const joinTrie = new KeywordTrie_1.KeywordTrie([
|
12
|
+
["join"],
|
13
|
+
["inner", "join"],
|
14
|
+
["cross", "join"],
|
15
|
+
["left", "join"],
|
16
|
+
["left", "outer", "join"],
|
17
|
+
["right", "join"],
|
18
|
+
["right", "outer", "join"],
|
19
|
+
["full", "join"],
|
20
|
+
["full", "outer", "join"],
|
21
|
+
["natural", "join"],
|
22
|
+
["natural", "inner", "join"],
|
23
|
+
["natural", "left", "join"],
|
24
|
+
["natural", "left", "outer", "join"],
|
25
|
+
["natural", "right", "join"],
|
26
|
+
["natural", "right", "outer", "join"],
|
27
|
+
["natural", "full", "join"],
|
28
|
+
["natural", "full", "outer", "join"],
|
29
|
+
]);
|
30
|
+
const keywordTrie = new KeywordTrie_1.KeywordTrie([
|
31
|
+
["with"],
|
32
|
+
["recursive"],
|
33
|
+
["materialized"],
|
34
|
+
["not", "materialized"],
|
35
|
+
["select"],
|
36
|
+
["from"],
|
37
|
+
["distinct"],
|
38
|
+
["distinct", "on"],
|
39
|
+
["where"],
|
40
|
+
["group", "by"],
|
41
|
+
["having"],
|
42
|
+
["order", "by"],
|
43
|
+
["limit"],
|
44
|
+
["offset"],
|
45
|
+
// for
|
46
|
+
["for"],
|
47
|
+
["update"],
|
48
|
+
["share"],
|
49
|
+
["key", "share"],
|
50
|
+
["no", "key", "update"],
|
51
|
+
// set operations
|
52
|
+
["union"],
|
53
|
+
["union", "all"],
|
54
|
+
["intersect"],
|
55
|
+
["intersect", "all"],
|
56
|
+
["except"],
|
57
|
+
["except", "all"],
|
58
|
+
// between and
|
59
|
+
["beteen"],
|
60
|
+
// window functions
|
61
|
+
["window"],
|
62
|
+
["over"],
|
63
|
+
["partition", "by"],
|
64
|
+
["range"],
|
65
|
+
["range"],
|
66
|
+
["rows"],
|
67
|
+
["groups"],
|
68
|
+
["groups"],
|
69
|
+
// window frame
|
70
|
+
["current", "row"],
|
71
|
+
["unbounded", "preceding"],
|
72
|
+
["unbounded", "following"],
|
73
|
+
["preceding"],
|
74
|
+
["following"],
|
75
|
+
// table join commands
|
76
|
+
["on"],
|
77
|
+
["using"],
|
78
|
+
["lateral"],
|
79
|
+
// case
|
80
|
+
["case"],
|
81
|
+
["case", "when"],
|
82
|
+
["when"],
|
83
|
+
["then"],
|
84
|
+
["else"],
|
85
|
+
["end"],
|
86
|
+
// others
|
87
|
+
["insert", "into"],
|
88
|
+
["update"],
|
89
|
+
["delete", "from"],
|
90
|
+
["merge", "into"],
|
91
|
+
["matched"],
|
92
|
+
["not", "matched"],
|
93
|
+
["update", "set"],
|
94
|
+
["do", "nothing"],
|
95
|
+
["values"],
|
96
|
+
["set"],
|
97
|
+
["returning"],
|
98
|
+
["create", "table"],
|
99
|
+
["create", "temporary", "table"],
|
100
|
+
["tablesample"],
|
101
|
+
["array"],
|
102
|
+
// cast
|
103
|
+
["as"],
|
104
|
+
// odrder
|
105
|
+
["asc"],
|
106
|
+
["desc"],
|
107
|
+
["nulls", "first"],
|
108
|
+
["nulls", "last"],
|
109
|
+
]);
|
110
|
+
const keywordParser = new KeywordParser_1.KeywordParser(keywordTrie);
|
111
|
+
exports.joinkeywordParser = new KeywordParser_1.KeywordParser(joinTrie);
|
112
|
+
class CommandTokenReader extends BaseTokenReader_1.BaseTokenReader {
|
113
|
+
tryRead(previous) {
|
114
|
+
if (this.isEndOfInput()) {
|
115
|
+
return null;
|
116
|
+
}
|
117
|
+
const keywordJoin = exports.joinkeywordParser.parse(this.input, this.position);
|
118
|
+
if (keywordJoin !== null) {
|
119
|
+
this.position = keywordJoin.newPosition;
|
120
|
+
return this.createLexeme(Lexeme_1.TokenType.Command, keywordJoin.keyword);
|
121
|
+
}
|
122
|
+
// Check for keyword identifiers
|
123
|
+
const keyword = keywordParser.parse(this.input, this.position);
|
124
|
+
if (keyword !== null) {
|
125
|
+
this.position = keyword.newPosition;
|
126
|
+
return this.createLexeme(Lexeme_1.TokenType.Command, keyword.keyword);
|
127
|
+
}
|
128
|
+
// check hint clause
|
129
|
+
if (this.canRead(2) && this.input[this.position] === '/' && this.input[this.position + 1] === '*' && this.input[this.position + 2] === '+') {
|
130
|
+
this.position += 3;
|
131
|
+
const start = this.position;
|
132
|
+
while (this.position + 1 < this.input.length) {
|
133
|
+
if (this.input[this.position] === '*' && this.input[this.position + 1] === '/') {
|
134
|
+
this.position += 2;
|
135
|
+
return this.createLexeme(Lexeme_1.TokenType.Command, '/*+ ' + this.input.slice(start, this.position - 2).trim() + ' */');
|
136
|
+
}
|
137
|
+
this.position++;
|
138
|
+
}
|
139
|
+
throw new Error(`Block comment is not closed. position: ${this.position}`);
|
140
|
+
}
|
141
|
+
return null;
|
142
|
+
}
|
143
|
+
}
|
144
|
+
exports.CommandTokenReader = CommandTokenReader;
|
145
|
+
//# sourceMappingURL=CommandTokenReader.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"CommandTokenReader.js","sourceRoot":"","sources":["../../src/tokenReaders/CommandTokenReader.ts"],"names":[],"mappings":";;;AAAA,uDAAoD;AACpD,6CAAqD;AACrD,uDAAoD;AACpD,4DAAyD;AAEzD,sDAAsD;AACtD,oEAAoE;AACpE,iIAAiI;AAEjI,MAAM,QAAQ,GAAG,IAAI,yBAAW,CAAC;IAC7B,CAAC,MAAM,CAAC;IACR,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;IACzB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC;IAC1B,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;IAEzB,CAAC,SAAS,EAAE,MAAM,CAAC;IACnB,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;IAC5B,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC;IAC3B,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;IACpC,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;IAC5B,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC;IACrC,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC;IAC3B,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;CACvC,CAAC,CAAC;AACH,MAAM,WAAW,GAAG,IAAI,yBAAW,CAAC;IAChC,CAAC,MAAM,CAAC;IACR,CAAC,WAAW,CAAC;IACb,CAAC,cAAc,CAAC;IAChB,CAAC,KAAK,EAAE,cAAc,CAAC;IACvB,CAAC,QAAQ,CAAC;IACV,CAAC,MAAM,CAAC;IACR,CAAC,UAAU,CAAC;IACZ,CAAC,UAAU,EAAE,IAAI,CAAC;IAClB,CAAC,OAAO,CAAC;IACT,CAAC,OAAO,EAAE,IAAI,CAAC;IACf,CAAC,QAAQ,CAAC;IACV,CAAC,OAAO,EAAE,IAAI,CAAC;IACf,CAAC,OAAO,CAAC;IACT,CAAC,QAAQ,CAAC;IACV,MAAM;IACN,CAAC,KAAK,CAAC;IACP,CAAC,QAAQ,CAAC;IACV,CAAC,OAAO,CAAC;IACT,CAAC,KAAK,EAAE,OAAO,CAAC;IAChB,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC;IACvB,iBAAiB;IACjB,CAAC,OAAO,CAAC;IACT,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,WAAW,CAAC;IACb,CAAC,WAAW,EAAE,KAAK,CAAC;IACpB,CAAC,QAAQ,CAAC;IACV,CAAC,QAAQ,EAAE,KAAK,CAAC;IACjB,cAAc;IACd,CAAC,QAAQ,CAAC;IACV,mBAAmB;IACnB,CAAC,QAAQ,CAAC;IACV,CAAC,MAAM,CAAC;IACR,CAAC,WAAW,EAAE,IAAI,CAAC;IACnB,CAAC,OAAO,CAAC;IACT,CAAC,OAAO,CAAC;IACT,CAAC,MAAM,CAAC;IACR,CAAC,QAAQ,CAAC;IACV,CAAC,QAAQ,CAAC;IACV,eAAe;IACf,CAAC,SAAS,EAAE,KAAK,CAAC;IAClB,CAAC,WAAW,EAAE,WAAW,CAAC;IAC1B,CAAC,WAAW,EAAE,WAAW,CAAC;IAC1B,CAAC,WAAW,CAAC;IACb,CAAC,WAAW,CAAC;IACb,sBAAsB;IACtB,CAAC,IAAI,CAAC;IACN,CAAC,OAAO,CAAC;IACT,CAAC,SAAS,CAAC;IACX,QAAQ;IACR,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;IACR,CAAC,KAAK,CAAC;IACP,SAAS;IACT,CAAC,QAAQ,EAAE,MAAM,CAAC;IAClB,CAAC,QAAQ,CAAC;IACV,CAAC,QAAQ,EAAE,MAAM,CAAC;IAClB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,SAAS,CAAC;IACX,CAAC,KAAK,EAAE,SAAS,CAAC;IAClB,CAAC,QAAQ,EAAE,KAAK,CAAC;IACjB,CAAC,IAAI,EAAE,SAAS,CAAC;IACjB,CAAC,QAAQ,CAAC;IACV,CAAC,KAAK,CAAC;IACP,CAAC,WAAW,CAAC;IACb,CAAC,QAAQ,EAAE,OAAO,CAAC;IACnB,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC;IAChC,CAAC,aAAa,CAAC;IACf,CAAC,OAAO,CAAC;IACT,OAAO;IACP,CAAC,IAAI,CAAC;IACN,SAAS;IACT,CAAC,KAAK,CAAC;IACP,CAAC,MAAM,CAAC;IACR,CAAC,OAAO,EAAE,OAAO,CAAC;IAClB,CAAC,OAAO,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AACH,MAAM,aAAa,GAAG,IAAI,6BAAa,CAAC,WAAW,CAAC,CAAC;AACxC,QAAA,iBAAiB,GAAG,IAAI,6BAAa,CAAC,QAAQ,CAAC,CAAC;AAE7D,MAAa,kBAAmB,SAAQ,iCAAe;IAC5C,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,yBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvE,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,WAAW,CAAC;YACxC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;QACrE,CAAC;QAED,gCAAgC;QAChC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;YACpC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACjE,CAAC;QAED,oBAAoB;QACpB,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACzI,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC5B,OAAO,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC3C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBAC7E,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;oBACnB,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC;gBACpH,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AAnCD,gDAmCC"}
|
@@ -0,0 +1,45 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.FunctionTokenReader = void 0;
|
4
|
+
const BaseTokenReader_1 = require("./BaseTokenReader");
|
5
|
+
const Lexeme_1 = require("../models/Lexeme");
|
6
|
+
const stringUtils_1 = require("../utils/stringUtils");
|
7
|
+
const KeywordTrie_1 = require("../models/KeywordTrie");
|
8
|
+
const KeywordParser_1 = require("../parsers/KeywordParser");
|
9
|
+
const trie = new KeywordTrie_1.KeywordTrie([
|
10
|
+
["grouping", "sets"],
|
11
|
+
]);
|
12
|
+
const keywordParser = new KeywordParser_1.KeywordParser(trie);
|
13
|
+
/**
|
14
|
+
* Reads SQL identifier tokens
|
15
|
+
*/
|
16
|
+
class FunctionTokenReader extends BaseTokenReader_1.BaseTokenReader {
|
17
|
+
/**
|
18
|
+
* Try to read an identifier token
|
19
|
+
*/
|
20
|
+
tryRead(previous) {
|
21
|
+
if (this.isEndOfInput()) {
|
22
|
+
return null;
|
23
|
+
}
|
24
|
+
// Check for keyword identifiers
|
25
|
+
const keyword = keywordParser.parse(this.input, this.position);
|
26
|
+
if (keyword !== null) {
|
27
|
+
this.position = keyword.newPosition;
|
28
|
+
return this.createLexeme(Lexeme_1.TokenType.Function, keyword.keyword);
|
29
|
+
}
|
30
|
+
// Regular identifier
|
31
|
+
const result = stringUtils_1.StringUtils.tryReadRegularIdentifier(this.input, this.position);
|
32
|
+
if (!result) {
|
33
|
+
return null;
|
34
|
+
}
|
35
|
+
this.position = result.newPosition;
|
36
|
+
// peek next token
|
37
|
+
var shift = stringUtils_1.StringUtils.readWhiteSpaceAndComment(this.input, this.position).position - this.position;
|
38
|
+
if (this.canRead(shift) && this.input[this.position + shift] === '(') {
|
39
|
+
return this.createLexeme(Lexeme_1.TokenType.Function, result.identifier);
|
40
|
+
}
|
41
|
+
return null;
|
42
|
+
}
|
43
|
+
}
|
44
|
+
exports.FunctionTokenReader = FunctionTokenReader;
|
45
|
+
//# sourceMappingURL=FunctionTokenReader.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"FunctionTokenReader.js","sourceRoot":"","sources":["../../src/tokenReaders/FunctionTokenReader.ts"],"names":[],"mappings":";;;AAAA,uDAAoD;AACpD,6CAAqD;AACrD,sDAAmD;AACnD,uDAAoD;AACpD,4DAAyD;AAEzD,MAAM,IAAI,GAAG,IAAI,yBAAW,CAAC;IACzB,CAAC,UAAU,EAAE,MAAM,CAAC;CACvB,CAAC,CAAC;AACH,MAAM,aAAa,GAAG,IAAI,6BAAa,CAAC,IAAI,CAAC,CAAC;AAE9C;;GAEG;AACH,MAAa,mBAAoB,SAAQ,iCAAe;IACpD;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,gCAAgC;QAChC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;YACpC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAClE,CAAC;QAED,qBAAqB;QACrB,MAAM,MAAM,GAAG,yBAAW,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;QAEnC,mBAAmB;QACnB,IAAI,KAAK,GAAG,yBAAW,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAErG,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YACnE,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AA/BD,kDA+BC"}
|
@@ -0,0 +1,70 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.IdentifierTokenReader = void 0;
|
4
|
+
const BaseTokenReader_1 = require("./BaseTokenReader");
|
5
|
+
const Lexeme_1 = require("../models/Lexeme");
|
6
|
+
const stringUtils_1 = require("../utils/stringUtils");
|
7
|
+
/**
|
8
|
+
* Reads SQL identifier tokens
|
9
|
+
*/
|
10
|
+
class IdentifierTokenReader extends BaseTokenReader_1.BaseTokenReader {
|
11
|
+
/**
|
12
|
+
* Try to read an identifier token
|
13
|
+
*/
|
14
|
+
tryRead(previous) {
|
15
|
+
if (this.isEndOfInput()) {
|
16
|
+
return null;
|
17
|
+
}
|
18
|
+
const char = this.input[this.position];
|
19
|
+
// wildcard identifier
|
20
|
+
if (char === '*') {
|
21
|
+
// Assume that the OperatorTokenReader is executed before the IdentifierTokenReader.
|
22
|
+
// Since we have determined that the OperatorTokenReader is not an Operator,
|
23
|
+
// we treat '*' here as a wildcard identifier.
|
24
|
+
this.position++;
|
25
|
+
return this.createLexeme(Lexeme_1.TokenType.Identifier, char);
|
26
|
+
}
|
27
|
+
// MySQL escaped identifier (escape character is backtick)
|
28
|
+
if (char === '`') {
|
29
|
+
const identifier = this.readEscapedIdentifier('`');
|
30
|
+
return this.createLexeme(Lexeme_1.TokenType.Identifier, identifier);
|
31
|
+
}
|
32
|
+
// Postgres escaped identifier (escape character is double quote)
|
33
|
+
if (char === '"') {
|
34
|
+
const identifier = this.readEscapedIdentifier('"');
|
35
|
+
return this.createLexeme(Lexeme_1.TokenType.Identifier, identifier);
|
36
|
+
}
|
37
|
+
// SQLServer escaped identifier (escape character is square bracket)
|
38
|
+
if (char === '[' && (previous === null || previous.value !== "array")) {
|
39
|
+
const identifier = this.readEscapedIdentifier(']');
|
40
|
+
return this.createLexeme(Lexeme_1.TokenType.Identifier, identifier);
|
41
|
+
}
|
42
|
+
// Regular identifier
|
43
|
+
const result = stringUtils_1.StringUtils.readRegularIdentifier(this.input, this.position);
|
44
|
+
this.position = result.newPosition;
|
45
|
+
return this.createLexeme(Lexeme_1.TokenType.Identifier, result.identifier);
|
46
|
+
}
|
47
|
+
/**
|
48
|
+
* Read an escaped identifier (surrounded by delimiters)
|
49
|
+
*/
|
50
|
+
readEscapedIdentifier(delimiter) {
|
51
|
+
const start = this.position;
|
52
|
+
// Skip the opening delimiter
|
53
|
+
this.position++;
|
54
|
+
while (this.canRead()) {
|
55
|
+
if (this.input[this.position] === delimiter) {
|
56
|
+
break;
|
57
|
+
}
|
58
|
+
this.position++;
|
59
|
+
}
|
60
|
+
if (start === this.position) {
|
61
|
+
throw new Error(`Closing delimiter is not found. position: ${start}, delimiter: ${delimiter}\n${this.getDebugPositionInfo(start)}}`);
|
62
|
+
}
|
63
|
+
// Skip the closing delimiter
|
64
|
+
this.position++;
|
65
|
+
// exclude the delimiter
|
66
|
+
return this.input.slice(start + 1, this.position - 1);
|
67
|
+
}
|
68
|
+
}
|
69
|
+
exports.IdentifierTokenReader = IdentifierTokenReader;
|
70
|
+
//# sourceMappingURL=IdentifierTokenReader.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"IdentifierTokenReader.js","sourceRoot":"","sources":["../../src/tokenReaders/IdentifierTokenReader.ts"],"names":[],"mappings":";;;AAAA,uDAAoD;AACpD,6CAAqD;AACrD,sDAAmD;AAEnD;;GAEG;AACH,MAAa,qBAAsB,SAAQ,iCAAe;IACtD;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,sBAAsB;QACtB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,oFAAoF;YACpF,4EAA4E;YAC5E,8CAA8C;YAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACzD,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAED,oEAAoE;QACpE,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,CAAC,EAAE,CAAC;YACpE,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnD,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/D,CAAC;QAED,qBAAqB;QACrB,MAAM,MAAM,GAAG,yBAAW,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5E,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;QACnC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,SAAiB;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE5B,6BAA6B;QAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC1C,MAAM;YACV,CAAC;YACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,6CAA6C,KAAK,gBAAgB,SAAS,KAAK,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzI,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,wBAAwB;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;CACJ;AAtED,sDAsEC"}
|
@@ -0,0 +1,189 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.LiteralTokenReader = exports.literalKeywordParser = void 0;
|
4
|
+
const BaseTokenReader_1 = require("./BaseTokenReader");
|
5
|
+
const Lexeme_1 = require("../models/Lexeme");
|
6
|
+
const charLookupTable_1 = require("../utils/charLookupTable");
|
7
|
+
const KeywordParser_1 = require("../parsers/KeywordParser");
|
8
|
+
const KeywordTrie_1 = require("../models/KeywordTrie");
|
9
|
+
/**
|
10
|
+
* Reads SQL literal tokens (numbers, strings)
|
11
|
+
*/
|
12
|
+
const keywords = [
|
13
|
+
["null"],
|
14
|
+
["true"],
|
15
|
+
["false"],
|
16
|
+
["current_date"],
|
17
|
+
["current_time"],
|
18
|
+
["current_timestamp"],
|
19
|
+
["localtime"],
|
20
|
+
["localtimestamp"],
|
21
|
+
["unbounded"],
|
22
|
+
["normalized"],
|
23
|
+
["nfc", "normalized"],
|
24
|
+
["nfd", "normalized"],
|
25
|
+
["nfkc", "normalized"],
|
26
|
+
["nfkd", "normalized"],
|
27
|
+
["nfc"],
|
28
|
+
["nfd"],
|
29
|
+
["nfkc"],
|
30
|
+
["nfkd"],
|
31
|
+
];
|
32
|
+
const trie = new KeywordTrie_1.KeywordTrie(keywords);
|
33
|
+
exports.literalKeywordParser = new KeywordParser_1.KeywordParser(trie);
|
34
|
+
class LiteralTokenReader extends BaseTokenReader_1.BaseTokenReader {
|
35
|
+
/**
|
36
|
+
* Try to read a literal token
|
37
|
+
*/
|
38
|
+
tryRead(previous) {
|
39
|
+
if (this.isEndOfInput()) {
|
40
|
+
return null;
|
41
|
+
}
|
42
|
+
const char = this.input[this.position];
|
43
|
+
// Check for keyword literals
|
44
|
+
const keyword = this.tryReadKeyword();
|
45
|
+
if (keyword) {
|
46
|
+
return keyword;
|
47
|
+
}
|
48
|
+
// Decimal token starting with a dot
|
49
|
+
if (char === '.' && this.canRead(1) && charLookupTable_1.CharLookupTable.isDigit(this.input[this.position + 1])) {
|
50
|
+
return this.createLexeme(Lexeme_1.TokenType.Literal, this.readDigit());
|
51
|
+
}
|
52
|
+
// String literal
|
53
|
+
if (char === '\'') {
|
54
|
+
const value = this.readSingleQuotedString(false);
|
55
|
+
return this.createLexeme(Lexeme_1.TokenType.Literal, value);
|
56
|
+
}
|
57
|
+
// Digit tokens
|
58
|
+
if (charLookupTable_1.CharLookupTable.isDigit(char)) {
|
59
|
+
return this.createLexeme(Lexeme_1.TokenType.Literal, this.readDigit());
|
60
|
+
}
|
61
|
+
// Signed number
|
62
|
+
if ((char === '+' || char === '-') && this.isValidNumericPrefix(previous)) {
|
63
|
+
const sign = char;
|
64
|
+
this.position++;
|
65
|
+
// Skip whitespace after sign
|
66
|
+
const pos = this.position;
|
67
|
+
while (this.canRead() && charLookupTable_1.CharLookupTable.isWhitespace(this.input[this.position])) {
|
68
|
+
this.position++;
|
69
|
+
}
|
70
|
+
if (this.canRead() && (charLookupTable_1.CharLookupTable.isDigit(this.input[this.position]) ||
|
71
|
+
(this.input[this.position] === '.' &&
|
72
|
+
this.canRead(1) &&
|
73
|
+
charLookupTable_1.CharLookupTable.isDigit(this.input[this.position + 1])))) {
|
74
|
+
return this.createLexeme(Lexeme_1.TokenType.Literal, sign === '-' ? sign + this.readDigit() : this.readDigit());
|
75
|
+
}
|
76
|
+
// Not a number, restore position
|
77
|
+
this.position = pos - 1; // Adjust for the increment at the beginning
|
78
|
+
}
|
79
|
+
return null;
|
80
|
+
}
|
81
|
+
tryReadKeyword() {
|
82
|
+
// Check for keyword literals
|
83
|
+
const result = exports.literalKeywordParser.parse(this.input, this.position);
|
84
|
+
if (result) {
|
85
|
+
this.position = result.newPosition;
|
86
|
+
return this.createLexeme(Lexeme_1.TokenType.Literal, result.keyword);
|
87
|
+
}
|
88
|
+
return null;
|
89
|
+
}
|
90
|
+
/**
|
91
|
+
* Check if the current context allows for a signed number
|
92
|
+
*/
|
93
|
+
isValidNumericPrefix(previous) {
|
94
|
+
return previous === null ||
|
95
|
+
(previous.type !== Lexeme_1.TokenType.Literal &&
|
96
|
+
previous.type !== Lexeme_1.TokenType.Identifier);
|
97
|
+
}
|
98
|
+
/**
|
99
|
+
* Read a numeric value
|
100
|
+
*/
|
101
|
+
readDigit() {
|
102
|
+
const start = this.position;
|
103
|
+
let hasDot = false;
|
104
|
+
let hasExponent = false;
|
105
|
+
// Consider 0x, 0b, 0o
|
106
|
+
if (this.canRead(1) &&
|
107
|
+
this.input[this.position] === '0' &&
|
108
|
+
"xbo".includes(this.input[this.position + 1].toLowerCase())) {
|
109
|
+
const prefixType = this.input[this.position + 1].toLowerCase();
|
110
|
+
this.position += 2;
|
111
|
+
// Continue to get numeric and hexadecimal notation strings
|
112
|
+
const isHex = prefixType === 'x';
|
113
|
+
while (this.canRead()) {
|
114
|
+
const c = this.input[this.position];
|
115
|
+
if (charLookupTable_1.CharLookupTable.isDigit(c) || (isHex && charLookupTable_1.CharLookupTable.isHexChar(c))) {
|
116
|
+
this.position++;
|
117
|
+
}
|
118
|
+
else {
|
119
|
+
break;
|
120
|
+
}
|
121
|
+
}
|
122
|
+
return this.input.slice(start, this.position);
|
123
|
+
}
|
124
|
+
// If starting with dot, note it
|
125
|
+
if (this.input[start] === '.') {
|
126
|
+
hasDot = true;
|
127
|
+
this.position++;
|
128
|
+
}
|
129
|
+
// Consider decimal point and exponential notation
|
130
|
+
while (this.canRead()) {
|
131
|
+
const char = this.input[this.position];
|
132
|
+
if (char === '.' && !hasDot) {
|
133
|
+
hasDot = true;
|
134
|
+
}
|
135
|
+
else if ((char === 'e' || char === 'E') && !hasExponent) {
|
136
|
+
hasExponent = true;
|
137
|
+
if (this.canRead(1) && (this.input[this.position + 1] === '+' || this.input[this.position + 1] === '-')) {
|
138
|
+
this.position++;
|
139
|
+
}
|
140
|
+
}
|
141
|
+
else if (!charLookupTable_1.CharLookupTable.isDigit(char)) {
|
142
|
+
break;
|
143
|
+
}
|
144
|
+
this.position++;
|
145
|
+
}
|
146
|
+
if (start === this.position) {
|
147
|
+
throw new Error(`Unexpected character. position: ${start}\n${this.getDebugPositionInfo(start)}`);
|
148
|
+
}
|
149
|
+
if (this.input[start] === '.') {
|
150
|
+
// If the number starts with a dot, add 0 to the front
|
151
|
+
return '0' + this.input.slice(start, this.position);
|
152
|
+
}
|
153
|
+
return this.input.slice(start, this.position);
|
154
|
+
}
|
155
|
+
/**
|
156
|
+
* Read a string literal
|
157
|
+
*/
|
158
|
+
readSingleQuotedString(includeSingleQuote) {
|
159
|
+
const start = this.position;
|
160
|
+
let closed = false;
|
161
|
+
this.read("'");
|
162
|
+
while (this.canRead()) {
|
163
|
+
const char = this.input[this.position];
|
164
|
+
this.position++;
|
165
|
+
// escape character check
|
166
|
+
if (char === "\\" && this.canRead(1)) {
|
167
|
+
this.position++;
|
168
|
+
continue;
|
169
|
+
}
|
170
|
+
else if (char === '\'') {
|
171
|
+
closed = true;
|
172
|
+
break;
|
173
|
+
}
|
174
|
+
}
|
175
|
+
if (closed === false) {
|
176
|
+
throw new Error(`Single quote is not closed. position: ${start}\n${this.getDebugPositionInfo(start)}`);
|
177
|
+
}
|
178
|
+
if (includeSingleQuote) {
|
179
|
+
const value = this.input.slice(start, this.position);
|
180
|
+
return value;
|
181
|
+
}
|
182
|
+
else {
|
183
|
+
const value = this.input.slice(start + 1, this.position - 1);
|
184
|
+
return value;
|
185
|
+
}
|
186
|
+
}
|
187
|
+
}
|
188
|
+
exports.LiteralTokenReader = LiteralTokenReader;
|
189
|
+
//# sourceMappingURL=LiteralTokenReader.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"LiteralTokenReader.js","sourceRoot":"","sources":["../../src/tokenReaders/LiteralTokenReader.ts"],"names":[],"mappings":";;;AAAA,uDAAoD;AACpD,6CAAqD;AACrD,8DAA2D;AAC3D,4DAAyD;AACzD,uDAAoD;AAEpD;;GAEG;AAEH,MAAM,QAAQ,GAAG;IACb,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;IACR,CAAC,OAAO,CAAC;IACT,CAAC,cAAc,CAAC;IAChB,CAAC,cAAc,CAAC;IAChB,CAAC,mBAAmB,CAAC;IACrB,CAAC,WAAW,CAAC;IACb,CAAC,gBAAgB,CAAC;IAClB,CAAC,WAAW,CAAC;IACb,CAAC,YAAY,CAAC;IACd,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,MAAM,EAAE,YAAY,CAAC;IACtB,CAAC,MAAM,EAAE,YAAY,CAAC;IACtB,CAAC,KAAK,CAAC;IACP,CAAC,KAAK,CAAC;IACP,CAAC,MAAM,CAAC;IACR,CAAC,MAAM,CAAC;CACX,CAAC;AACF,MAAM,IAAI,GAAG,IAAI,yBAAW,CAAC,QAAQ,CAAC,CAAC;AAC1B,QAAA,oBAAoB,GAAG,IAAI,6BAAa,CAAC,IAAI,CAAC,CAAC;AAE5D,MAAa,kBAAmB,SAAQ,iCAAe;IACnD;;OAEG;IACI,OAAO,CAAC,QAAuB;QAClC,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,iCAAiC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACV,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,iCAAe,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5F,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,iBAAiB;QACjB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;YACjD,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;QAED,eAAe;QACf,IAAI,iCAAe,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,gBAAgB;QAChB,IAAI,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxE,MAAM,IAAI,GAAG,IAAI,CAAC;YAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,6BAA6B;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC1B,OAAO,IAAI,CAAC,OAAO,EAAE,IAAI,iCAAe,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/E,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpB,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,CAClB,iCAAe,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAClD,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG;oBAC9B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACf,iCAAe,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAC9D,EAAE,CAAC;gBACA,OAAO,IAAI,CAAC,YAAY,CACpB,kBAAS,CAAC,OAAO,EACjB,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAC5D,CAAC;YACN,CAAC;YAED,iCAAiC;YACjC,IAAI,CAAC,QAAQ,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,4CAA4C;QACzE,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,cAAc;QAClB,6BAA6B;QAC7B,MAAM,MAAM,GAAG,4BAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrE,IAAI,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC;YACnC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAS,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,QAAuB;QAChD,OAAO,QAAQ,KAAK,IAAI;YACpB,CAAC,QAAQ,CAAC,IAAI,KAAK,kBAAS,CAAC,OAAO;gBAChC,QAAQ,CAAC,IAAI,KAAK,kBAAS,CAAC,UAAU,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACK,SAAS;QACb,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,WAAW,GAAG,KAAK,CAAC;QAExB,sBAAsB;QACtB,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG;YACjC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAE9D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/D,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YAEnB,2DAA2D;YAC3D,MAAM,KAAK,GAAG,UAAU,KAAK,GAAG,CAAC;YACjC,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;gBACpB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACpC,IAAI,iCAAe,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,iCAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACpB,CAAC;qBAAM,CAAC;oBACJ,MAAM;gBACV,CAAC;YACL,CAAC;YAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC;QAED,gCAAgC;QAChC,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,GAAG,IAAI,CAAC;YACd,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;QAED,kDAAkD;QAClD,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEvC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,IAAI,CAAC;YAClB,CAAC;iBAAM,IAAI,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxD,WAAW,GAAG,IAAI,CAAC;gBACnB,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBACtG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACpB,CAAC;YACL,CAAC;iBAAM,IAAI,CAAC,iCAAe,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,MAAM;YACV,CAAC;YAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,KAAK,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC;YAC5B,sDAAsD;YACtD,OAAO,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,kBAA2B;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEhB,yBAAyB;YACzB,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAChB,SAAS;YACb,CAAC;iBACI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,GAAG,IAAI,CAAC;gBACd,MAAM;YACV,CAAC;QACL,CAAC;QAED,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,KAAK,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3G,CAAC;QAED,IAAI,kBAAkB,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrD,OAAO,KAAK,CAAC;QACjB,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;CACJ;AAvLD,gDAuLC"}
|