tla-ts 0.0.3-dev.e5b05b9 → 0.0.3-dev.e8bef80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +1 -1
- package/dist/cli.js +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +150 -48
- package/dist/index.d.ts +150 -48
- package/dist/index.js +1 -1
- package/dist/parallel-worker.cjs +1 -1
- package/dist/parallel-worker.js +2 -2
- package/dist/parallel.cjs +1 -1
- package/dist/parallel.js +2 -2
- package/dist/run-cli.cjs +1351 -22
- package/dist/run-cli.js +1352 -23
- package/dist/transfer.cjs +83 -43
- package/dist/transfer.js +83 -43
- package/package.json +5 -1
- package/src/ast.ts +28 -4
- package/src/bags.ts +4 -7
- package/src/builtins.ts +29 -17
- package/src/bundled-modules.ts +3 -3
- package/src/canonical.ts +4 -3
- package/src/check.ts +1317 -428
- package/src/cli.ts +8 -1
- package/src/codegen.ts +68 -21
- package/src/config.ts +226 -207
- package/src/environment.ts +221 -75
- package/src/errors.ts +130 -15
- package/src/evaluate.ts +713 -259
- package/src/exit-codes.ts +6 -24
- package/src/fingerprints.ts +1 -2
- package/src/index.ts +1 -0
- package/src/instances.ts +3 -1
- package/src/java-overrides.ts +104 -25
- package/src/levels.ts +6 -1
- package/src/lexer.ts +122 -29
- package/src/live-check.ts +204 -148
- package/src/liveness.ts +191 -122
- package/src/obligation-expression.ts +2 -2
- package/src/obligation-source.ts +2 -0
- package/src/parallel-worker.ts +6 -1
- package/src/parallel.ts +57 -6
- package/src/parser.ts +327 -147
- package/src/pcal-ast.ts +770 -0
- package/src/pcal-parse.ts +1523 -0
- package/src/pcal-tokenize.ts +516 -0
- package/src/pcal-translate.ts +1230 -0
- package/src/pcal-validate.ts +454 -0
- package/src/randomization.ts +24 -7
- package/src/resolve.ts +252 -87
- package/src/run-cli.ts +1162 -200
- package/src/runtime.ts +16 -0
- package/src/sany-builtins.ts +147 -0
- package/src/sany-generator.ts +3128 -0
- package/src/sany-levels.ts +1208 -0
- package/src/sany-lexer.ts +410 -480
- package/src/sany-linter.ts +122 -0
- package/src/sany-modules.ts +1398 -0
- package/src/sany-nodes.ts +463 -0
- package/src/sany-parse-error.ts +2033 -3015
- package/src/sany-report.ts +356 -0
- package/src/sany-selectors.ts +738 -0
- package/src/sany-source.ts +248 -0
- package/src/sany-xml.ts +1017 -0
- package/src/spec-description.ts +25 -5
- package/src/standard-modules.ts +172 -129
- package/src/state-graph.ts +29 -6
- package/src/subexpressions.ts +121 -26
- package/src/successors.ts +141 -173
- package/src/symmetry.ts +168 -50
- package/src/tableau.ts +3 -1
- package/src/tlc-ext.ts +13 -17
- package/src/tlc-messages.ts +329 -0
- package/src/tlc-module.ts +49 -21
- package/src/tlc-modules.ts +0 -4
- package/src/tlc-order.ts +55 -9
- package/src/tlc-random.ts +28 -3
- package/src/tlc-replay.ts +655 -0
- package/src/tlc-version.ts +18 -0
- package/src/transfer.ts +15 -3
- package/src/values.ts +664 -317
package/src/sany-lexer.ts
CHANGED
|
@@ -1,222 +1,123 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TlaError } from "./errors";
|
|
2
2
|
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
export const BEGIN_PRAGMA = 2;
|
|
6
|
-
export const _BM1 = 3;
|
|
7
|
-
export const NUMBER = 20;
|
|
8
|
-
export const _BM2 = 21;
|
|
9
|
-
export const _BM0 = 35;
|
|
10
|
-
export const SEPARATOR = 36;
|
|
11
|
-
export const END_MODULE = 37;
|
|
12
|
-
export const ACTION = 38;
|
|
13
|
-
export const ASSUME = 39;
|
|
14
|
-
export const BOXASSUME = 40;
|
|
15
|
-
export const ASSUMPTION = 41;
|
|
16
|
-
export const CASE = 42;
|
|
17
|
-
export const CHOOSE = 43;
|
|
18
|
-
export const CONSTANT = 44;
|
|
19
|
-
export const ELSE = 45;
|
|
20
|
-
export const EXCEPT = 46;
|
|
21
|
-
export const EXISTS = 47;
|
|
22
|
-
export const EXTENDS = 48;
|
|
23
|
-
export const FORALL = 49;
|
|
24
|
-
export const IF = 50;
|
|
25
|
-
export const INSTANCE = 51;
|
|
26
|
-
export const LET = 52;
|
|
27
|
-
export const LETIN = 53;
|
|
28
|
-
export const LOCAL = 54;
|
|
29
|
-
export const MODULE = 55;
|
|
30
|
-
export const NEW = 56;
|
|
31
|
-
export const OTHER = 57;
|
|
32
|
-
export const PROPOSITION = 58;
|
|
33
|
-
export const SF = 59;
|
|
34
|
-
export const T_EXISTS = 60;
|
|
35
|
-
export const T_FORALL = 61;
|
|
36
|
-
export const THEN = 62;
|
|
37
|
-
export const BY = 63;
|
|
38
|
-
export const ONLY = 64;
|
|
39
|
-
export const DEFINE = 65;
|
|
40
|
-
export const DF = 66;
|
|
41
|
-
export const THEOREM = 67;
|
|
42
|
-
export const USE = 68;
|
|
43
|
-
export const HIDE = 69;
|
|
44
|
-
export const HAVE = 70;
|
|
45
|
-
export const OBVIOUS = 71;
|
|
46
|
-
export const OMITTED = 72;
|
|
47
|
-
export const LAMBDA = 73;
|
|
48
|
-
export const TAKE = 74;
|
|
49
|
-
export const PROOF = 75;
|
|
50
|
-
export const PROVE = 76;
|
|
51
|
-
export const BOXPROVE = 77;
|
|
52
|
-
export const QED = 78;
|
|
53
|
-
export const RECURSIVE = 79;
|
|
54
|
-
export const STATE = 80;
|
|
55
|
-
export const TEMPORAL = 81;
|
|
56
|
-
export const PICK = 82;
|
|
57
|
-
export const WITNESS = 83;
|
|
58
|
-
export const SUFFICES = 84;
|
|
59
|
-
export const VARIABLE = 85;
|
|
60
|
-
export const WF = 86;
|
|
61
|
-
export const WITH = 87;
|
|
62
|
-
export const COMMA = 88;
|
|
63
|
-
export const COLON = 89;
|
|
64
|
-
export const COLONCOLON = 90;
|
|
65
|
-
export const DOT = 91;
|
|
66
|
-
export const US = 92;
|
|
67
|
-
export const DEF = 93;
|
|
68
|
-
export const LBR = 94;
|
|
69
|
-
export const RBR = 95;
|
|
70
|
-
export const DEFBREAK = 96;
|
|
71
|
-
export const LSB = 97;
|
|
72
|
-
export const ARSB = 98;
|
|
73
|
-
export const RSB = 99;
|
|
74
|
-
export const LBC = 100;
|
|
75
|
-
export const RBC = 101;
|
|
76
|
-
export const LAB = 102;
|
|
77
|
-
export const ARAB = 103;
|
|
78
|
-
export const RAB = 104;
|
|
79
|
-
export const BANG = 105;
|
|
80
|
-
export const ARROW = 106;
|
|
81
|
-
export const SUBSTITUTE = 107;
|
|
82
|
-
export const MAPTO = 108;
|
|
83
|
-
export const NUMBER_LITERAL = 109;
|
|
84
|
-
export const STRING_LITERAL = 110;
|
|
85
|
-
const BAND = 111;
|
|
86
|
-
const BOR = 112;
|
|
87
|
-
export const PRIME = 113;
|
|
88
|
-
export const NEGATIVE = 117;
|
|
89
|
-
export const CASESEP = 121;
|
|
90
|
-
export const AND = 129;
|
|
91
|
-
export const OR = 132;
|
|
92
|
-
export const IN = 147;
|
|
93
|
-
export const EQUALS = 185;
|
|
94
|
-
export const MINUS = 193;
|
|
95
|
-
export const IDENTIFIER = 289;
|
|
96
|
-
export const PROOF_STEP = 290;
|
|
97
|
-
export const PROOF_IMPLICIT_STEP = 291;
|
|
98
|
-
export const PROOF_STEP_DOT = 292;
|
|
99
|
-
export const BARE_LEVEL = 293;
|
|
100
|
-
export const UNNUMBERED_STEP = 294;
|
|
3
|
+
// An emulation of the token manager that JavaCC generates for SANY's grammar, javacc/tla+.jj: the
|
|
4
|
+
// tokens it produces, their kinds in TLAplusParserConstants order, and their positions.
|
|
101
5
|
|
|
102
6
|
export interface SanyToken {
|
|
103
7
|
kind: number;
|
|
104
8
|
image: string;
|
|
105
9
|
beginLine: number;
|
|
106
10
|
beginColumn: number;
|
|
107
|
-
endLine: number;
|
|
108
|
-
endColumn: number;
|
|
109
11
|
next: SanyToken | undefined;
|
|
110
12
|
}
|
|
111
13
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
[
|
|
129
|
-
[
|
|
130
|
-
[
|
|
131
|
-
[
|
|
132
|
-
[
|
|
133
|
-
[
|
|
134
|
-
[
|
|
135
|
-
[
|
|
136
|
-
[
|
|
137
|
-
[
|
|
138
|
-
[
|
|
139
|
-
[
|
|
140
|
-
[
|
|
141
|
-
[
|
|
142
|
-
[
|
|
143
|
-
[
|
|
144
|
-
[
|
|
145
|
-
[
|
|
146
|
-
[
|
|
147
|
-
[
|
|
148
|
-
[
|
|
149
|
-
[
|
|
150
|
-
[
|
|
151
|
-
[
|
|
152
|
-
[
|
|
153
|
-
[
|
|
154
|
-
[
|
|
155
|
-
[
|
|
156
|
-
[
|
|
157
|
-
[
|
|
158
|
-
[
|
|
159
|
-
[
|
|
160
|
-
[
|
|
161
|
-
[
|
|
162
|
-
[
|
|
163
|
-
[
|
|
164
|
-
[
|
|
165
|
-
[
|
|
166
|
-
[
|
|
167
|
-
[
|
|
168
|
-
[
|
|
169
|
-
[
|
|
170
|
-
[
|
|
171
|
-
[
|
|
172
|
-
[
|
|
173
|
-
[
|
|
174
|
-
[
|
|
175
|
-
[
|
|
176
|
-
[
|
|
177
|
-
[
|
|
178
|
-
[
|
|
179
|
-
[
|
|
180
|
-
[
|
|
181
|
-
[
|
|
182
|
-
[
|
|
183
|
-
[
|
|
184
|
-
[
|
|
185
|
-
[
|
|
186
|
-
[
|
|
187
|
-
[
|
|
188
|
-
[
|
|
189
|
-
[
|
|
190
|
-
[
|
|
191
|
-
[
|
|
192
|
-
[
|
|
193
|
-
[
|
|
194
|
-
[
|
|
195
|
-
[
|
|
196
|
-
[
|
|
197
|
-
[
|
|
198
|
-
[
|
|
199
|
-
[
|
|
200
|
-
[
|
|
201
|
-
[
|
|
202
|
-
[
|
|
203
|
-
[
|
|
204
|
-
[
|
|
205
|
-
[
|
|
206
|
-
[
|
|
207
|
-
[
|
|
208
|
-
[
|
|
209
|
-
[
|
|
210
|
-
[
|
|
211
|
-
[
|
|
212
|
-
[
|
|
213
|
-
[
|
|
14
|
+
// The token kinds, in the order of TLAplusParserConstants, each with the literal images that
|
|
15
|
+
// produce it. JavaCC breaks a tie between matches of the same length by this order.
|
|
16
|
+
const KINDS: [string, ...string[]][] = [
|
|
17
|
+
["EOF"],
|
|
18
|
+
["BEGIN_PRAGMA", "--->"],
|
|
19
|
+
["_BM1"],
|
|
20
|
+
["NUMBER"],
|
|
21
|
+
["_BM2"],
|
|
22
|
+
["_BM0"],
|
|
23
|
+
["SEPARATOR"],
|
|
24
|
+
["END_MODULE"],
|
|
25
|
+
["ACTION", "ACTION", "ACTIONS"],
|
|
26
|
+
["ASSUME", "ASSUME"],
|
|
27
|
+
["BOXASSUME", "[]ASSUME"],
|
|
28
|
+
["ASSUMPTION", "ASSUMPTION", "AXIOM"],
|
|
29
|
+
["CASE", "CASE"],
|
|
30
|
+
["CHOOSE", "CHOOSE"],
|
|
31
|
+
["CONSTANT", "CONSTANT", "CONSTANTS"],
|
|
32
|
+
["ELSE", "ELSE"],
|
|
33
|
+
["EXCEPT", "EXCEPT"],
|
|
34
|
+
["EXISTS", "\\E", "\\exists", "∃"],
|
|
35
|
+
["EXTENDS", "EXTENDS"],
|
|
36
|
+
["FORALL", "\\A", "\\forall", "∀"],
|
|
37
|
+
["IF", "IF"],
|
|
38
|
+
["INSTANCE", "INSTANCE"],
|
|
39
|
+
["LET", "LET"],
|
|
40
|
+
["LETIN", "IN"],
|
|
41
|
+
["LOCAL", "LOCAL"],
|
|
42
|
+
["MODULE", "MODULE"],
|
|
43
|
+
["NEW", "NEW"],
|
|
44
|
+
["OTHER", "OTHER"],
|
|
45
|
+
["PROPOSITION", "PROPOSITION", "LEMMA", "COROLLARY"],
|
|
46
|
+
["SF", "SF_"],
|
|
47
|
+
["T_EXISTS", "\\EE"],
|
|
48
|
+
["T_FORALL", "\\AA"],
|
|
49
|
+
["THEN", "THEN"],
|
|
50
|
+
["BY", "BY"],
|
|
51
|
+
["ONLY", "ONLY"],
|
|
52
|
+
["DEFINE", "DEFINE"],
|
|
53
|
+
["DF", "DEF", "DEFS"],
|
|
54
|
+
["THEOREM", "THEOREM"],
|
|
55
|
+
["USE", "USE"],
|
|
56
|
+
["HIDE", "HIDE"],
|
|
57
|
+
["HAVE", "HAVE"],
|
|
58
|
+
["OBVIOUS", "OBVIOUS"],
|
|
59
|
+
["OMITTED", "OMITTED"],
|
|
60
|
+
["LAMBDA", "LAMBDA"],
|
|
61
|
+
["TAKE", "TAKE"],
|
|
62
|
+
["PROOF", "PROOF"],
|
|
63
|
+
["PROVE", "PROVE"],
|
|
64
|
+
["BOXPROVE", "[]PROVE"],
|
|
65
|
+
["QED", "QED"],
|
|
66
|
+
["RECURSIVE", "RECURSIVE"],
|
|
67
|
+
["STATE", "STATE"],
|
|
68
|
+
["TEMPORAL", "TEMPORAL", "TEMPORALS"],
|
|
69
|
+
["PICK", "PICK"],
|
|
70
|
+
["WITNESS", "WITNESS"],
|
|
71
|
+
["SUFFICES", "SUFFICES"],
|
|
72
|
+
["VARIABLE", "VARIABLE", "VARIABLES"],
|
|
73
|
+
["WF", "WF_"],
|
|
74
|
+
["WITH", "WITH"],
|
|
75
|
+
["COMMA", ","],
|
|
76
|
+
["COLON", ":"],
|
|
77
|
+
["COLONCOLON", "::", "∷"],
|
|
78
|
+
["DOT", "."],
|
|
79
|
+
["US", "_"],
|
|
80
|
+
["DEF", "==", "≜"],
|
|
81
|
+
["LBR", "("],
|
|
82
|
+
["RBR", ")"],
|
|
83
|
+
["DEFBREAK", "-|-"],
|
|
84
|
+
["LSB", "["],
|
|
85
|
+
["ARSB", "]_"],
|
|
86
|
+
["RSB", "]"],
|
|
87
|
+
["LBC", "{"],
|
|
88
|
+
["RBC", "}"],
|
|
89
|
+
["LAB", "<<", "⟨"],
|
|
90
|
+
["ARAB", ">>_", "⟩_"],
|
|
91
|
+
["RAB", ">>", "⟩"],
|
|
92
|
+
["BANG", "!"],
|
|
93
|
+
["ARROW", "->", "→"],
|
|
94
|
+
["SUBSTITUTE", "<-", "←"],
|
|
95
|
+
["MAPTO", "|->", "↦"],
|
|
96
|
+
["NUMBER_LITERAL"],
|
|
97
|
+
["STRING_LITERAL"],
|
|
98
|
+
["BAND"],
|
|
99
|
+
["BOR"],
|
|
100
|
+
["op_57", "'"],
|
|
101
|
+
["op_68", "^+", "⁺"],
|
|
102
|
+
["op_69", "^*"],
|
|
103
|
+
["op_70", "^#"],
|
|
104
|
+
["op_76", "-."],
|
|
105
|
+
["op_26", "\\lnot", "¬"],
|
|
106
|
+
["op_29", "\\neg"],
|
|
107
|
+
["op_58", "~"],
|
|
108
|
+
["CASESEP", "[]", "□"],
|
|
109
|
+
["op_61", "<>", "◇"],
|
|
110
|
+
["op_112", "ENABLED"],
|
|
111
|
+
["op_113", "UNCHANGED"],
|
|
112
|
+
["op_114", "SUBSET"],
|
|
113
|
+
["op_115", "UNION"],
|
|
114
|
+
["op_116", "DOMAIN"],
|
|
115
|
+
["op_1", "//"],
|
|
116
|
+
["AND", "/\\", "∧"],
|
|
117
|
+
["op_3", "/="],
|
|
118
|
+
["op_4", "/"],
|
|
119
|
+
["OR", "\\/", "∨"],
|
|
214
120
|
...[
|
|
215
|
-
"//",
|
|
216
|
-
"/\\",
|
|
217
|
-
"/=",
|
|
218
|
-
"/",
|
|
219
|
-
"\\/",
|
|
220
121
|
"\\approx",
|
|
221
122
|
"\\asymp",
|
|
222
123
|
"\\bigcirc",
|
|
@@ -231,7 +132,9 @@ const LITERALS: [number, string[]][] = [
|
|
|
231
132
|
"\\equiv",
|
|
232
133
|
"\\geq",
|
|
233
134
|
"\\gg",
|
|
234
|
-
|
|
135
|
+
].map((image): [string, string] => ["infix", image]),
|
|
136
|
+
["IN", "\\in", "∈"],
|
|
137
|
+
...[
|
|
235
138
|
"\\intersect",
|
|
236
139
|
"\\union",
|
|
237
140
|
"\\land",
|
|
@@ -269,15 +172,11 @@ const LITERALS: [number, string[]][] = [
|
|
|
269
172
|
"=>",
|
|
270
173
|
"=<",
|
|
271
174
|
"=|",
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
"--",
|
|
278
|
-
"-|",
|
|
279
|
-
"-+->",
|
|
280
|
-
"-",
|
|
175
|
+
].map((image): [string, string] => ["infix", image]),
|
|
176
|
+
["EQUALS", "="],
|
|
177
|
+
...["##", "#", "^^", "^", "--", "-|", "-+->"].map((image): [string, string] => ["infix", image]),
|
|
178
|
+
["op_77", "-"],
|
|
179
|
+
...[
|
|
281
180
|
"**",
|
|
282
181
|
"*",
|
|
283
182
|
"++",
|
|
@@ -312,304 +211,335 @@ const LITERALS: [number, string[]][] = [
|
|
|
312
211
|
"(/)",
|
|
313
212
|
"(\\X)",
|
|
314
213
|
"\\notin",
|
|
315
|
-
...
|
|
214
|
+
..."≈≔≍◯⩴●∩⋅∘≅∪÷≐‥…≡‼≥≫⇔⇒⫤↝≤≪⊣≠∉⊙⊖⊕⊘⊗⇸≺⪯∝⁇⊨⊢∼≃⊓⊔⊏⊑⊐⊒⋆⊂⊆≻⪰⊃⊇×⊎‖≀",
|
|
316
215
|
"\\times",
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
[
|
|
320
|
-
[
|
|
321
|
-
[
|
|
216
|
+
].map((image): [string, string] => ["infix", image]),
|
|
217
|
+
["op_119", "\\X"],
|
|
218
|
+
["IDENTIFIER"],
|
|
219
|
+
["ProofStepLexeme"],
|
|
220
|
+
["ProofImplicitStepLexeme"],
|
|
221
|
+
["ProofStepDotLexeme"],
|
|
222
|
+
["BareLevelLexeme"],
|
|
223
|
+
["UnnumberedStepLexeme"],
|
|
322
224
|
];
|
|
323
225
|
|
|
324
|
-
const
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
literals.set(image[0], [...(literals.get(image[0]) ?? []), [entry[0], image]]);
|
|
329
|
-
}
|
|
330
|
-
return literals;
|
|
331
|
-
};
|
|
226
|
+
const KIND_NAMES = KINDS.map((entry) => entry[0]);
|
|
227
|
+
const KIND_INDEXES = new Map(
|
|
228
|
+
KIND_NAMES.map((name, kind): [string, number] => [name, kind]).reverse(),
|
|
229
|
+
);
|
|
332
230
|
|
|
333
|
-
|
|
231
|
+
// The kind named name. The anonymous infix operators share the name "infix", which no caller
|
|
232
|
+
// looks up.
|
|
233
|
+
export const getKind = (name: string) => {
|
|
234
|
+
const kind = KIND_INDEXES.get(name);
|
|
235
|
+
if (kind === undefined) throw new TlaError(`Unknown SANY token kind ${name}`);
|
|
236
|
+
return kind;
|
|
237
|
+
};
|
|
334
238
|
|
|
335
|
-
const
|
|
239
|
+
const LITERALS = new Map(
|
|
240
|
+
KINDS.flatMap((entry, kind) => entry.slice(1).map((image): [string, number] => [image, kind])),
|
|
241
|
+
);
|
|
242
|
+
const LONGEST_LITERAL = Math.max(...[...LITERALS.keys()].map((image) => image.length));
|
|
243
|
+
const LITERAL_LENGTHS = Array.from(
|
|
244
|
+
{ length: LONGEST_LITERAL },
|
|
245
|
+
(_, index) => LONGEST_LITERAL - index,
|
|
246
|
+
);
|
|
336
247
|
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
/[
|
|
340
|
-
/[
|
|
248
|
+
const BEGIN_MODULE = /----+ *MODULE/y;
|
|
249
|
+
const IDENTIFIER_PATTERNS = [
|
|
250
|
+
/_\w*[A-Za-z]\w*/y,
|
|
251
|
+
/[0-9]\w*[A-Za-z]\w*/y,
|
|
252
|
+
/[WS](?:[a-zA-EG-Z_0-9]\w*)?/y,
|
|
253
|
+
/[WS]F(?:[A-Za-z0-9]\w*)?/y,
|
|
254
|
+
/[a-zA-RT-VX-Z]\w*/y,
|
|
255
|
+
/@|ℕ|ℤ|ℝ/y,
|
|
341
256
|
];
|
|
257
|
+
const getJunctionPatterns = (junction: string) =>
|
|
258
|
+
[/[0-9][A-Za-z0-9]*/, /[WS][a-zA-EG-Z0-9][A-Za-z0-9]*/, /[a-zA-RT-VX-Z][A-Za-z0-9]*/].map(
|
|
259
|
+
(prefix) => new RegExp(`${prefix.source}${junction}`, "y"),
|
|
260
|
+
);
|
|
261
|
+
const BAND_PATTERNS = getJunctionPatterns(String.raw`\.\/\\`);
|
|
262
|
+
const BOR_PATTERNS = getJunctionPatterns(String.raw`\.\\\/`);
|
|
263
|
+
const STEP = String.raw`<(?:[0-9]+|\+|\*)>`;
|
|
342
264
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
265
|
+
// The tokens of the SPEC lexical state that are not literals, with their kinds.
|
|
266
|
+
const SPEC_PATTERNS: [string, RegExp[]][] = [
|
|
267
|
+
["_BM0", [BEGIN_MODULE]],
|
|
268
|
+
["SEPARATOR", [/----+/y]],
|
|
269
|
+
["END_MODULE", [/====+/y]],
|
|
270
|
+
["NUMBER_LITERAL", [/[0-9]+|\\[oO][0-7]+|\\[bB][01]+|\\[hH][0-9a-fA-F]+/y]],
|
|
271
|
+
["STRING_LITERAL", [/"(?:[^"\n\r\\]|\\[ntrf\\"])*"/y]],
|
|
272
|
+
["BAND", BAND_PATTERNS],
|
|
273
|
+
["BOR", BOR_PATTERNS],
|
|
274
|
+
["IDENTIFIER", IDENTIFIER_PATTERNS],
|
|
275
|
+
["ProofStepLexeme", [/<[0-9]+>\w+/y]],
|
|
276
|
+
["ProofImplicitStepLexeme", [/<[+*]>[A-Za-z0-9]+/y]],
|
|
277
|
+
["ProofStepDotLexeme", [new RegExp(String.raw`${STEP}\w+\.+`, "y")]],
|
|
278
|
+
["BareLevelLexeme", [new RegExp(STEP, "y")]],
|
|
279
|
+
["UnnumberedStepLexeme", [new RegExp(String.raw`${STEP}(?:\*\.*|-\.*|\.+)`, "y")]],
|
|
347
280
|
];
|
|
348
281
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
/[WS]F(?:[A-Za-z0-9][A-Za-z0-9_]*)?/y,
|
|
354
|
-
/[a-zA-RT-VX-Z][A-Za-z0-9_]*/y,
|
|
355
|
-
/[@ℕℤℝ]/y,
|
|
282
|
+
// The tokens of the DEFAULT and PRAGMA lexical states, which skip every other character.
|
|
283
|
+
const DEFAULT_PATTERNS: [string, RegExp[]][] = [
|
|
284
|
+
["BEGIN_PRAGMA", [/--->/y]],
|
|
285
|
+
["_BM1", [BEGIN_MODULE]],
|
|
356
286
|
];
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
[
|
|
360
|
-
[
|
|
361
|
-
[
|
|
362
|
-
[
|
|
363
|
-
[STRING_LITERAL, [/"(?:[^"\n\r\\]|\\[ntrf\\"])*"/y]],
|
|
364
|
-
[BAND, BAND_PATTERNS],
|
|
365
|
-
[BOR, BOR_PATTERNS],
|
|
366
|
-
[IDENTIFIER, IDENTIFIER_PATTERNS],
|
|
367
|
-
[PROOF_STEP, [/<[0-9]+>[A-Za-z0-9_]+/y]],
|
|
368
|
-
[PROOF_IMPLICIT_STEP, [/<[+*]>[A-Za-z0-9]+/y]],
|
|
369
|
-
[PROOF_STEP_DOT, [/<(?:[0-9]+|\+|\*)>[A-Za-z0-9_]+\.+/y]],
|
|
370
|
-
[BARE_LEVEL, [/<(?:[0-9]+|\+|\*)>/y]],
|
|
371
|
-
[UNNUMBERED_STEP, [/<(?:[0-9]+|\+|\*)>(?:\*\.*|-\.*|\.+)/y]],
|
|
287
|
+
const PRAGMA_PATTERNS: [string, RegExp[]][] = [
|
|
288
|
+
["NUMBER", [/[0-9]+/y]],
|
|
289
|
+
["_BM2", [BEGIN_MODULE]],
|
|
290
|
+
["BAND", BAND_PATTERNS],
|
|
291
|
+
["BOR", BOR_PATTERNS],
|
|
292
|
+
["IDENTIFIER", IDENTIFIER_PATTERNS],
|
|
372
293
|
];
|
|
373
294
|
|
|
374
|
-
const
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
[BOR, BOR_PATTERNS],
|
|
379
|
-
[IDENTIFIER, IDENTIFIER_PATTERNS],
|
|
380
|
-
];
|
|
295
|
+
const getMatchLength = (pattern: RegExp, source: string, offset: number) => {
|
|
296
|
+
pattern.lastIndex = offset;
|
|
297
|
+
return pattern.exec(source)?.[0].length ?? 0;
|
|
298
|
+
};
|
|
381
299
|
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
})
|
|
389
|
-
|
|
300
|
+
// The longest match at offset, the earliest kind among matches of that length.
|
|
301
|
+
const matchPatterns = (patterns: [string, RegExp[]][], source: string, offset: number) =>
|
|
302
|
+
patterns
|
|
303
|
+
.map(([name, regexes]) => ({
|
|
304
|
+
kind: getKind(name),
|
|
305
|
+
length: Math.max(...regexes.map((regex) => getMatchLength(regex, source, offset))),
|
|
306
|
+
}))
|
|
307
|
+
.reduce(
|
|
308
|
+
(best, match) =>
|
|
309
|
+
match.length > best.length || (match.length === best.length && match.kind < best.kind)
|
|
310
|
+
? match
|
|
311
|
+
: best,
|
|
312
|
+
{ kind: -1, length: 0 },
|
|
313
|
+
);
|
|
390
314
|
|
|
391
|
-
|
|
392
|
-
const
|
|
393
|
-
|
|
394
|
-
index: number,
|
|
395
|
-
patterns: [number, RegExp[]][],
|
|
396
|
-
isSpec: boolean,
|
|
397
|
-
) => {
|
|
398
|
-
const candidates: [number, number][] = patterns.map(([kind, kindPatterns]) => [
|
|
399
|
-
kind,
|
|
400
|
-
getPatternLength(kindPatterns, source, index),
|
|
401
|
-
]);
|
|
402
|
-
if (isSpec) {
|
|
403
|
-
for (const [kind, image] of LITERALS_BY_FIRST_CHARACTER.get(source[index]) ?? []) {
|
|
404
|
-
if (source.startsWith(image, index)) candidates.push([kind, image.length]);
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
return candidates.reduce<[number, number]>(
|
|
408
|
-
(best, candidate) =>
|
|
409
|
-
candidate[1] > best[1] ||
|
|
410
|
-
(candidate[1] === best[1] && candidate[1] > 0 && candidate[0] < best[0])
|
|
411
|
-
? candidate
|
|
412
|
-
: best,
|
|
413
|
-
[EOF, 0],
|
|
315
|
+
const matchSpecToken = (source: string, offset: number) => {
|
|
316
|
+
const literalLength = LITERAL_LENGTHS.find((length) =>
|
|
317
|
+
LITERALS.has(source.slice(offset, offset + length)),
|
|
414
318
|
);
|
|
319
|
+
const pattern = matchPatterns(SPEC_PATTERNS, source, offset);
|
|
320
|
+
if (literalLength === undefined) return pattern;
|
|
321
|
+
const literal = {
|
|
322
|
+
kind: LITERALS.get(source.slice(offset, offset + literalLength)) ?? -1,
|
|
323
|
+
length: literalLength,
|
|
324
|
+
};
|
|
325
|
+
return literal.length > pattern.length ||
|
|
326
|
+
(literal.length === pattern.length && literal.kind < pattern.kind)
|
|
327
|
+
? literal
|
|
328
|
+
: pattern;
|
|
415
329
|
};
|
|
416
330
|
|
|
417
|
-
|
|
418
|
-
const getViablePrefixLength = (source: string, index: number) => {
|
|
419
|
-
if (source[index] === "?") return 1;
|
|
420
|
-
if (source[index] !== '"') return 0;
|
|
421
|
-
const stringPrefix = /"(?:[^"\n\r\\]|\\[ntrf\\"])*\\?/y;
|
|
422
|
-
stringPrefix.lastIndex = index;
|
|
423
|
-
return stringPrefix.exec(source)?.[0].length ?? 0;
|
|
424
|
-
};
|
|
331
|
+
export const getTabStop = (column: number) => column + 8 - (column % 8);
|
|
425
332
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
["\n", "\\n"],
|
|
430
|
-
["\f", "\\f"],
|
|
431
|
-
["\r", "\\r"],
|
|
432
|
-
['"', '\\"'],
|
|
433
|
-
["'", "\\'"],
|
|
434
|
-
["\\", "\\\\"],
|
|
435
|
-
["\0", ""],
|
|
436
|
-
]);
|
|
437
|
-
|
|
438
|
-
// Escapes text as JavaCC's error messages print it.
|
|
439
|
-
export const escapeImage = (text: string) =>
|
|
440
|
-
text
|
|
441
|
-
.split("")
|
|
442
|
-
.map((unit) => {
|
|
443
|
-
const code = unit.charCodeAt(0);
|
|
444
|
-
return (
|
|
445
|
-
ESCAPES.get(unit) ??
|
|
446
|
-
(code < 0x20 || code > 0x7e ? `\\u${code.toString(16).padStart(4, "0")}` : unit)
|
|
447
|
-
);
|
|
448
|
-
})
|
|
449
|
-
.join("");
|
|
450
|
-
|
|
451
|
-
// Reads SANY tokens one at a time, so a lexical error surfaces only when the parser reaches it.
|
|
452
|
-
export const createSanyLexer = (source: string): SanyLexer => {
|
|
333
|
+
// SimpleCharStream.UpdateLineColumn: the line and column of each character, with tabs stopping
|
|
334
|
+
// at multiples of 8.
|
|
335
|
+
const getPositions = (source: string) => {
|
|
453
336
|
const lines: number[] = [];
|
|
454
337
|
const columns: number[] = [];
|
|
455
|
-
const
|
|
456
|
-
for (const character of source
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
338
|
+
const state = { line: 1, column: 0, isAfterCr: false, isAfterLf: false };
|
|
339
|
+
for (const character of source) {
|
|
340
|
+
for (let unit = 0; unit < character.length; unit++) {
|
|
341
|
+
state.column++;
|
|
342
|
+
if (state.isAfterLf) {
|
|
343
|
+
state.isAfterLf = false;
|
|
344
|
+
state.line++;
|
|
345
|
+
state.column = 1;
|
|
346
|
+
} else if (state.isAfterCr) {
|
|
347
|
+
state.isAfterCr = false;
|
|
348
|
+
if (character === "\n") state.isAfterLf = true;
|
|
349
|
+
else {
|
|
350
|
+
state.line++;
|
|
351
|
+
state.column = 1;
|
|
352
|
+
}
|
|
468
353
|
}
|
|
354
|
+
if (character === "\r") state.isAfterCr = true;
|
|
355
|
+
else if (character === "\n") state.isAfterLf = true;
|
|
356
|
+
else if (character === "\t") state.column = getTabStop(state.column - 1);
|
|
357
|
+
lines.push(state.line);
|
|
358
|
+
columns.push(state.column);
|
|
469
359
|
}
|
|
470
|
-
if (character === "\r") position.isAfterCarriageReturn = true;
|
|
471
|
-
if (character === "\n") position.isAfterLineFeed = true;
|
|
472
|
-
if (character === "\t")
|
|
473
|
-
position.column = position.column - 1 + (8 - ((position.column - 1) % 8));
|
|
474
|
-
lines.push(position.line);
|
|
475
|
-
columns.push(position.column);
|
|
476
360
|
}
|
|
477
|
-
|
|
478
|
-
|
|
361
|
+
return { lines, columns };
|
|
362
|
+
};
|
|
363
|
+
|
|
364
|
+
// The characters of a string literal that a string literal could still continue.
|
|
365
|
+
const STRING_PREFIX = /"(?:[^"\n\r\\]|\\[ntrf\\"])*\\?/y;
|
|
366
|
+
const LITERAL_PREFIXES = new Set(
|
|
367
|
+
[...LITERALS.keys()].flatMap((image) =>
|
|
368
|
+
Array.from({ length: image.length - 1 }, (_, index) => image.slice(0, index + 1)),
|
|
369
|
+
),
|
|
370
|
+
);
|
|
371
|
+
|
|
372
|
+
// The characters JavaCC's token manager reads at offset while some token could still continue.
|
|
373
|
+
const getTokenPrefix = (source: string, offset: number) => {
|
|
374
|
+
let length = 0;
|
|
375
|
+
while (LITERAL_PREFIXES.has(source.slice(offset, offset + length + 1))) length++;
|
|
376
|
+
return source.slice(
|
|
377
|
+
offset,
|
|
378
|
+
offset + Math.max(length, getMatchLength(STRING_PREFIX, source, offset)),
|
|
379
|
+
);
|
|
380
|
+
};
|
|
381
|
+
|
|
382
|
+
// A lazy token stream over source, which returns SANY's message for the TokenMgrError JavaCC's
|
|
383
|
+
// token manager throws at a character no token continues with, or at the end of the file in a
|
|
384
|
+
// comment.
|
|
385
|
+
export const createSanyLexer = (source: string) => {
|
|
386
|
+
const positions = getPositions(source);
|
|
387
|
+
const state = { offset: 0, mode: "DEFAULT" };
|
|
388
|
+
const createToken = (kind: number, start: number, length: number): SanyToken => ({
|
|
479
389
|
kind,
|
|
480
|
-
image: source.slice(
|
|
481
|
-
beginLine: lines[
|
|
482
|
-
beginColumn: columns[
|
|
483
|
-
endLine: lines[end - 1],
|
|
484
|
-
endColumn: columns[end - 1],
|
|
390
|
+
image: source.slice(start, start + length),
|
|
391
|
+
beginLine: positions.lines[start] ?? 0,
|
|
392
|
+
beginColumn: positions.columns[start] ?? 0,
|
|
485
393
|
next: undefined,
|
|
486
394
|
});
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
395
|
+
// JavaCC places the end of the file at the file's last character.
|
|
396
|
+
const getEndOfFile = () => createToken(getKind("EOF"), source.length - 1, 0);
|
|
397
|
+
// The lexer read the characters from start to end, and TLAplusParser.parse reports the end of
|
|
398
|
+
// the file in a token that starts on an earlier line as an open comment.
|
|
399
|
+
const getLexicalError = (start: number, end: number, after: string) => {
|
|
400
|
+
const index = end - 1;
|
|
401
|
+
const line = positions.lines[index];
|
|
402
|
+
const column = positions.columns[index];
|
|
403
|
+
if (end < source.length) return formatLexicalError(line, column, source[index], after);
|
|
404
|
+
if (positions.lines[start] !== line) {
|
|
405
|
+
return `Lexical {error: EOF reached, possibly open comment starting around line ${positions.lines[start]}`;
|
|
492
406
|
}
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
throw new TlaParseError(
|
|
497
|
-
lines[begin] === lines[last]
|
|
498
|
-
? message
|
|
499
|
-
: `Lexical {error: EOF reached, possibly open comment starting around line ${lines[begin]}`,
|
|
500
|
-
);
|
|
407
|
+
return source[index] === "\n" || source[index] === "\r"
|
|
408
|
+
? formatLexicalError(line + 1, 0, undefined, after)
|
|
409
|
+
: formatLexicalError(line, column + 1, undefined, after);
|
|
501
410
|
};
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
411
|
+
// A comment is a chain of MORE matches, which restarts after each nested comment bracket, and the
|
|
412
|
+
// end of the file right after a bracket ends the token stream.
|
|
413
|
+
const skipComment = () => {
|
|
414
|
+
const start = state.offset;
|
|
415
|
+
if (source.startsWith("\\*", start)) {
|
|
416
|
+
const end = source.slice(start).search(/[\n\r]/);
|
|
417
|
+
if (end === -1) return getLexicalError(start, source.length, "");
|
|
418
|
+
state.offset += end + 1;
|
|
419
|
+
return undefined;
|
|
420
|
+
}
|
|
421
|
+
let depth = 0;
|
|
422
|
+
let chainStart = start;
|
|
423
|
+
let offset = start;
|
|
424
|
+
while (offset < source.length) {
|
|
425
|
+
if (source.startsWith("(*", offset)) {
|
|
426
|
+
depth++;
|
|
427
|
+
offset += 2;
|
|
428
|
+
if (depth > 1) chainStart = offset;
|
|
429
|
+
} else if (source.startsWith("*)", offset)) {
|
|
430
|
+
depth--;
|
|
431
|
+
offset += 2;
|
|
432
|
+
chainStart = offset;
|
|
433
|
+
if (depth === 0) {
|
|
434
|
+
state.offset = offset;
|
|
519
435
|
return undefined;
|
|
520
436
|
}
|
|
521
|
-
} else
|
|
522
|
-
cursor.index += opening;
|
|
523
|
-
cursor.commentDepth++;
|
|
524
|
-
cursor.state = LexicalState.Embedded;
|
|
525
|
-
return undefined;
|
|
526
|
-
} else if (source.startsWith("*)", cursor.index)) {
|
|
527
|
-
cursor.index += 2;
|
|
528
|
-
if (cursor.state === LexicalState.InComment) cursor.state = LexicalState.Spec;
|
|
529
|
-
else {
|
|
530
|
-
cursor.commentDepth--;
|
|
531
|
-
if (cursor.commentDepth === 0) cursor.state = LexicalState.InComment;
|
|
532
|
-
}
|
|
533
|
-
return undefined;
|
|
534
|
-
}
|
|
535
|
-
cursor.index++;
|
|
437
|
+
} else offset++;
|
|
536
438
|
}
|
|
439
|
+
state.offset = offset;
|
|
440
|
+
return chainStart === source.length ? undefined : getLexicalError(chainStart, offset, "");
|
|
537
441
|
};
|
|
538
|
-
const
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
endLine: lines[last] ?? 0,
|
|
548
|
-
endColumn: columns[last] ?? 0,
|
|
549
|
-
next: undefined,
|
|
550
|
-
};
|
|
442
|
+
const getSkippingToken = (patterns: [string, RegExp[]][]): SanyToken => {
|
|
443
|
+
while (state.offset < source.length) {
|
|
444
|
+
const match = matchPatterns(patterns, source, state.offset);
|
|
445
|
+
if (match.length > 0) {
|
|
446
|
+
const token = createToken(match.kind, state.offset, match.length);
|
|
447
|
+
state.offset += match.length;
|
|
448
|
+
if (KIND_NAMES[match.kind].startsWith("_BM")) state.mode = "SPEC";
|
|
449
|
+
if (KIND_NAMES[match.kind] === "BEGIN_PRAGMA") state.mode = "PRAGMA";
|
|
450
|
+
return token;
|
|
551
451
|
}
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
continue;
|
|
575
|
-
}
|
|
576
|
-
cursor.index += length;
|
|
577
|
-
if (kind === _BM2) cursor.state = LexicalState.Spec;
|
|
578
|
-
return createToken(kind, begin, cursor.index);
|
|
579
|
-
}
|
|
580
|
-
if (cursor.state !== LexicalState.Spec) {
|
|
581
|
-
readComment(begin);
|
|
582
|
-
continue;
|
|
583
|
-
}
|
|
584
|
-
if (" \t\n\r".includes(source[cursor.index])) {
|
|
585
|
-
cursor.index++;
|
|
586
|
-
continue;
|
|
587
|
-
}
|
|
588
|
-
if (source.startsWith("(*", cursor.index) || source.startsWith("\\*", cursor.index)) {
|
|
589
|
-
cursor.state =
|
|
590
|
-
source[cursor.index] === "(" ? LexicalState.InComment : LexicalState.InEndOfLineComment;
|
|
591
|
-
cursor.index += source.startsWith("(*.", cursor.index) ? 3 : 2;
|
|
592
|
-
readComment(begin);
|
|
593
|
-
continue;
|
|
594
|
-
}
|
|
595
|
-
const [kind, length] = matchToken(source, cursor.index, SPEC_PATTERNS, true);
|
|
596
|
-
if (length === 0) {
|
|
597
|
-
const viable = getViablePrefixLength(source, cursor.index);
|
|
598
|
-
const killer = cursor.index + viable;
|
|
599
|
-
if (killer < source.length) {
|
|
600
|
-
return throwLexicalError(
|
|
601
|
-
begin,
|
|
602
|
-
killer + 1 < source.length || viable === 0
|
|
603
|
-
? source.slice(begin, killer)
|
|
604
|
-
: source.slice(begin, killer + 1),
|
|
605
|
-
killer,
|
|
606
|
-
);
|
|
452
|
+
state.offset++;
|
|
453
|
+
}
|
|
454
|
+
return getEndOfFile();
|
|
455
|
+
};
|
|
456
|
+
const getNextToken = (): SanyToken | string => {
|
|
457
|
+
if (state.mode === "DEFAULT") return getSkippingToken(DEFAULT_PATTERNS);
|
|
458
|
+
if (state.mode === "PRAGMA") return getSkippingToken(PRAGMA_PATTERNS);
|
|
459
|
+
while (state.offset < source.length) {
|
|
460
|
+
const offset = state.offset;
|
|
461
|
+
if (" \t\n\r".includes(source[offset])) state.offset++;
|
|
462
|
+
else if (source.startsWith("(*", offset) || source.startsWith("\\*", offset)) {
|
|
463
|
+
const error = skipComment();
|
|
464
|
+
if (error !== undefined) return error;
|
|
465
|
+
} else {
|
|
466
|
+
const match = matchSpecToken(source, offset);
|
|
467
|
+
if (match.length === 0) {
|
|
468
|
+
// JavaCC reads on while a token could continue, and one more character.
|
|
469
|
+
const prefix = getTokenPrefix(source, offset);
|
|
470
|
+
const end = Math.min(offset + prefix.length + 1, source.length);
|
|
471
|
+
const read = source.slice(offset, end);
|
|
472
|
+
const after = end < source.length ? prefix : read.length > 1 ? read : "";
|
|
473
|
+
return getLexicalError(offset, end, after);
|
|
607
474
|
}
|
|
608
|
-
|
|
475
|
+
const token = createToken(match.kind, offset, match.length);
|
|
476
|
+
state.offset += match.length;
|
|
477
|
+
return token;
|
|
609
478
|
}
|
|
610
|
-
cursor.index += length;
|
|
611
|
-
return createToken(kind, begin, cursor.index);
|
|
612
479
|
}
|
|
480
|
+
return getEndOfFile();
|
|
613
481
|
};
|
|
614
|
-
return
|
|
482
|
+
return getNextToken;
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
// JavaCC's TokenMgrError.addEscapes, which escapes the characters of a lexical error's message.
|
|
486
|
+
const LEXEME_ESCAPES = new Map([
|
|
487
|
+
["\b", "\\b"],
|
|
488
|
+
["\t", "\\t"],
|
|
489
|
+
["\n", "\\n"],
|
|
490
|
+
["\f", "\\f"],
|
|
491
|
+
["\r", "\\r"],
|
|
492
|
+
['"', '\\"'],
|
|
493
|
+
["'", "\\'"],
|
|
494
|
+
["\\", "\\\\"],
|
|
495
|
+
]);
|
|
496
|
+
|
|
497
|
+
export const escapeLexeme = (text: string) =>
|
|
498
|
+
text.replace(
|
|
499
|
+
// eslint-disable-next-line no-control-regex
|
|
500
|
+
/[\0-\x1f"'\\\x7f-\uffff]/g,
|
|
501
|
+
(character) =>
|
|
502
|
+
LEXEME_ESCAPES.get(character) ??
|
|
503
|
+
(character === "\0" ? "" : `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`),
|
|
504
|
+
);
|
|
505
|
+
|
|
506
|
+
// TokenMgrError.LexicalError: the character the lexer stopped at, or the end of the file, and the
|
|
507
|
+
// characters of the token it had read.
|
|
508
|
+
export const formatLexicalError = (
|
|
509
|
+
line: number,
|
|
510
|
+
column: number,
|
|
511
|
+
character: string | undefined,
|
|
512
|
+
after: string,
|
|
513
|
+
) =>
|
|
514
|
+
`Lexical error at line ${line}, column ${column}. Encountered: ${character === undefined ? "<EOF> " : `"${escapeLexeme(character)}" (${character.charCodeAt(0)}), `}after : "${escapeLexeme(after)}"`;
|
|
515
|
+
|
|
516
|
+
// TLC 1.7.4's lexer, which JavaCC generated without UNICODE_INPUT, reads a character past U+00FF
|
|
517
|
+
// outside a comment by its low byte, and stops at one whose low byte is below 0x80.
|
|
518
|
+
export const getSany174LexicalError = (source: string) => {
|
|
519
|
+
const readToken = createSanyLexer(source);
|
|
520
|
+
for (
|
|
521
|
+
let token = readToken();
|
|
522
|
+
typeof token !== "string" && token.kind !== getKind("EOF");
|
|
523
|
+
token = readToken()
|
|
524
|
+
) {
|
|
525
|
+
const index = token.image.split("").findIndex((character) => {
|
|
526
|
+
const code = character.charCodeAt(0);
|
|
527
|
+
return code > 0xff && (code & 0xff) < 0x80;
|
|
528
|
+
});
|
|
529
|
+
if (index === -1) continue;
|
|
530
|
+
const column = token.image
|
|
531
|
+
.slice(1, index + 1)
|
|
532
|
+
.split("")
|
|
533
|
+
.reduce(
|
|
534
|
+
(previous, character) => (character === "\t" ? getTabStop(previous) : previous + 1),
|
|
535
|
+
token.beginColumn,
|
|
536
|
+
);
|
|
537
|
+
return formatLexicalError(
|
|
538
|
+
token.beginLine,
|
|
539
|
+
column,
|
|
540
|
+
token.image[index],
|
|
541
|
+
token.image.slice(0, index),
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
return undefined;
|
|
615
545
|
};
|