tla-ts 0.0.3-dev.e5b05b9 → 0.0.3-dev.e8bef80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/cli.cjs +1 -1
  2. package/dist/cli.js +1 -1
  3. package/dist/index.cjs +1 -1
  4. package/dist/index.d.cts +150 -48
  5. package/dist/index.d.ts +150 -48
  6. package/dist/index.js +1 -1
  7. package/dist/parallel-worker.cjs +1 -1
  8. package/dist/parallel-worker.js +2 -2
  9. package/dist/parallel.cjs +1 -1
  10. package/dist/parallel.js +2 -2
  11. package/dist/run-cli.cjs +1351 -22
  12. package/dist/run-cli.js +1352 -23
  13. package/dist/transfer.cjs +83 -43
  14. package/dist/transfer.js +83 -43
  15. package/package.json +5 -1
  16. package/src/ast.ts +28 -4
  17. package/src/bags.ts +4 -7
  18. package/src/builtins.ts +29 -17
  19. package/src/bundled-modules.ts +3 -3
  20. package/src/canonical.ts +4 -3
  21. package/src/check.ts +1317 -428
  22. package/src/cli.ts +8 -1
  23. package/src/codegen.ts +68 -21
  24. package/src/config.ts +226 -207
  25. package/src/environment.ts +221 -75
  26. package/src/errors.ts +130 -15
  27. package/src/evaluate.ts +713 -259
  28. package/src/exit-codes.ts +6 -24
  29. package/src/fingerprints.ts +1 -2
  30. package/src/index.ts +1 -0
  31. package/src/instances.ts +3 -1
  32. package/src/java-overrides.ts +104 -25
  33. package/src/levels.ts +6 -1
  34. package/src/lexer.ts +122 -29
  35. package/src/live-check.ts +204 -148
  36. package/src/liveness.ts +191 -122
  37. package/src/obligation-expression.ts +2 -2
  38. package/src/obligation-source.ts +2 -0
  39. package/src/parallel-worker.ts +6 -1
  40. package/src/parallel.ts +57 -6
  41. package/src/parser.ts +327 -147
  42. package/src/pcal-ast.ts +770 -0
  43. package/src/pcal-parse.ts +1523 -0
  44. package/src/pcal-tokenize.ts +516 -0
  45. package/src/pcal-translate.ts +1230 -0
  46. package/src/pcal-validate.ts +454 -0
  47. package/src/randomization.ts +24 -7
  48. package/src/resolve.ts +252 -87
  49. package/src/run-cli.ts +1162 -200
  50. package/src/runtime.ts +16 -0
  51. package/src/sany-builtins.ts +147 -0
  52. package/src/sany-generator.ts +3128 -0
  53. package/src/sany-levels.ts +1208 -0
  54. package/src/sany-lexer.ts +410 -480
  55. package/src/sany-linter.ts +122 -0
  56. package/src/sany-modules.ts +1398 -0
  57. package/src/sany-nodes.ts +463 -0
  58. package/src/sany-parse-error.ts +2033 -3015
  59. package/src/sany-report.ts +356 -0
  60. package/src/sany-selectors.ts +738 -0
  61. package/src/sany-source.ts +248 -0
  62. package/src/sany-xml.ts +1017 -0
  63. package/src/spec-description.ts +25 -5
  64. package/src/standard-modules.ts +172 -129
  65. package/src/state-graph.ts +29 -6
  66. package/src/subexpressions.ts +121 -26
  67. package/src/successors.ts +141 -173
  68. package/src/symmetry.ts +168 -50
  69. package/src/tableau.ts +3 -1
  70. package/src/tlc-ext.ts +13 -17
  71. package/src/tlc-messages.ts +329 -0
  72. package/src/tlc-module.ts +49 -21
  73. package/src/tlc-modules.ts +0 -4
  74. package/src/tlc-order.ts +55 -9
  75. package/src/tlc-random.ts +28 -3
  76. package/src/tlc-replay.ts +655 -0
  77. package/src/tlc-version.ts +18 -0
  78. package/src/transfer.ts +15 -3
  79. package/src/values.ts +664 -317
package/src/sany-lexer.ts CHANGED
@@ -1,222 +1,123 @@
1
- import { TlaParseError } from "./errors";
1
+ import { TlaError } from "./errors";
2
2
 
3
- // The token kinds of SANY's TLA+ grammar, numbered as in its TLAplusParserConstants.
4
- export const EOF = 0;
5
- export const BEGIN_PRAGMA = 2;
6
- export const _BM1 = 3;
7
- export const NUMBER = 20;
8
- export const _BM2 = 21;
9
- export const _BM0 = 35;
10
- export const SEPARATOR = 36;
11
- export const END_MODULE = 37;
12
- export const ACTION = 38;
13
- export const ASSUME = 39;
14
- export const BOXASSUME = 40;
15
- export const ASSUMPTION = 41;
16
- export const CASE = 42;
17
- export const CHOOSE = 43;
18
- export const CONSTANT = 44;
19
- export const ELSE = 45;
20
- export const EXCEPT = 46;
21
- export const EXISTS = 47;
22
- export const EXTENDS = 48;
23
- export const FORALL = 49;
24
- export const IF = 50;
25
- export const INSTANCE = 51;
26
- export const LET = 52;
27
- export const LETIN = 53;
28
- export const LOCAL = 54;
29
- export const MODULE = 55;
30
- export const NEW = 56;
31
- export const OTHER = 57;
32
- export const PROPOSITION = 58;
33
- export const SF = 59;
34
- export const T_EXISTS = 60;
35
- export const T_FORALL = 61;
36
- export const THEN = 62;
37
- export const BY = 63;
38
- export const ONLY = 64;
39
- export const DEFINE = 65;
40
- export const DF = 66;
41
- export const THEOREM = 67;
42
- export const USE = 68;
43
- export const HIDE = 69;
44
- export const HAVE = 70;
45
- export const OBVIOUS = 71;
46
- export const OMITTED = 72;
47
- export const LAMBDA = 73;
48
- export const TAKE = 74;
49
- export const PROOF = 75;
50
- export const PROVE = 76;
51
- export const BOXPROVE = 77;
52
- export const QED = 78;
53
- export const RECURSIVE = 79;
54
- export const STATE = 80;
55
- export const TEMPORAL = 81;
56
- export const PICK = 82;
57
- export const WITNESS = 83;
58
- export const SUFFICES = 84;
59
- export const VARIABLE = 85;
60
- export const WF = 86;
61
- export const WITH = 87;
62
- export const COMMA = 88;
63
- export const COLON = 89;
64
- export const COLONCOLON = 90;
65
- export const DOT = 91;
66
- export const US = 92;
67
- export const DEF = 93;
68
- export const LBR = 94;
69
- export const RBR = 95;
70
- export const DEFBREAK = 96;
71
- export const LSB = 97;
72
- export const ARSB = 98;
73
- export const RSB = 99;
74
- export const LBC = 100;
75
- export const RBC = 101;
76
- export const LAB = 102;
77
- export const ARAB = 103;
78
- export const RAB = 104;
79
- export const BANG = 105;
80
- export const ARROW = 106;
81
- export const SUBSTITUTE = 107;
82
- export const MAPTO = 108;
83
- export const NUMBER_LITERAL = 109;
84
- export const STRING_LITERAL = 110;
85
- const BAND = 111;
86
- const BOR = 112;
87
- export const PRIME = 113;
88
- export const NEGATIVE = 117;
89
- export const CASESEP = 121;
90
- export const AND = 129;
91
- export const OR = 132;
92
- export const IN = 147;
93
- export const EQUALS = 185;
94
- export const MINUS = 193;
95
- export const IDENTIFIER = 289;
96
- export const PROOF_STEP = 290;
97
- export const PROOF_IMPLICIT_STEP = 291;
98
- export const PROOF_STEP_DOT = 292;
99
- export const BARE_LEVEL = 293;
100
- export const UNNUMBERED_STEP = 294;
3
+ // An emulation of the token manager that JavaCC generates for SANY's grammar, javacc/tla+.jj: the
4
+ // tokens it produces, their kinds in TLAplusParserConstants order, and their positions.
101
5
 
102
6
  export interface SanyToken {
103
7
  kind: number;
104
8
  image: string;
105
9
  beginLine: number;
106
10
  beginColumn: number;
107
- endLine: number;
108
- endColumn: number;
109
11
  next: SanyToken | undefined;
110
12
  }
111
13
 
112
- export interface SanyLexer {
113
- getNextToken: () => SanyToken;
114
- }
115
-
116
- const enum LexicalState {
117
- Default,
118
- Pragma,
119
- Spec,
120
- InComment,
121
- Embedded,
122
- InEndOfLineComment,
123
- }
124
-
125
- const UNICODE_INFIX_OPERATORS = "≈≔≍◯⩴●∩⋅∘≅∪÷≐‥…≡‼≥≫⇔⇒⫤↝≤≪⊣≠∉⊙⊖⊕⊘⊗⇸≺⪯∝⁇⊨⊢∼≃⊓⊔⊏⊑⊐⊒⋆⊂⊆≻⪰⊃⊇×⊎‖≀";
126
-
127
- const LITERALS: [number, string[]][] = [
128
- [ACTION, ["ACTION", "ACTIONS"]],
129
- [ASSUME, ["ASSUME"]],
130
- [BOXASSUME, ["[]ASSUME"]],
131
- [ASSUMPTION, ["ASSUMPTION", "AXIOM"]],
132
- [CASE, ["CASE"]],
133
- [CHOOSE, ["CHOOSE"]],
134
- [CONSTANT, ["CONSTANT", "CONSTANTS"]],
135
- [ELSE, ["ELSE"]],
136
- [EXCEPT, ["EXCEPT"]],
137
- [EXISTS, ["\\E", "\\exists", "∃"]],
138
- [EXTENDS, ["EXTENDS"]],
139
- [FORALL, ["\\A", "\\forall", "∀"]],
140
- [IF, ["IF"]],
141
- [INSTANCE, ["INSTANCE"]],
142
- [LET, ["LET"]],
143
- [LETIN, ["IN"]],
144
- [LOCAL, ["LOCAL"]],
145
- [MODULE, ["MODULE"]],
146
- [NEW, ["NEW"]],
147
- [OTHER, ["OTHER"]],
148
- [PROPOSITION, ["PROPOSITION", "LEMMA", "COROLLARY"]],
149
- [SF, ["SF_"]],
150
- [T_EXISTS, ["\\EE"]],
151
- [T_FORALL, ["\\AA"]],
152
- [THEN, ["THEN"]],
153
- [BY, ["BY"]],
154
- [ONLY, ["ONLY"]],
155
- [DEFINE, ["DEFINE"]],
156
- [DF, ["DEF", "DEFS"]],
157
- [THEOREM, ["THEOREM"]],
158
- [USE, ["USE"]],
159
- [HIDE, ["HIDE"]],
160
- [HAVE, ["HAVE"]],
161
- [OBVIOUS, ["OBVIOUS"]],
162
- [OMITTED, ["OMITTED"]],
163
- [LAMBDA, ["LAMBDA"]],
164
- [TAKE, ["TAKE"]],
165
- [PROOF, ["PROOF"]],
166
- [PROVE, ["PROVE"]],
167
- [BOXPROVE, ["[]PROVE"]],
168
- [QED, ["QED"]],
169
- [RECURSIVE, ["RECURSIVE"]],
170
- [STATE, ["STATE"]],
171
- [TEMPORAL, ["TEMPORAL", "TEMPORALS"]],
172
- [PICK, ["PICK"]],
173
- [WITNESS, ["WITNESS"]],
174
- [SUFFICES, ["SUFFICES"]],
175
- [VARIABLE, ["VARIABLE", "VARIABLES"]],
176
- [WF, ["WF_"]],
177
- [WITH, ["WITH"]],
178
- [COMMA, [","]],
179
- [COLON, [":"]],
180
- [COLONCOLON, ["::", "∷"]],
181
- [DOT, ["."]],
182
- [US, ["_"]],
183
- [DEF, ["==", "≜"]],
184
- [LBR, ["("]],
185
- [RBR, [")"]],
186
- [DEFBREAK, ["-|-"]],
187
- [LSB, ["["]],
188
- [ARSB, ["]_"]],
189
- [RSB, ["]"]],
190
- [LBC, ["{"]],
191
- [RBC, ["}"]],
192
- [LAB, ["<<", "⟨"]],
193
- [ARAB, [">>_", "⟩_"]],
194
- [RAB, [">>", "⟩"]],
195
- [BANG, ["!"]],
196
- [ARROW, ["->", "→"]],
197
- [SUBSTITUTE, ["<-", "←"]],
198
- [MAPTO, ["|->", "↦"]],
199
- [PRIME, ["'"]],
200
- [114, ["^+", "⁺"]],
201
- [115, ["^*"]],
202
- [116, ["^#"]],
203
- [NEGATIVE, ["-."]],
204
- [118, ["\\lnot", "¬"]],
205
- [119, ["\\neg"]],
206
- [120, ["~"]],
207
- [CASESEP, ["[]", "□"]],
208
- [122, ["<>", "◇"]],
209
- [123, ["ENABLED"]],
210
- [124, ["UNCHANGED"]],
211
- [125, ["SUBSET"]],
212
- [126, ["UNION"]],
213
- [127, ["DOMAIN"]],
14
+ // The token kinds, in the order of TLAplusParserConstants, each with the literal images that
15
+ // produce it. JavaCC breaks a tie between matches of the same length by this order.
16
+ const KINDS: [string, ...string[]][] = [
17
+ ["EOF"],
18
+ ["BEGIN_PRAGMA", "--->"],
19
+ ["_BM1"],
20
+ ["NUMBER"],
21
+ ["_BM2"],
22
+ ["_BM0"],
23
+ ["SEPARATOR"],
24
+ ["END_MODULE"],
25
+ ["ACTION", "ACTION", "ACTIONS"],
26
+ ["ASSUME", "ASSUME"],
27
+ ["BOXASSUME", "[]ASSUME"],
28
+ ["ASSUMPTION", "ASSUMPTION", "AXIOM"],
29
+ ["CASE", "CASE"],
30
+ ["CHOOSE", "CHOOSE"],
31
+ ["CONSTANT", "CONSTANT", "CONSTANTS"],
32
+ ["ELSE", "ELSE"],
33
+ ["EXCEPT", "EXCEPT"],
34
+ ["EXISTS", "\\E", "\\exists", "∃"],
35
+ ["EXTENDS", "EXTENDS"],
36
+ ["FORALL", "\\A", "\\forall", "∀"],
37
+ ["IF", "IF"],
38
+ ["INSTANCE", "INSTANCE"],
39
+ ["LET", "LET"],
40
+ ["LETIN", "IN"],
41
+ ["LOCAL", "LOCAL"],
42
+ ["MODULE", "MODULE"],
43
+ ["NEW", "NEW"],
44
+ ["OTHER", "OTHER"],
45
+ ["PROPOSITION", "PROPOSITION", "LEMMA", "COROLLARY"],
46
+ ["SF", "SF_"],
47
+ ["T_EXISTS", "\\EE"],
48
+ ["T_FORALL", "\\AA"],
49
+ ["THEN", "THEN"],
50
+ ["BY", "BY"],
51
+ ["ONLY", "ONLY"],
52
+ ["DEFINE", "DEFINE"],
53
+ ["DF", "DEF", "DEFS"],
54
+ ["THEOREM", "THEOREM"],
55
+ ["USE", "USE"],
56
+ ["HIDE", "HIDE"],
57
+ ["HAVE", "HAVE"],
58
+ ["OBVIOUS", "OBVIOUS"],
59
+ ["OMITTED", "OMITTED"],
60
+ ["LAMBDA", "LAMBDA"],
61
+ ["TAKE", "TAKE"],
62
+ ["PROOF", "PROOF"],
63
+ ["PROVE", "PROVE"],
64
+ ["BOXPROVE", "[]PROVE"],
65
+ ["QED", "QED"],
66
+ ["RECURSIVE", "RECURSIVE"],
67
+ ["STATE", "STATE"],
68
+ ["TEMPORAL", "TEMPORAL", "TEMPORALS"],
69
+ ["PICK", "PICK"],
70
+ ["WITNESS", "WITNESS"],
71
+ ["SUFFICES", "SUFFICES"],
72
+ ["VARIABLE", "VARIABLE", "VARIABLES"],
73
+ ["WF", "WF_"],
74
+ ["WITH", "WITH"],
75
+ ["COMMA", ","],
76
+ ["COLON", ":"],
77
+ ["COLONCOLON", "::", "∷"],
78
+ ["DOT", "."],
79
+ ["US", "_"],
80
+ ["DEF", "==", "≜"],
81
+ ["LBR", "("],
82
+ ["RBR", ")"],
83
+ ["DEFBREAK", "-|-"],
84
+ ["LSB", "["],
85
+ ["ARSB", "]_"],
86
+ ["RSB", "]"],
87
+ ["LBC", "{"],
88
+ ["RBC", "}"],
89
+ ["LAB", "<<", "⟨"],
90
+ ["ARAB", ">>_", "⟩_"],
91
+ ["RAB", ">>", "⟩"],
92
+ ["BANG", "!"],
93
+ ["ARROW", "->", "→"],
94
+ ["SUBSTITUTE", "<-", "←"],
95
+ ["MAPTO", "|->", "↦"],
96
+ ["NUMBER_LITERAL"],
97
+ ["STRING_LITERAL"],
98
+ ["BAND"],
99
+ ["BOR"],
100
+ ["op_57", "'"],
101
+ ["op_68", "^+", "⁺"],
102
+ ["op_69", "^*"],
103
+ ["op_70", "^#"],
104
+ ["op_76", "-."],
105
+ ["op_26", "\\lnot", "¬"],
106
+ ["op_29", "\\neg"],
107
+ ["op_58", "~"],
108
+ ["CASESEP", "[]", "□"],
109
+ ["op_61", "<>", "◇"],
110
+ ["op_112", "ENABLED"],
111
+ ["op_113", "UNCHANGED"],
112
+ ["op_114", "SUBSET"],
113
+ ["op_115", "UNION"],
114
+ ["op_116", "DOMAIN"],
115
+ ["op_1", "//"],
116
+ ["AND", "/\\", "∧"],
117
+ ["op_3", "/="],
118
+ ["op_4", "/"],
119
+ ["OR", "\\/", "∨"],
214
120
  ...[
215
- "//",
216
- "/\\",
217
- "/=",
218
- "/",
219
- "\\/",
220
121
  "\\approx",
221
122
  "\\asymp",
222
123
  "\\bigcirc",
@@ -231,7 +132,9 @@ const LITERALS: [number, string[]][] = [
231
132
  "\\equiv",
232
133
  "\\geq",
233
134
  "\\gg",
234
- "\\in",
135
+ ].map((image): [string, string] => ["infix", image]),
136
+ ["IN", "\\in", "∈"],
137
+ ...[
235
138
  "\\intersect",
236
139
  "\\union",
237
140
  "\\land",
@@ -269,15 +172,11 @@ const LITERALS: [number, string[]][] = [
269
172
  "=>",
270
173
  "=<",
271
174
  "=|",
272
- "=",
273
- "##",
274
- "#",
275
- "^^",
276
- "^",
277
- "--",
278
- "-|",
279
- "-+->",
280
- "-",
175
+ ].map((image): [string, string] => ["infix", image]),
176
+ ["EQUALS", "="],
177
+ ...["##", "#", "^^", "^", "--", "-|", "-+->"].map((image): [string, string] => ["infix", image]),
178
+ ["op_77", "-"],
179
+ ...[
281
180
  "**",
282
181
  "*",
283
182
  "++",
@@ -312,304 +211,335 @@ const LITERALS: [number, string[]][] = [
312
211
  "(/)",
313
212
  "(\\X)",
314
213
  "\\notin",
315
- ...UNICODE_INFIX_OPERATORS,
214
+ ..."≈≔≍◯⩴●∩⋅∘≅∪÷≐‥…≡‼≥≫⇔⇒⫤↝≤≪⊣≠∉⊙⊖⊕⊘⊗⇸≺⪯∝⁇⊨⊢∼≃⊓⊔⊏⊑⊐⊒⋆⊂⊆≻⪰⊃⊇×⊎‖≀",
316
215
  "\\times",
317
- "\\X",
318
- ].map((operator, index): [number, string[]] => [128 + index, [operator]]),
319
- [AND, ["∧"]],
320
- [OR, ["∨"]],
321
- [IN, ["∈"]],
216
+ ].map((image): [string, string] => ["infix", image]),
217
+ ["op_119", "\\X"],
218
+ ["IDENTIFIER"],
219
+ ["ProofStepLexeme"],
220
+ ["ProofImplicitStepLexeme"],
221
+ ["ProofStepDotLexeme"],
222
+ ["BareLevelLexeme"],
223
+ ["UnnumberedStepLexeme"],
322
224
  ];
323
225
 
324
- const getLiteralsByFirstCharacter = () => {
325
- const literals = new Map<string, [number, string][]>();
326
- for (const entry of LITERALS) {
327
- for (const image of entry[1])
328
- literals.set(image[0], [...(literals.get(image[0]) ?? []), [entry[0], image]]);
329
- }
330
- return literals;
331
- };
226
+ const KIND_NAMES = KINDS.map((entry) => entry[0]);
227
+ const KIND_INDEXES = new Map(
228
+ KIND_NAMES.map((name, kind): [string, number] => [name, kind]).reverse(),
229
+ );
332
230
 
333
- const LITERALS_BY_FIRST_CHARACTER = getLiteralsByFirstCharacter();
231
+ // The kind named name. The anonymous infix operators share the name "infix", which no caller
232
+ // looks up.
233
+ export const getKind = (name: string) => {
234
+ const kind = KIND_INDEXES.get(name);
235
+ if (kind === undefined) throw new TlaError(`Unknown SANY token kind ${name}`);
236
+ return kind;
237
+ };
334
238
 
335
- const BEGIN_MODULE = /-{4,} *MODULE/y;
239
+ const LITERALS = new Map(
240
+ KINDS.flatMap((entry, kind) => entry.slice(1).map((image): [string, number] => [image, kind])),
241
+ );
242
+ const LONGEST_LITERAL = Math.max(...[...LITERALS.keys()].map((image) => image.length));
243
+ const LITERAL_LENGTHS = Array.from(
244
+ { length: LONGEST_LITERAL },
245
+ (_, index) => LONGEST_LITERAL - index,
246
+ );
336
247
 
337
- const BAND_PATTERNS = [
338
- /[0-9][A-Za-z0-9]*\.\/\\/y,
339
- /[WS][a-zA-EG-Z0-9][A-Za-z0-9]*\.\/\\/y,
340
- /[a-zA-RT-VX-Z][A-Za-z0-9]*\.\/\\/y,
248
+ const BEGIN_MODULE = /----+ *MODULE/y;
249
+ const IDENTIFIER_PATTERNS = [
250
+ /_\w*[A-Za-z]\w*/y,
251
+ /[0-9]\w*[A-Za-z]\w*/y,
252
+ /[WS](?:[a-zA-EG-Z_0-9]\w*)?/y,
253
+ /[WS]F(?:[A-Za-z0-9]\w*)?/y,
254
+ /[a-zA-RT-VX-Z]\w*/y,
255
+ /@|ℕ|ℤ|ℝ/y,
341
256
  ];
257
+ const getJunctionPatterns = (junction: string) =>
258
+ [/[0-9][A-Za-z0-9]*/, /[WS][a-zA-EG-Z0-9][A-Za-z0-9]*/, /[a-zA-RT-VX-Z][A-Za-z0-9]*/].map(
259
+ (prefix) => new RegExp(`${prefix.source}${junction}`, "y"),
260
+ );
261
+ const BAND_PATTERNS = getJunctionPatterns(String.raw`\.\/\\`);
262
+ const BOR_PATTERNS = getJunctionPatterns(String.raw`\.\\\/`);
263
+ const STEP = String.raw`<(?:[0-9]+|\+|\*)>`;
342
264
 
343
- const BOR_PATTERNS = [
344
- /[0-9][A-Za-z0-9]*\.\\\//y,
345
- /[WS][a-zA-EG-Z0-9][A-Za-z0-9]*\.\\\//y,
346
- /[a-zA-RT-VX-Z][A-Za-z0-9]*\.\\\//y,
265
+ // The tokens of the SPEC lexical state that are not literals, with their kinds.
266
+ const SPEC_PATTERNS: [string, RegExp[]][] = [
267
+ ["_BM0", [BEGIN_MODULE]],
268
+ ["SEPARATOR", [/----+/y]],
269
+ ["END_MODULE", [/====+/y]],
270
+ ["NUMBER_LITERAL", [/[0-9]+|\\[oO][0-7]+|\\[bB][01]+|\\[hH][0-9a-fA-F]+/y]],
271
+ ["STRING_LITERAL", [/"(?:[^"\n\r\\]|\\[ntrf\\"])*"/y]],
272
+ ["BAND", BAND_PATTERNS],
273
+ ["BOR", BOR_PATTERNS],
274
+ ["IDENTIFIER", IDENTIFIER_PATTERNS],
275
+ ["ProofStepLexeme", [/<[0-9]+>\w+/y]],
276
+ ["ProofImplicitStepLexeme", [/<[+*]>[A-Za-z0-9]+/y]],
277
+ ["ProofStepDotLexeme", [new RegExp(String.raw`${STEP}\w+\.+`, "y")]],
278
+ ["BareLevelLexeme", [new RegExp(STEP, "y")]],
279
+ ["UnnumberedStepLexeme", [new RegExp(String.raw`${STEP}(?:\*\.*|-\.*|\.+)`, "y")]],
347
280
  ];
348
281
 
349
- const IDENTIFIER_PATTERNS = [
350
- /_[A-Za-z0-9_]*[A-Za-z][A-Za-z0-9_]*/y,
351
- /[0-9][A-Za-z0-9_]*[A-Za-z][A-Za-z0-9_]*/y,
352
- /[WS](?:[a-zA-EG-Z_0-9][A-Za-z0-9_]*)?/y,
353
- /[WS]F(?:[A-Za-z0-9][A-Za-z0-9_]*)?/y,
354
- /[a-zA-RT-VX-Z][A-Za-z0-9_]*/y,
355
- /[@ℕℤℝ]/y,
282
+ // The tokens of the DEFAULT and PRAGMA lexical states, which skip every other character.
283
+ const DEFAULT_PATTERNS: [string, RegExp[]][] = [
284
+ ["BEGIN_PRAGMA", [/--->/y]],
285
+ ["_BM1", [BEGIN_MODULE]],
356
286
  ];
357
-
358
- const SPEC_PATTERNS: [number, RegExp[]][] = [
359
- [_BM0, [BEGIN_MODULE]],
360
- [SEPARATOR, [/-{4,}/y]],
361
- [END_MODULE, [/={4,}/y]],
362
- [NUMBER_LITERAL, [/[0-9]+|\\[oO][0-7]+|\\[bB][01]+|\\[hH][0-9a-fA-F]+/y]],
363
- [STRING_LITERAL, [/"(?:[^"\n\r\\]|\\[ntrf\\"])*"/y]],
364
- [BAND, BAND_PATTERNS],
365
- [BOR, BOR_PATTERNS],
366
- [IDENTIFIER, IDENTIFIER_PATTERNS],
367
- [PROOF_STEP, [/<[0-9]+>[A-Za-z0-9_]+/y]],
368
- [PROOF_IMPLICIT_STEP, [/<[+*]>[A-Za-z0-9]+/y]],
369
- [PROOF_STEP_DOT, [/<(?:[0-9]+|\+|\*)>[A-Za-z0-9_]+\.+/y]],
370
- [BARE_LEVEL, [/<(?:[0-9]+|\+|\*)>/y]],
371
- [UNNUMBERED_STEP, [/<(?:[0-9]+|\+|\*)>(?:\*\.*|-\.*|\.+)/y]],
287
+ const PRAGMA_PATTERNS: [string, RegExp[]][] = [
288
+ ["NUMBER", [/[0-9]+/y]],
289
+ ["_BM2", [BEGIN_MODULE]],
290
+ ["BAND", BAND_PATTERNS],
291
+ ["BOR", BOR_PATTERNS],
292
+ ["IDENTIFIER", IDENTIFIER_PATTERNS],
372
293
  ];
373
294
 
374
- const PRAGMA_PATTERNS: [number, RegExp[]][] = [
375
- [NUMBER, [/[0-9]+/y]],
376
- [_BM2, [BEGIN_MODULE]],
377
- [BAND, BAND_PATTERNS],
378
- [BOR, BOR_PATTERNS],
379
- [IDENTIFIER, IDENTIFIER_PATTERNS],
380
- ];
295
+ const getMatchLength = (pattern: RegExp, source: string, offset: number) => {
296
+ pattern.lastIndex = offset;
297
+ return pattern.exec(source)?.[0].length ?? 0;
298
+ };
381
299
 
382
- const getPatternLength = (patterns: RegExp[], source: string, index: number) =>
383
- Math.max(
384
- 0,
385
- ...patterns.map((pattern) => {
386
- pattern.lastIndex = index;
387
- return pattern.exec(source)?.[0].length ?? 0;
388
- }),
389
- );
300
+ // The longest match at offset, the earliest kind among matches of that length.
301
+ const matchPatterns = (patterns: [string, RegExp[]][], source: string, offset: number) =>
302
+ patterns
303
+ .map(([name, regexes]) => ({
304
+ kind: getKind(name),
305
+ length: Math.max(...regexes.map((regex) => getMatchLength(regex, source, offset))),
306
+ }))
307
+ .reduce(
308
+ (best, match) =>
309
+ match.length > best.length || (match.length === best.length && match.kind < best.kind)
310
+ ? match
311
+ : best,
312
+ { kind: -1, length: 0 },
313
+ );
390
314
 
391
- // The longest token at index, the lowest kind winning a tie, as JavaCC's token manager picks it.
392
- const matchToken = (
393
- source: string,
394
- index: number,
395
- patterns: [number, RegExp[]][],
396
- isSpec: boolean,
397
- ) => {
398
- const candidates: [number, number][] = patterns.map(([kind, kindPatterns]) => [
399
- kind,
400
- getPatternLength(kindPatterns, source, index),
401
- ]);
402
- if (isSpec) {
403
- for (const [kind, image] of LITERALS_BY_FIRST_CHARACTER.get(source[index]) ?? []) {
404
- if (source.startsWith(image, index)) candidates.push([kind, image.length]);
405
- }
406
- }
407
- return candidates.reduce<[number, number]>(
408
- (best, candidate) =>
409
- candidate[1] > best[1] ||
410
- (candidate[1] === best[1] && candidate[1] > 0 && candidate[0] < best[0])
411
- ? candidate
412
- : best,
413
- [EOF, 0],
315
+ const matchSpecToken = (source: string, offset: number) => {
316
+ const literalLength = LITERAL_LENGTHS.find((length) =>
317
+ LITERALS.has(source.slice(offset, offset + length)),
414
318
  );
319
+ const pattern = matchPatterns(SPEC_PATTERNS, source, offset);
320
+ if (literalLength === undefined) return pattern;
321
+ const literal = {
322
+ kind: LITERALS.get(source.slice(offset, offset + literalLength)) ?? -1,
323
+ length: literalLength,
324
+ };
325
+ return literal.length > pattern.length ||
326
+ (literal.length === pattern.length && literal.kind < pattern.kind)
327
+ ? literal
328
+ : pattern;
415
329
  };
416
330
 
417
- // The length of input at index that some token could still begin with, when no token matches it.
418
- const getViablePrefixLength = (source: string, index: number) => {
419
- if (source[index] === "?") return 1;
420
- if (source[index] !== '"') return 0;
421
- const stringPrefix = /"(?:[^"\n\r\\]|\\[ntrf\\"])*\\?/y;
422
- stringPrefix.lastIndex = index;
423
- return stringPrefix.exec(source)?.[0].length ?? 0;
424
- };
331
+ export const getTabStop = (column: number) => column + 8 - (column % 8);
425
332
 
426
- const ESCAPES = new Map([
427
- ["\b", "\\b"],
428
- ["\t", "\\t"],
429
- ["\n", "\\n"],
430
- ["\f", "\\f"],
431
- ["\r", "\\r"],
432
- ['"', '\\"'],
433
- ["'", "\\'"],
434
- ["\\", "\\\\"],
435
- ["\0", ""],
436
- ]);
437
-
438
- // Escapes text as JavaCC's error messages print it.
439
- export const escapeImage = (text: string) =>
440
- text
441
- .split("")
442
- .map((unit) => {
443
- const code = unit.charCodeAt(0);
444
- return (
445
- ESCAPES.get(unit) ??
446
- (code < 0x20 || code > 0x7e ? `\\u${code.toString(16).padStart(4, "0")}` : unit)
447
- );
448
- })
449
- .join("");
450
-
451
- // Reads SANY tokens one at a time, so a lexical error surfaces only when the parser reaches it.
452
- export const createSanyLexer = (source: string): SanyLexer => {
333
+ // SimpleCharStream.UpdateLineColumn: the line and column of each character, with tabs stopping
334
+ // at multiples of 8.
335
+ const getPositions = (source: string) => {
453
336
  const lines: number[] = [];
454
337
  const columns: number[] = [];
455
- const position = { line: 1, column: 0, isAfterCarriageReturn: false, isAfterLineFeed: false };
456
- for (const character of source.split("")) {
457
- position.column++;
458
- if (position.isAfterLineFeed) {
459
- position.isAfterLineFeed = false;
460
- position.line++;
461
- position.column = 1;
462
- } else if (position.isAfterCarriageReturn) {
463
- position.isAfterCarriageReturn = false;
464
- if (character === "\n") position.isAfterLineFeed = true;
465
- else {
466
- position.line++;
467
- position.column = 1;
338
+ const state = { line: 1, column: 0, isAfterCr: false, isAfterLf: false };
339
+ for (const character of source) {
340
+ for (let unit = 0; unit < character.length; unit++) {
341
+ state.column++;
342
+ if (state.isAfterLf) {
343
+ state.isAfterLf = false;
344
+ state.line++;
345
+ state.column = 1;
346
+ } else if (state.isAfterCr) {
347
+ state.isAfterCr = false;
348
+ if (character === "\n") state.isAfterLf = true;
349
+ else {
350
+ state.line++;
351
+ state.column = 1;
352
+ }
468
353
  }
354
+ if (character === "\r") state.isAfterCr = true;
355
+ else if (character === "\n") state.isAfterLf = true;
356
+ else if (character === "\t") state.column = getTabStop(state.column - 1);
357
+ lines.push(state.line);
358
+ columns.push(state.column);
469
359
  }
470
- if (character === "\r") position.isAfterCarriageReturn = true;
471
- if (character === "\n") position.isAfterLineFeed = true;
472
- if (character === "\t")
473
- position.column = position.column - 1 + (8 - ((position.column - 1) % 8));
474
- lines.push(position.line);
475
- columns.push(position.column);
476
360
  }
477
- const cursor = { state: LexicalState.Default, index: 0, commentDepth: 0 };
478
- const createToken = (kind: number, begin: number, end: number): SanyToken => ({
361
+ return { lines, columns };
362
+ };
363
+
364
+ // The characters of a string literal that a string literal could still continue.
365
+ const STRING_PREFIX = /"(?:[^"\n\r\\]|\\[ntrf\\"])*\\?/y;
366
+ const LITERAL_PREFIXES = new Set(
367
+ [...LITERALS.keys()].flatMap((image) =>
368
+ Array.from({ length: image.length - 1 }, (_, index) => image.slice(0, index + 1)),
369
+ ),
370
+ );
371
+
372
+ // The characters JavaCC's token manager reads at offset while some token could still continue.
373
+ const getTokenPrefix = (source: string, offset: number) => {
374
+ let length = 0;
375
+ while (LITERAL_PREFIXES.has(source.slice(offset, offset + length + 1))) length++;
376
+ return source.slice(
377
+ offset,
378
+ offset + Math.max(length, getMatchLength(STRING_PREFIX, source, offset)),
379
+ );
380
+ };
381
+
382
+ // A lazy token stream over source, which returns SANY's message for the TokenMgrError JavaCC's
383
+ // token manager throws at a character no token continues with, or at the end of the file in a
384
+ // comment.
385
+ export const createSanyLexer = (source: string) => {
386
+ const positions = getPositions(source);
387
+ const state = { offset: 0, mode: "DEFAULT" };
388
+ const createToken = (kind: number, start: number, length: number): SanyToken => ({
479
389
  kind,
480
- image: source.slice(begin, end),
481
- beginLine: lines[begin],
482
- beginColumn: columns[begin],
483
- endLine: lines[end - 1],
484
- endColumn: columns[end - 1],
390
+ image: source.slice(start, start + length),
391
+ beginLine: positions.lines[start] ?? 0,
392
+ beginColumn: positions.columns[start] ?? 0,
485
393
  next: undefined,
486
394
  });
487
- const throwLexicalError = (begin: number, after: string, killer: number | undefined) => {
488
- if (killer !== undefined && killer + 1 < source.length) {
489
- throw new TlaParseError(
490
- `Lexical error at line ${lines[killer]}, column ${columns[killer]}. Encountered: "${escapeImage(source[killer])}" (${source.charCodeAt(killer)}), after : "${escapeImage(after)}"`,
491
- );
395
+ // JavaCC places the end of the file at the file's last character.
396
+ const getEndOfFile = () => createToken(getKind("EOF"), source.length - 1, 0);
397
+ // The lexer read the characters from start to end, and TLAplusParser.parse reports the end of
398
+ // the file in a token that starts on an earlier line as an open comment.
399
+ const getLexicalError = (start: number, end: number, after: string) => {
400
+ const index = end - 1;
401
+ const line = positions.lines[index];
402
+ const column = positions.columns[index];
403
+ if (end < source.length) return formatLexicalError(line, column, source[index], after);
404
+ if (positions.lines[start] !== line) {
405
+ return `Lexical {error: EOF reached, possibly open comment starting around line ${positions.lines[start]}`;
492
406
  }
493
- const last = source.length - 1;
494
- const isLineEnd = source[last] === "\n" || source[last] === "\r";
495
- const message = `Lexical error at line ${isLineEnd ? lines[last] + 1 : lines[last]}, column ${isLineEnd ? 0 : columns[last] + 1}. Encountered: <EOF> after : "${escapeImage(after)}"`;
496
- throw new TlaParseError(
497
- lines[begin] === lines[last]
498
- ? message
499
- : `Lexical {error: EOF reached, possibly open comment starting around line ${lines[begin]}`,
500
- );
407
+ return source[index] === "\n" || source[index] === "\r"
408
+ ? formatLexicalError(line + 1, 0, undefined, after)
409
+ : formatLexicalError(line, column + 1, undefined, after);
501
410
  };
502
- const readComment = (begin: number): SanyToken | undefined => {
503
- for (;;) {
504
- if (cursor.index >= source.length) return throwLexicalError(begin, "", undefined);
505
- const opening = source.startsWith("(*.", cursor.index)
506
- ? 3
507
- : source.startsWith("(*", cursor.index)
508
- ? 2
509
- : 0;
510
- if (cursor.state === LexicalState.InEndOfLineComment) {
511
- const lineEnd = source.startsWith("\r\n", cursor.index)
512
- ? 2
513
- : source[cursor.index] === "\n" || source[cursor.index] === "\r"
514
- ? 1
515
- : 0;
516
- if (lineEnd > 0) {
517
- cursor.index += lineEnd;
518
- cursor.state = LexicalState.Spec;
411
+ // A comment is a chain of MORE matches, which restarts after each nested comment bracket, and the
412
+ // end of the file right after a bracket ends the token stream.
413
+ const skipComment = () => {
414
+ const start = state.offset;
415
+ if (source.startsWith("\\*", start)) {
416
+ const end = source.slice(start).search(/[\n\r]/);
417
+ if (end === -1) return getLexicalError(start, source.length, "");
418
+ state.offset += end + 1;
419
+ return undefined;
420
+ }
421
+ let depth = 0;
422
+ let chainStart = start;
423
+ let offset = start;
424
+ while (offset < source.length) {
425
+ if (source.startsWith("(*", offset)) {
426
+ depth++;
427
+ offset += 2;
428
+ if (depth > 1) chainStart = offset;
429
+ } else if (source.startsWith("*)", offset)) {
430
+ depth--;
431
+ offset += 2;
432
+ chainStart = offset;
433
+ if (depth === 0) {
434
+ state.offset = offset;
519
435
  return undefined;
520
436
  }
521
- } else if (opening > 0) {
522
- cursor.index += opening;
523
- cursor.commentDepth++;
524
- cursor.state = LexicalState.Embedded;
525
- return undefined;
526
- } else if (source.startsWith("*)", cursor.index)) {
527
- cursor.index += 2;
528
- if (cursor.state === LexicalState.InComment) cursor.state = LexicalState.Spec;
529
- else {
530
- cursor.commentDepth--;
531
- if (cursor.commentDepth === 0) cursor.state = LexicalState.InComment;
532
- }
533
- return undefined;
534
- }
535
- cursor.index++;
437
+ } else offset++;
536
438
  }
439
+ state.offset = offset;
440
+ return chainStart === source.length ? undefined : getLexicalError(chainStart, offset, "");
537
441
  };
538
- const getNextToken = (): SanyToken => {
539
- for (;;) {
540
- if (cursor.index >= source.length) {
541
- const last = source.length - 1;
542
- return {
543
- kind: EOF,
544
- image: "",
545
- beginLine: lines[last] ?? 0,
546
- beginColumn: columns[last] ?? 0,
547
- endLine: lines[last] ?? 0,
548
- endColumn: columns[last] ?? 0,
549
- next: undefined,
550
- };
442
+ const getSkippingToken = (patterns: [string, RegExp[]][]): SanyToken => {
443
+ while (state.offset < source.length) {
444
+ const match = matchPatterns(patterns, source, state.offset);
445
+ if (match.length > 0) {
446
+ const token = createToken(match.kind, state.offset, match.length);
447
+ state.offset += match.length;
448
+ if (KIND_NAMES[match.kind].startsWith("_BM")) state.mode = "SPEC";
449
+ if (KIND_NAMES[match.kind] === "BEGIN_PRAGMA") state.mode = "PRAGMA";
450
+ return token;
551
451
  }
552
- const begin = cursor.index;
553
- if (cursor.state === LexicalState.Default) {
554
- const pragma = source.startsWith("--->", cursor.index) ? 4 : 0;
555
- BEGIN_MODULE.lastIndex = cursor.index;
556
- const module = BEGIN_MODULE.exec(source)?.[0].length ?? 0;
557
- if (module > 0 && module >= pragma) {
558
- cursor.index += module;
559
- cursor.state = LexicalState.Spec;
560
- return createToken(_BM1, begin, cursor.index);
561
- }
562
- if (pragma > 0) {
563
- cursor.index += pragma;
564
- cursor.state = LexicalState.Pragma;
565
- return createToken(BEGIN_PRAGMA, begin, cursor.index);
566
- }
567
- cursor.index++;
568
- continue;
569
- }
570
- if (cursor.state === LexicalState.Pragma) {
571
- const [kind, length] = matchToken(source, cursor.index, PRAGMA_PATTERNS, false);
572
- if (length === 0) {
573
- cursor.index++;
574
- continue;
575
- }
576
- cursor.index += length;
577
- if (kind === _BM2) cursor.state = LexicalState.Spec;
578
- return createToken(kind, begin, cursor.index);
579
- }
580
- if (cursor.state !== LexicalState.Spec) {
581
- readComment(begin);
582
- continue;
583
- }
584
- if (" \t\n\r".includes(source[cursor.index])) {
585
- cursor.index++;
586
- continue;
587
- }
588
- if (source.startsWith("(*", cursor.index) || source.startsWith("\\*", cursor.index)) {
589
- cursor.state =
590
- source[cursor.index] === "(" ? LexicalState.InComment : LexicalState.InEndOfLineComment;
591
- cursor.index += source.startsWith("(*.", cursor.index) ? 3 : 2;
592
- readComment(begin);
593
- continue;
594
- }
595
- const [kind, length] = matchToken(source, cursor.index, SPEC_PATTERNS, true);
596
- if (length === 0) {
597
- const viable = getViablePrefixLength(source, cursor.index);
598
- const killer = cursor.index + viable;
599
- if (killer < source.length) {
600
- return throwLexicalError(
601
- begin,
602
- killer + 1 < source.length || viable === 0
603
- ? source.slice(begin, killer)
604
- : source.slice(begin, killer + 1),
605
- killer,
606
- );
452
+ state.offset++;
453
+ }
454
+ return getEndOfFile();
455
+ };
456
+ const getNextToken = (): SanyToken | string => {
457
+ if (state.mode === "DEFAULT") return getSkippingToken(DEFAULT_PATTERNS);
458
+ if (state.mode === "PRAGMA") return getSkippingToken(PRAGMA_PATTERNS);
459
+ while (state.offset < source.length) {
460
+ const offset = state.offset;
461
+ if (" \t\n\r".includes(source[offset])) state.offset++;
462
+ else if (source.startsWith("(*", offset) || source.startsWith("\\*", offset)) {
463
+ const error = skipComment();
464
+ if (error !== undefined) return error;
465
+ } else {
466
+ const match = matchSpecToken(source, offset);
467
+ if (match.length === 0) {
468
+ // JavaCC reads on while a token could continue, and one more character.
469
+ const prefix = getTokenPrefix(source, offset);
470
+ const end = Math.min(offset + prefix.length + 1, source.length);
471
+ const read = source.slice(offset, end);
472
+ const after = end < source.length ? prefix : read.length > 1 ? read : "";
473
+ return getLexicalError(offset, end, after);
607
474
  }
608
- return throwLexicalError(begin, viable <= 1 ? "" : source.slice(begin), undefined);
475
+ const token = createToken(match.kind, offset, match.length);
476
+ state.offset += match.length;
477
+ return token;
609
478
  }
610
- cursor.index += length;
611
- return createToken(kind, begin, cursor.index);
612
479
  }
480
+ return getEndOfFile();
613
481
  };
614
- return { getNextToken };
482
+ return getNextToken;
483
+ };
484
+
485
+ // JavaCC's TokenMgrError.addEscapes, which escapes the characters of a lexical error's message.
486
+ const LEXEME_ESCAPES = new Map([
487
+ ["\b", "\\b"],
488
+ ["\t", "\\t"],
489
+ ["\n", "\\n"],
490
+ ["\f", "\\f"],
491
+ ["\r", "\\r"],
492
+ ['"', '\\"'],
493
+ ["'", "\\'"],
494
+ ["\\", "\\\\"],
495
+ ]);
496
+
497
+ export const escapeLexeme = (text: string) =>
498
+ text.replace(
499
+ // eslint-disable-next-line no-control-regex
500
+ /[\0-\x1f"'\\\x7f-\uffff]/g,
501
+ (character) =>
502
+ LEXEME_ESCAPES.get(character) ??
503
+ (character === "\0" ? "" : `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`),
504
+ );
505
+
506
+ // TokenMgrError.LexicalError: the character the lexer stopped at, or the end of the file, and the
507
+ // characters of the token it had read.
508
+ export const formatLexicalError = (
509
+ line: number,
510
+ column: number,
511
+ character: string | undefined,
512
+ after: string,
513
+ ) =>
514
+ `Lexical error at line ${line}, column ${column}. Encountered: ${character === undefined ? "<EOF> " : `"${escapeLexeme(character)}" (${character.charCodeAt(0)}), `}after : "${escapeLexeme(after)}"`;
515
+
516
+ // TLC 1.7.4's lexer, which JavaCC generated without UNICODE_INPUT, reads a character past U+00FF
517
+ // outside a comment by its low byte, and stops at one whose low byte is below 0x80.
518
+ export const getSany174LexicalError = (source: string) => {
519
+ const readToken = createSanyLexer(source);
520
+ for (
521
+ let token = readToken();
522
+ typeof token !== "string" && token.kind !== getKind("EOF");
523
+ token = readToken()
524
+ ) {
525
+ const index = token.image.split("").findIndex((character) => {
526
+ const code = character.charCodeAt(0);
527
+ return code > 0xff && (code & 0xff) < 0x80;
528
+ });
529
+ if (index === -1) continue;
530
+ const column = token.image
531
+ .slice(1, index + 1)
532
+ .split("")
533
+ .reduce(
534
+ (previous, character) => (character === "\t" ? getTabStop(previous) : previous + 1),
535
+ token.beginColumn,
536
+ );
537
+ return formatLexicalError(
538
+ token.beginLine,
539
+ column,
540
+ token.image[index],
541
+ token.image.slice(0, index),
542
+ );
543
+ }
544
+ return undefined;
615
545
  };