tla-ts 0.0.3-dev.e5b05b9 → 0.0.3-dev.e8bef80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/cli.cjs +1 -1
  2. package/dist/cli.js +1 -1
  3. package/dist/index.cjs +1 -1
  4. package/dist/index.d.cts +150 -48
  5. package/dist/index.d.ts +150 -48
  6. package/dist/index.js +1 -1
  7. package/dist/parallel-worker.cjs +1 -1
  8. package/dist/parallel-worker.js +2 -2
  9. package/dist/parallel.cjs +1 -1
  10. package/dist/parallel.js +2 -2
  11. package/dist/run-cli.cjs +1351 -22
  12. package/dist/run-cli.js +1352 -23
  13. package/dist/transfer.cjs +83 -43
  14. package/dist/transfer.js +83 -43
  15. package/package.json +5 -1
  16. package/src/ast.ts +28 -4
  17. package/src/bags.ts +4 -7
  18. package/src/builtins.ts +29 -17
  19. package/src/bundled-modules.ts +3 -3
  20. package/src/canonical.ts +4 -3
  21. package/src/check.ts +1317 -428
  22. package/src/cli.ts +8 -1
  23. package/src/codegen.ts +68 -21
  24. package/src/config.ts +226 -207
  25. package/src/environment.ts +221 -75
  26. package/src/errors.ts +130 -15
  27. package/src/evaluate.ts +713 -259
  28. package/src/exit-codes.ts +6 -24
  29. package/src/fingerprints.ts +1 -2
  30. package/src/index.ts +1 -0
  31. package/src/instances.ts +3 -1
  32. package/src/java-overrides.ts +104 -25
  33. package/src/levels.ts +6 -1
  34. package/src/lexer.ts +122 -29
  35. package/src/live-check.ts +204 -148
  36. package/src/liveness.ts +191 -122
  37. package/src/obligation-expression.ts +2 -2
  38. package/src/obligation-source.ts +2 -0
  39. package/src/parallel-worker.ts +6 -1
  40. package/src/parallel.ts +57 -6
  41. package/src/parser.ts +327 -147
  42. package/src/pcal-ast.ts +770 -0
  43. package/src/pcal-parse.ts +1523 -0
  44. package/src/pcal-tokenize.ts +516 -0
  45. package/src/pcal-translate.ts +1230 -0
  46. package/src/pcal-validate.ts +454 -0
  47. package/src/randomization.ts +24 -7
  48. package/src/resolve.ts +252 -87
  49. package/src/run-cli.ts +1162 -200
  50. package/src/runtime.ts +16 -0
  51. package/src/sany-builtins.ts +147 -0
  52. package/src/sany-generator.ts +3128 -0
  53. package/src/sany-levels.ts +1208 -0
  54. package/src/sany-lexer.ts +410 -480
  55. package/src/sany-linter.ts +122 -0
  56. package/src/sany-modules.ts +1398 -0
  57. package/src/sany-nodes.ts +463 -0
  58. package/src/sany-parse-error.ts +2033 -3015
  59. package/src/sany-report.ts +356 -0
  60. package/src/sany-selectors.ts +738 -0
  61. package/src/sany-source.ts +248 -0
  62. package/src/sany-xml.ts +1017 -0
  63. package/src/spec-description.ts +25 -5
  64. package/src/standard-modules.ts +172 -129
  65. package/src/state-graph.ts +29 -6
  66. package/src/subexpressions.ts +121 -26
  67. package/src/successors.ts +141 -173
  68. package/src/symmetry.ts +168 -50
  69. package/src/tableau.ts +3 -1
  70. package/src/tlc-ext.ts +13 -17
  71. package/src/tlc-messages.ts +329 -0
  72. package/src/tlc-module.ts +49 -21
  73. package/src/tlc-modules.ts +0 -4
  74. package/src/tlc-order.ts +55 -9
  75. package/src/tlc-random.ts +28 -3
  76. package/src/tlc-replay.ts +655 -0
  77. package/src/tlc-version.ts +18 -0
  78. package/src/transfer.ts +15 -3
  79. package/src/values.ts +664 -317
package/src/exit-codes.ts CHANGED
@@ -26,19 +26,7 @@ export const setExitCode = <Thrown>(error: Thrown, code: number) => {
26
26
 
27
27
  export const getExitCode = (error: Error) => exitCodes.get(error);
28
28
 
29
- const ownExitCodes = new WeakMap<Error, number>();
30
-
31
- // The exit status of the error code TLC gives an EvalException, which TLC uses only where it
32
- // doesn't report the error again as a nested expression.
33
- export const setOwnExitCode = <Thrown extends Error>(error: Thrown, code: number) => {
34
- ownExitCodes.set(error, code);
35
- return error;
36
- };
37
-
38
- export const getOwnExitCode = (error: unknown) =>
39
- error instanceof Error ? ownExitCodes.get(error) : undefined;
40
-
41
- // TLC exits with 255 when it fails to process or evaluate a liveness property.
29
+ // TLC exits with 255 when it fails to process or evaluate a liveness property, in any phase.
42
30
  const withErrorExitCode = <Result>(run: () => Result) => {
43
31
  try {
44
32
  return run();
@@ -53,17 +41,11 @@ export const withLivenessExitCode = (
53
41
  const checker = withErrorExitCode(create);
54
42
  return (
55
43
  checker && {
56
- // TLC replays the initial states to report an error, but skips the liveness check of a
57
- // state it has already seen, so it exits with the status of the error's own code.
58
- addInitialState: (index, state) => {
59
- try {
60
- checker.addInitialState(index, state);
61
- } catch (error) {
62
- throw setExitCode(error, getOwnExitCode(error) ?? EXIT_STATUS.error);
63
- }
64
- },
65
- addState: (index, state, successors) =>
66
- withErrorExitCode(() => checker.addState(index, state, successors)),
44
+ branches: checker.branches,
45
+ addInitialState: (index, state, tracing) =>
46
+ withErrorExitCode(() => checker.addInitialState(index, state, tracing)),
47
+ addState: (index, state, successors, tracing) =>
48
+ withErrorExitCode(() => checker.addState(index, state, successors, tracing)),
67
49
  findViolation: () => withErrorExitCode(() => checker.findViolation()),
68
50
  getViolatedProperties: (path, cycleStart) =>
69
51
  withErrorExitCode(() => checker.getViolatedProperties(path, cycleStart)),
@@ -1,4 +1,4 @@
1
- import { assertFingerprintable, getValueFingerprint, getValueHash, type Value } from "./values";
1
+ import { getValueFingerprint, getValueHash, type Value } from "./values";
2
2
 
3
3
  const EMPTY = 0;
4
4
  const INITIAL_CAPACITY = 1 << 12;
@@ -20,7 +20,6 @@ export const writeStateFingerprint = (state: Value[], fingerprint: Int32Array) =
20
20
  first = Math.imul(first ^ getValueHash(state[index]), 0x01000193);
21
21
  second = Math.imul(second ^ getValueFingerprint(state[index]), 0x5bd1e995);
22
22
  }
23
- assertFingerprintable(state);
24
23
  fingerprint[0] = mix(first) || 1;
25
24
  fingerprint[1] = mix(second);
26
25
  };
package/src/index.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  export type {
2
2
  CheckOptions,
3
+ CheckProgress,
3
4
  CheckResult,
4
5
  CheckStatus,
5
6
  EvaluationFailure,
package/src/instances.ts CHANGED
@@ -183,6 +183,8 @@ export const mapChildren = (
183
183
  function: plain(expression.function),
184
184
  args: expression.args.map(plain),
185
185
  };
186
+ case "Bracket":
187
+ return { ...expression, args: expression.args.map(plain) };
186
188
  case "FieldAccess":
187
189
  return { ...expression, record: plain(expression.record) };
188
190
  case "Product":
@@ -705,7 +707,7 @@ const expandLetInstances = (
705
707
  const index = position === -1 ? current.length : position;
706
708
  return [...current.slice(0, index), ...instantiate(instance, inner), ...current.slice(index)];
707
709
  }, mapped.definitions);
708
- return { kind: "Let", definitions, body: mapped.body, location: mapped.location };
710
+ return { ...mapped, definitions };
709
711
  };
710
712
 
711
713
  // A named theorem's statement is linked as its definition.
@@ -12,9 +12,20 @@ interface PoolEntry {
12
12
  interface JavaMethod {
13
13
  name: string;
14
14
  arity: number;
15
+ // Method#toString, which TLC prints as the signature of an override.
16
+ signature: string;
15
17
  code: Uint8Array | undefined;
16
18
  }
17
19
 
20
+ // A public static method of the class named after a module, which TLC reports as it loads the
21
+ // class, with whether it overrides an operator.
22
+ export interface JavaOverride {
23
+ module: string;
24
+ operator: string;
25
+ signature: string;
26
+ isMatched: boolean;
27
+ }
28
+
18
29
  interface ObjectReference {
19
30
  className: string;
20
31
  value: Expression | undefined;
@@ -44,12 +55,44 @@ const createReader = (bytes: Uint8Array) => {
44
55
  };
45
56
  };
46
57
 
47
- const getParameterCount = (descriptor: string) => {
48
- const parameters = descriptor.slice(1, descriptor.indexOf(")"));
49
- return [...parameters.matchAll(/\[*(?:L[^;]*;|[BCDFIJSZ])/g)].length;
50
- };
58
+ const PRIMITIVE_TYPES = new Map([
59
+ ["B", "byte"],
60
+ ["C", "char"],
61
+ ["D", "double"],
62
+ ["F", "float"],
63
+ ["I", "int"],
64
+ ["J", "long"],
65
+ ["S", "short"],
66
+ ["Z", "boolean"],
67
+ ["V", "void"],
68
+ ]);
69
+
70
+ // The modifiers Method#toString prints, in the order of Modifier#toString.
71
+ const METHOD_MODIFIERS: [number, string][] = [
72
+ [0x0001, "public"],
73
+ [0x0004, "protected"],
74
+ [0x0002, "private"],
75
+ [0x0400, "abstract"],
76
+ [0x0008, "static"],
77
+ [0x0010, "final"],
78
+ [0x0020, "synchronized"],
79
+ [0x0100, "native"],
80
+ [0x0800, "strictfp"],
81
+ ];
82
+
83
+ const getTypeNames = (descriptor: string) =>
84
+ [...descriptor.matchAll(/(\[*)(?:L([^;]*);|([BCDFIJSZV]))/g)].map(
85
+ (match) =>
86
+ `${match[2]?.replaceAll("/", ".") ?? PRIMITIVE_TYPES.get(match[3])}${"[]".repeat(match[1].length)}`,
87
+ );
51
88
 
52
- // The public static methods of a class file and their bytecode, as java.lang.Class lists them.
89
+ const getClassName = (pool: PoolEntry[], index: number) => pool[pool[index].first].text;
90
+
91
+ const getJavaClassName = (pool: PoolEntry[], index: number) =>
92
+ getClassName(pool, index).replaceAll("/", ".");
93
+
94
+ // The public static methods of a class file and their bytecode, in the order of the class file.
95
+ // Class#getDeclaredMethods orders them by where HotSpot allocated the symbols of their names.
53
96
  const readMethods = (bytes: Uint8Array) => {
54
97
  const reader = createReader(bytes);
55
98
  if (reader.readUint32() !== CLASS_MAGIC)
@@ -80,7 +123,7 @@ const readMethods = (bytes: Uint8Array) => {
80
123
  }
81
124
  }
82
125
  reader.readUint16();
83
- reader.readUint16();
126
+ const className = getJavaClassName(pool, reader.readUint16());
84
127
  reader.readUint16();
85
128
  reader.readBytes(reader.readUint16() * 2);
86
129
  for (let fields = reader.readUint16(); fields > 0; fields--) {
@@ -95,23 +138,38 @@ const readMethods = (bytes: Uint8Array) => {
95
138
  const access = reader.readUint16();
96
139
  const name = pool[reader.readUint16()].text;
97
140
  const descriptor = pool[reader.readUint16()].text;
98
- const method: JavaMethod = { name, arity: getParameterCount(descriptor), code: undefined };
99
- for (let attributes = reader.readUint16(); attributes > 0; attributes--) {
100
- const attributeName = pool[reader.readUint16()].text;
101
- const attribute = reader.readBytes(reader.readUint32());
102
- if (attributeName === "Code") {
103
- const length = new DataView(attribute.buffer, attribute.byteOffset + 4, 4).getUint32(0);
104
- method.code = attribute.subarray(8, 8 + length);
105
- }
106
- }
141
+ const parametersEnd = descriptor.indexOf(")");
142
+ const parameters = getTypeNames(descriptor.slice(1, parametersEnd));
143
+ const attributes = Array.from({ length: reader.readUint16() }, () => ({
144
+ name: pool[reader.readUint16()].text,
145
+ reader: createReader(reader.readBytes(reader.readUint32())),
146
+ }));
107
147
  const isOverride = (access & ACCESS_PUBLIC) !== 0 && (access & ACCESS_STATIC) !== 0;
108
- if (isOverride) methods.push(method);
148
+ if (!isOverride) continue;
149
+ const code = attributes.find((attribute) => attribute.name === "Code")?.reader;
150
+ // A Code attribute holds max_stack and max_locals before the length of the bytecode.
151
+ code?.readBytes(4);
152
+ const thrown = attributes.find((attribute) => attribute.name === "Exceptions")?.reader;
153
+ const exceptions = thrown
154
+ ? Array.from({ length: thrown.readUint16() }, () =>
155
+ getJavaClassName(pool, thrown.readUint16()),
156
+ )
157
+ : [];
158
+ const modifiers = METHOD_MODIFIERS.filter(([flag]) => (access & flag) !== 0).map(
159
+ ([, modifier]) => modifier,
160
+ );
161
+ const returnType = getTypeNames(descriptor.slice(parametersEnd + 1))[0];
162
+ const throwsClause = exceptions.length === 0 ? "" : ` throws ${exceptions.join(",")}`;
163
+ methods.push({
164
+ name,
165
+ arity: parameters.length,
166
+ code: code?.readBytes(code.readUint32()),
167
+ signature: `${modifiers.join(" ")} ${returnType} ${className}.${name}(${parameters.join(",")})${throwsClause}`,
168
+ });
109
169
  }
110
170
  return { pool, methods };
111
171
  };
112
172
 
113
- const getClassName = (pool: PoolEntry[], index: number) => pool[pool[index].first].text;
114
-
115
173
  const getMember = (pool: PoolEntry[], index: number) => {
116
174
  const reference = pool[index];
117
175
  const nameAndType = pool[reference.second];
@@ -220,16 +278,29 @@ const getReturnedLiteral = (
220
278
  throw unsupported();
221
279
  };
222
280
 
223
- // TLC loads a class named after each module and replaces every operator the module defines
224
- // with the public static method of the same name and arity.
225
- export const overrideDefinitions = (module: Module, bytes: Uint8Array | undefined): Module => {
226
- if (!bytes) return module;
281
+ // TLC loads a class named after each module and replaces each operator of the module that the
282
+ // root module sees with the public static method of the same name and arity. It reports whether
283
+ // each method matched an operator.
284
+ export const overrideDefinitions = (
285
+ module: Module,
286
+ bytes: Uint8Array | undefined,
287
+ isRoot: boolean,
288
+ ): { module: Module; overrides: JavaOverride[] } => {
289
+ if (!bytes) return { module, overrides: [] };
227
290
  const classFile = readMethods(bytes);
291
+ const arities = new Map(
292
+ module.definitions
293
+ .filter((definition) => isRoot || !definition.isLocal)
294
+ .map((definition) => [
295
+ definition.name,
296
+ definition.kind === "Operator" ? definition.params.length : 0,
297
+ ]),
298
+ );
299
+ const isMatched = (method: JavaMethod) => arities.get(method.name) === method.arity;
228
300
  const definitions = module.definitions.map((definition): Definition => {
229
301
  if (definition.kind !== "Operator") return definition;
230
302
  const method = classFile.methods.find(
231
- (candidate) =>
232
- candidate.name === definition.name && candidate.arity === definition.params.length,
303
+ (candidate) => candidate.name === definition.name && isMatched(candidate),
233
304
  );
234
305
  if (!method) return definition;
235
306
  return {
@@ -237,5 +308,13 @@ export const overrideDefinitions = (module: Module, bytes: Uint8Array | undefine
237
308
  body: getReturnedLiteral(method, classFile.pool, module.name, definition.location),
238
309
  };
239
310
  });
240
- return { ...module, definitions };
311
+ return {
312
+ module: { ...module, definitions },
313
+ overrides: classFile.methods.map((method) => ({
314
+ module: module.name,
315
+ operator: method.name,
316
+ signature: method.signature,
317
+ isMatched: isMatched(method),
318
+ })),
319
+ };
241
320
  };
package/src/levels.ts CHANGED
@@ -181,8 +181,10 @@ const bindFormals = (scope: Scope, formals: OperatorParam[]) =>
181
181
 
182
182
  const toFormals = (names: string[]) => names.map((name) => ({ name, arity: 0 }));
183
183
 
184
+ // The levels of tla2sany.semantic.LevelConstants.
185
+ export const CONSTANT_LEVEL = 0;
186
+ export const VARIABLE_LEVEL = 1;
184
187
  export const ACTION_LEVEL = 2;
185
-
186
188
  export const TEMPORAL_LEVEL = 3;
187
189
 
188
190
  // SANY reports a level error at the range of the expression it rejects.
@@ -690,6 +692,9 @@ export const createLevelAnalyzer = (context: LevelContext) => {
690
692
  ? toOperand(expression.args[0], scope)
691
693
  : { data: applyBuiltin(SET_LEVELS, expression.args, scope) },
692
694
  ]);
695
+ // SANY's semantic checker rejects the expression without computing its level.
696
+ case "Bracket":
697
+ return createData();
693
698
  case "FieldAccess":
694
699
  return applyDefined(SELECT_LEVELS, [
695
700
  toOperand(expression.record, scope),
package/src/lexer.ts CHANGED
@@ -12,9 +12,11 @@ type TokenKind =
12
12
  | "ModuleEnd"
13
13
  | "End";
14
14
 
15
+ // spelling holds the text of an operator written as a synonym, such as (+) for \oplus.
15
16
  export interface Token {
16
17
  kind: TokenKind;
17
18
  text: string;
19
+ spelling?: string;
18
20
  line: number;
19
21
  column: number;
20
22
  endColumn: number;
@@ -232,38 +234,84 @@ const SYMBOL_OPERATORS = [
232
234
  "?",
233
235
  ];
234
236
 
237
+ // The Unicode forms of the operators SANY's grammar, javacc/tla+.jj, reads, as their ASCII forms.
235
238
  const UNICODE_OPERATORS = new Map([
239
+ ["∃", "\\E"],
240
+ ["∀", "\\A"],
241
+ ["∷", "::"],
242
+ ["≜", "=="],
243
+ ["⟨", "<<"],
244
+ ["⟩_", ">>_"],
245
+ ["⟩", ">>"],
246
+ ["→", "->"],
247
+ ["←", "<-"],
248
+ ["↦", "|->"],
249
+ ["⁺", "^+"],
250
+ ["¬", "~"],
251
+ ["□", "[]"],
252
+ ["◇", "<>"],
236
253
  ["∧", "/\\"],
237
254
  ["∨", "\\/"],
238
- ["¬", "~"],
239
- ["⇒", "=>"],
240
- ["⟹", "=>"],
255
+ ["∈", "\\in"],
256
+ ["≈", "\\approx"],
257
+ ["≔", ":="],
258
+ ["≍", "\\asymp"],
259
+ ["◯", "\\bigcirc"],
260
+ ["⩴", "::="],
261
+ ["●", "\\bullet"],
262
+ ["∩", "\\cap"],
263
+ ["⋅", "\\cdot"],
264
+ ["∘", "\\o"],
265
+ ["≅", "\\cong"],
266
+ ["∪", "\\cup"],
267
+ ["÷", "\\div"],
268
+ ["≐", "\\doteq"],
269
+ ["‥", ".."],
270
+ ["…", "..."],
241
271
  ["≡", "<=>"],
272
+ ["‼", "!!"],
273
+ ["≥", ">="],
274
+ ["≫", "\\gg"],
242
275
  ["⇔", "<=>"],
243
- ["∈", "\\in"],
276
+ ["⇒", "=>"],
277
+ ["⫤", "=|"],
278
+ ["↝", "~>"],
279
+ ["≤", "<="],
280
+ ["≪", "\\ll"],
281
+ ["⊣", "-|"],
282
+ ["≠", "#"],
244
283
  ["∉", "\\notin"],
245
- ["⊆", "\\subseteq"],
284
+ ["⊙", "\\odot"],
285
+ ["⊖", "\\ominus"],
286
+ ["⊕", "\\oplus"],
287
+ ["⊘", "\\oslash"],
288
+ ["⊗", "\\otimes"],
289
+ ["⇸", "-+->"],
290
+ ["≺", "\\prec"],
291
+ ["⪯", "\\preceq"],
292
+ ["∝", "\\propto"],
293
+ ["⁇", "??"],
294
+ ["⊨", "|="],
295
+ ["⊢", "|-"],
296
+ ["∼", "\\sim"],
297
+ ["≃", "\\simeq"],
298
+ ["⊓", "\\sqcap"],
299
+ ["⊔", "\\sqcup"],
300
+ ["⊏", "\\sqsubset"],
301
+ ["⊑", "\\sqsubseteq"],
302
+ ["⊐", "\\sqsupset"],
303
+ ["⊒", "\\sqsupseteq"],
304
+ ["⋆", "\\star"],
246
305
  ["⊂", "\\subset"],
247
- ["⊇", "\\supseteq"],
306
+ ["⊆", "\\subseteq"],
307
+ ["≻", "\\succ"],
308
+ ["⪰", "\\succeq"],
248
309
  ["⊃", "\\supset"],
249
- ["∪", "\\cup"],
250
- ["∩", "\\cap"],
310
+ ["⊇", "\\supseteq"],
251
311
  ["×", "\\X"],
252
- ["∀", "\\A"],
253
- ["∃", "\\E"],
254
- ["≠", "#"],
255
- ["≤", "<="],
256
- ["≥", ">="],
257
- ["↦", "|->"],
258
- ["→", "->"],
259
- ["⟨", "<<"],
260
- ["⟩", ">>"],
261
- ["□", "[]"],
262
- ["◇", "<>"],
263
- ["≜", "=="],
264
- ["′", "'"],
265
- ["∘", "\\o"],
266
- ["⋅", "\\cdot"],
312
+ ["⊎", "\\uplus"],
313
+ ["‖", "||"],
314
+ ["≀", "\\wr"],
267
315
  ]);
268
316
 
269
317
  const BASED_NUMBER = /^\\([bB][01]+|[oO][0-7]+|[hH][0-9a-fA-F]+)/;
@@ -301,13 +349,27 @@ export const matchOperator = (source: string, index: number): OperatorMatch | un
301
349
  const operator = BACKSLASH_OPERATORS.get(word.slice(0, length.value));
302
350
  return { text: operator ?? "\\", length: operator === undefined ? 1 : length.value + 1 };
303
351
  }
304
- const unicodeOperator = UNICODE_OPERATORS.get(character);
305
- if (unicodeOperator) return { text: unicodeOperator, length: character.length };
352
+ const unicodeSymbol = [source.slice(index, index + 2), character].find((candidate) =>
353
+ UNICODE_OPERATORS.has(candidate),
354
+ );
355
+ if (unicodeSymbol !== undefined) {
356
+ return { text: UNICODE_OPERATORS.get(unicodeSymbol) ?? "", length: unicodeSymbol.length };
357
+ }
306
358
  const symbol = SYMBOL_OPERATORS.find((candidate) => source.startsWith(candidate, index));
307
359
  if (symbol) return { text: SYMBOL_ALIASES.get(symbol) ?? symbol, length: symbol.length };
308
360
  return character === "\\" ? { text: "\\", length: 1 } : undefined;
309
361
  };
310
362
 
363
+ // TLC's configuration parser reads its tokens with SANY's lexer, so it reads a Unicode operator, or
364
+ // ℕ, ℤ, or ℝ, as one token, which it compares as written with the words it expects.
365
+ const UNICODE_NAMES = new Set(["ℕ", "ℤ", "ℝ"]);
366
+ export const matchUnicodeToken = (source: string, index: number): OperatorMatch | undefined => {
367
+ const token = [source.slice(index, index + 2), source[index]].find(
368
+ (candidate) => UNICODE_OPERATORS.has(candidate) || UNICODE_NAMES.has(candidate),
369
+ );
370
+ return token === undefined ? undefined : { text: token, length: token.length };
371
+ };
372
+
311
373
  const isDigit = (character: string | undefined) =>
312
374
  character !== undefined && character >= "0" && character <= "9";
313
375
 
@@ -335,6 +397,10 @@ const matchStep = (source: string, index: number) => {
335
397
  );
336
398
  };
337
399
 
400
+ // SANY's lexer reads @ as an identifier, which stands for the value an EXCEPT changes only alone.
401
+ export const readAtAsIdentifier = (token: Token): Token =>
402
+ token.kind === "Operator" && token.text === "@" ? { ...token, kind: "Identifier" } : token;
403
+
338
404
  const TAB_SIZE = 8;
339
405
 
340
406
  const getWidthAfter = (width: number, character: string | undefined) =>
@@ -367,9 +433,16 @@ export const tokenize = (source: string): Token[] => {
367
433
  }
368
434
  return getWidthAfter(scan.width, source[cursor.index]);
369
435
  };
370
- const pushToken = (kind: TokenKind, text: string, length: number) => {
436
+ const pushToken = (kind: TokenKind, text: string, length: number, spelling?: string) => {
371
437
  const column = getColumn();
372
- tokens.push({ kind, text, line: cursor.line, column, endColumn: column + length - 1 });
438
+ tokens.push({
439
+ kind,
440
+ text,
441
+ spelling,
442
+ line: cursor.line,
443
+ column,
444
+ endColumn: column + length - 1,
445
+ });
373
446
  cursor.index += length;
374
447
  };
375
448
  const advanceOver = (length: number) => {
@@ -442,11 +515,14 @@ export const tokenize = (source: string): Token[] => {
442
515
  const characters: string[] = [];
443
516
  advanceOver(1);
444
517
  while (source[cursor.index] !== '"') {
445
- if (cursor.index >= source.length || source[cursor.index] === "\n") {
518
+ if (cursor.index >= source.length || "\n\r".includes(source[cursor.index])) {
446
519
  throw createError("Unterminated string");
447
520
  }
448
521
  if (source[cursor.index] === "\\") {
449
522
  const escaped = source[cursor.index + 1];
523
+ if (escaped === undefined || !'ntrf\\"'.includes(escaped)) {
524
+ throw createError("Invalid escape in string");
525
+ }
450
526
  characters.push({ n: "\n", t: "\t", r: "\r", f: "\f" }[escaped] ?? escaped);
451
527
  advanceOver(2);
452
528
  continue;
@@ -479,6 +555,12 @@ export const tokenize = (source: string): Token[] => {
479
555
  pushToken(number.includes(".") ? "Decimal" : "Number", number, number.length);
480
556
  continue;
481
557
  }
558
+ // A word needs a letter to be a name, so SANY reads one without as underscores and numbers.
559
+ const part = /^(_|\d+)/.exec(word)?.[0];
560
+ if (part !== undefined && !/[A-Za-z]/.test(word)) {
561
+ pushToken(part === "_" ? "Operator" : "Number", part, part.length);
562
+ continue;
563
+ }
482
564
  if (word.startsWith("WF_") || word.startsWith("SF_")) {
483
565
  pushToken("Keyword", word.slice(0, 3), 3);
484
566
  continue;
@@ -488,6 +570,11 @@ export const tokenize = (source: string): Token[] => {
488
570
  pushToken(KEYWORDS.has(word) ? "Keyword" : "Identifier", name, word.length);
489
571
  continue;
490
572
  }
573
+ if ("ℕℤℝ".includes(character)) {
574
+ pushToken("Identifier", names.get(character) ?? character, 1);
575
+ names.set(character, character);
576
+ continue;
577
+ }
491
578
  const stepLength = matchStep(source, cursor.index);
492
579
  if (stepLength > 0) {
493
580
  pushToken("Step", source.slice(cursor.index, cursor.index + stepLength), stepLength);
@@ -495,7 +582,13 @@ export const tokenize = (source: string): Token[] => {
495
582
  }
496
583
  const operator = matchOperator(source, cursor.index);
497
584
  if (operator) {
498
- pushToken("Operator", operator.text, operator.length);
585
+ const spelling = source.slice(cursor.index, cursor.index + operator.length);
586
+ pushToken(
587
+ "Operator",
588
+ operator.text,
589
+ operator.length,
590
+ spelling === operator.text ? undefined : spelling,
591
+ );
499
592
  continue;
500
593
  }
501
594
  throw createError(`Unexpected character ${JSON.stringify(character)}`);