tla-ts 0.0.3-dev.c8c988d → 0.0.3-dev.ccbff8c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/cli.cjs +1 -1
  2. package/dist/cli.js +1 -1
  3. package/dist/index.cjs +1 -1
  4. package/dist/index.d.cts +163 -46
  5. package/dist/index.d.ts +163 -46
  6. package/dist/index.js +1 -1
  7. package/dist/parallel-worker.cjs +1 -1
  8. package/dist/parallel-worker.js +2 -2
  9. package/dist/parallel.cjs +1 -1
  10. package/dist/parallel.js +2 -2
  11. package/dist/run-cli.cjs +1351 -22
  12. package/dist/run-cli.js +1351 -22
  13. package/dist/transfer.cjs +83 -33
  14. package/dist/transfer.js +83 -33
  15. package/package.json +5 -1
  16. package/src/ast.ts +51 -6
  17. package/src/bags.ts +4 -7
  18. package/src/builtins.ts +29 -17
  19. package/src/bundled-modules.ts +3 -3
  20. package/src/canonical.ts +4 -3
  21. package/src/check.ts +1317 -428
  22. package/src/cli.ts +8 -1
  23. package/src/codegen.ts +68 -21
  24. package/src/config.ts +225 -206
  25. package/src/environment.ts +417 -129
  26. package/src/errors.ts +155 -18
  27. package/src/evaluate.ts +713 -259
  28. package/src/exit-codes.ts +6 -24
  29. package/src/fingerprints.ts +1 -2
  30. package/src/index.ts +1 -0
  31. package/src/instances.ts +3 -1
  32. package/src/java-overrides.ts +104 -25
  33. package/src/levels.ts +44 -28
  34. package/src/lexer.ts +32 -4
  35. package/src/live-check.ts +204 -148
  36. package/src/liveness.ts +191 -122
  37. package/src/obligation-expression.ts +2 -2
  38. package/src/obligation-source.ts +2 -0
  39. package/src/parallel-worker.ts +6 -1
  40. package/src/parallel.ts +57 -6
  41. package/src/parser.ts +405 -167
  42. package/src/pcal-ast.ts +770 -0
  43. package/src/pcal-parse.ts +1523 -0
  44. package/src/pcal-tokenize.ts +516 -0
  45. package/src/pcal-translate.ts +1230 -0
  46. package/src/pcal-validate.ts +454 -0
  47. package/src/randomization.ts +24 -7
  48. package/src/resolve.ts +1397 -457
  49. package/src/run-cli.ts +1161 -196
  50. package/src/runtime.ts +16 -0
  51. package/src/sany-builtins.ts +147 -0
  52. package/src/sany-generator.ts +3112 -0
  53. package/src/sany-levels.ts +1208 -0
  54. package/src/sany-lexer.ts +545 -0
  55. package/src/sany-linter.ts +122 -0
  56. package/src/sany-modules.ts +1398 -0
  57. package/src/sany-nodes.ts +463 -0
  58. package/src/sany-parse-error.ts +2161 -0
  59. package/src/sany-report.ts +354 -0
  60. package/src/sany-selectors.ts +738 -0
  61. package/src/sany-source.ts +248 -0
  62. package/src/sany-xml.ts +1017 -0
  63. package/src/spec-description.ts +25 -5
  64. package/src/standard-modules.ts +172 -129
  65. package/src/standard-ranges.ts +116 -0
  66. package/src/state-graph.ts +29 -6
  67. package/src/subexpressions.ts +221 -161
  68. package/src/substitution-levels.ts +5 -8
  69. package/src/successors.ts +141 -173
  70. package/src/symmetry.ts +168 -50
  71. package/src/tableau.ts +3 -1
  72. package/src/tlc-ext.ts +13 -17
  73. package/src/tlc-messages.ts +329 -0
  74. package/src/tlc-module.ts +49 -21
  75. package/src/tlc-modules.ts +0 -4
  76. package/src/tlc-order.ts +55 -9
  77. package/src/tlc-random.ts +28 -3
  78. package/src/tlc-replay.ts +655 -0
  79. package/src/tlc-version.ts +18 -0
  80. package/src/transfer.ts +15 -3
  81. package/src/values.ts +664 -317
package/src/exit-codes.ts CHANGED
@@ -26,19 +26,7 @@ export const setExitCode = <Thrown>(error: Thrown, code: number) => {
26
26
 
27
27
  export const getExitCode = (error: Error) => exitCodes.get(error);
28
28
 
29
- const ownExitCodes = new WeakMap<Error, number>();
30
-
31
- // The exit status of the error code TLC gives an EvalException, which TLC uses only where it
32
- // doesn't report the error again as a nested expression.
33
- export const setOwnExitCode = <Thrown extends Error>(error: Thrown, code: number) => {
34
- ownExitCodes.set(error, code);
35
- return error;
36
- };
37
-
38
- export const getOwnExitCode = (error: unknown) =>
39
- error instanceof Error ? ownExitCodes.get(error) : undefined;
40
-
41
- // TLC exits with 255 when it fails to process or evaluate a liveness property.
29
+ // TLC exits with 255 when it fails to process or evaluate a liveness property, in any phase.
42
30
  const withErrorExitCode = <Result>(run: () => Result) => {
43
31
  try {
44
32
  return run();
@@ -53,17 +41,11 @@ export const withLivenessExitCode = (
53
41
  const checker = withErrorExitCode(create);
54
42
  return (
55
43
  checker && {
56
- // TLC replays the initial states to report an error, but skips the liveness check of a
57
- // state it has already seen, so it exits with the status of the error's own code.
58
- addInitialState: (index, state) => {
59
- try {
60
- checker.addInitialState(index, state);
61
- } catch (error) {
62
- throw setExitCode(error, getOwnExitCode(error) ?? EXIT_STATUS.error);
63
- }
64
- },
65
- addState: (index, state, successors) =>
66
- withErrorExitCode(() => checker.addState(index, state, successors)),
44
+ branches: checker.branches,
45
+ addInitialState: (index, state, tracing) =>
46
+ withErrorExitCode(() => checker.addInitialState(index, state, tracing)),
47
+ addState: (index, state, successors, tracing) =>
48
+ withErrorExitCode(() => checker.addState(index, state, successors, tracing)),
67
49
  findViolation: () => withErrorExitCode(() => checker.findViolation()),
68
50
  getViolatedProperties: (path, cycleStart) =>
69
51
  withErrorExitCode(() => checker.getViolatedProperties(path, cycleStart)),
@@ -1,4 +1,4 @@
1
- import { assertFingerprintable, getValueFingerprint, getValueHash, type Value } from "./values";
1
+ import { getValueFingerprint, getValueHash, type Value } from "./values";
2
2
 
3
3
  const EMPTY = 0;
4
4
  const INITIAL_CAPACITY = 1 << 12;
@@ -20,7 +20,6 @@ export const writeStateFingerprint = (state: Value[], fingerprint: Int32Array) =
20
20
  first = Math.imul(first ^ getValueHash(state[index]), 0x01000193);
21
21
  second = Math.imul(second ^ getValueFingerprint(state[index]), 0x5bd1e995);
22
22
  }
23
- assertFingerprintable(state);
24
23
  fingerprint[0] = mix(first) || 1;
25
24
  fingerprint[1] = mix(second);
26
25
  };
package/src/index.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  export type {
2
2
  CheckOptions,
3
+ CheckProgress,
3
4
  CheckResult,
4
5
  CheckStatus,
5
6
  EvaluationFailure,
package/src/instances.ts CHANGED
@@ -183,6 +183,8 @@ export const mapChildren = (
183
183
  function: plain(expression.function),
184
184
  args: expression.args.map(plain),
185
185
  };
186
+ case "Bracket":
187
+ return { ...expression, args: expression.args.map(plain) };
186
188
  case "FieldAccess":
187
189
  return { ...expression, record: plain(expression.record) };
188
190
  case "Product":
@@ -705,7 +707,7 @@ const expandLetInstances = (
705
707
  const index = position === -1 ? current.length : position;
706
708
  return [...current.slice(0, index), ...instantiate(instance, inner), ...current.slice(index)];
707
709
  }, mapped.definitions);
708
- return { kind: "Let", definitions, body: mapped.body, location: mapped.location };
710
+ return { ...mapped, definitions };
709
711
  };
710
712
 
711
713
  // A named theorem's statement is linked as its definition.
@@ -12,9 +12,20 @@ interface PoolEntry {
12
12
  interface JavaMethod {
13
13
  name: string;
14
14
  arity: number;
15
+ // Method#toString, which TLC prints as the signature of an override.
16
+ signature: string;
15
17
  code: Uint8Array | undefined;
16
18
  }
17
19
 
20
+ // A public static method of the class named after a module, which TLC reports as it loads the
21
+ // class, with whether it overrides an operator.
22
+ export interface JavaOverride {
23
+ module: string;
24
+ operator: string;
25
+ signature: string;
26
+ isMatched: boolean;
27
+ }
28
+
18
29
  interface ObjectReference {
19
30
  className: string;
20
31
  value: Expression | undefined;
@@ -44,12 +55,44 @@ const createReader = (bytes: Uint8Array) => {
44
55
  };
45
56
  };
46
57
 
47
- const getParameterCount = (descriptor: string) => {
48
- const parameters = descriptor.slice(1, descriptor.indexOf(")"));
49
- return [...parameters.matchAll(/\[*(?:L[^;]*;|[BCDFIJSZ])/g)].length;
50
- };
58
+ const PRIMITIVE_TYPES = new Map([
59
+ ["B", "byte"],
60
+ ["C", "char"],
61
+ ["D", "double"],
62
+ ["F", "float"],
63
+ ["I", "int"],
64
+ ["J", "long"],
65
+ ["S", "short"],
66
+ ["Z", "boolean"],
67
+ ["V", "void"],
68
+ ]);
69
+
70
+ // The modifiers Method#toString prints, in the order of Modifier#toString.
71
+ const METHOD_MODIFIERS: [number, string][] = [
72
+ [0x0001, "public"],
73
+ [0x0004, "protected"],
74
+ [0x0002, "private"],
75
+ [0x0400, "abstract"],
76
+ [0x0008, "static"],
77
+ [0x0010, "final"],
78
+ [0x0020, "synchronized"],
79
+ [0x0100, "native"],
80
+ [0x0800, "strictfp"],
81
+ ];
82
+
83
+ const getTypeNames = (descriptor: string) =>
84
+ [...descriptor.matchAll(/(\[*)(?:L([^;]*);|([BCDFIJSZV]))/g)].map(
85
+ (match) =>
86
+ `${match[2]?.replaceAll("/", ".") ?? PRIMITIVE_TYPES.get(match[3])}${"[]".repeat(match[1].length)}`,
87
+ );
51
88
 
52
- // The public static methods of a class file and their bytecode, as java.lang.Class lists them.
89
+ const getClassName = (pool: PoolEntry[], index: number) => pool[pool[index].first].text;
90
+
91
+ const getJavaClassName = (pool: PoolEntry[], index: number) =>
92
+ getClassName(pool, index).replaceAll("/", ".");
93
+
94
+ // The public static methods of a class file and their bytecode, in the order of the class file.
95
+ // Class#getDeclaredMethods orders them by where HotSpot allocated the symbols of their names.
53
96
  const readMethods = (bytes: Uint8Array) => {
54
97
  const reader = createReader(bytes);
55
98
  if (reader.readUint32() !== CLASS_MAGIC)
@@ -80,7 +123,7 @@ const readMethods = (bytes: Uint8Array) => {
80
123
  }
81
124
  }
82
125
  reader.readUint16();
83
- reader.readUint16();
126
+ const className = getJavaClassName(pool, reader.readUint16());
84
127
  reader.readUint16();
85
128
  reader.readBytes(reader.readUint16() * 2);
86
129
  for (let fields = reader.readUint16(); fields > 0; fields--) {
@@ -95,23 +138,38 @@ const readMethods = (bytes: Uint8Array) => {
95
138
  const access = reader.readUint16();
96
139
  const name = pool[reader.readUint16()].text;
97
140
  const descriptor = pool[reader.readUint16()].text;
98
- const method: JavaMethod = { name, arity: getParameterCount(descriptor), code: undefined };
99
- for (let attributes = reader.readUint16(); attributes > 0; attributes--) {
100
- const attributeName = pool[reader.readUint16()].text;
101
- const attribute = reader.readBytes(reader.readUint32());
102
- if (attributeName === "Code") {
103
- const length = new DataView(attribute.buffer, attribute.byteOffset + 4, 4).getUint32(0);
104
- method.code = attribute.subarray(8, 8 + length);
105
- }
106
- }
141
+ const parametersEnd = descriptor.indexOf(")");
142
+ const parameters = getTypeNames(descriptor.slice(1, parametersEnd));
143
+ const attributes = Array.from({ length: reader.readUint16() }, () => ({
144
+ name: pool[reader.readUint16()].text,
145
+ reader: createReader(reader.readBytes(reader.readUint32())),
146
+ }));
107
147
  const isOverride = (access & ACCESS_PUBLIC) !== 0 && (access & ACCESS_STATIC) !== 0;
108
- if (isOverride) methods.push(method);
148
+ if (!isOverride) continue;
149
+ const code = attributes.find((attribute) => attribute.name === "Code")?.reader;
150
+ // A Code attribute holds max_stack and max_locals before the length of the bytecode.
151
+ code?.readBytes(4);
152
+ const thrown = attributes.find((attribute) => attribute.name === "Exceptions")?.reader;
153
+ const exceptions = thrown
154
+ ? Array.from({ length: thrown.readUint16() }, () =>
155
+ getJavaClassName(pool, thrown.readUint16()),
156
+ )
157
+ : [];
158
+ const modifiers = METHOD_MODIFIERS.filter(([flag]) => (access & flag) !== 0).map(
159
+ ([, modifier]) => modifier,
160
+ );
161
+ const returnType = getTypeNames(descriptor.slice(parametersEnd + 1))[0];
162
+ const throwsClause = exceptions.length === 0 ? "" : ` throws ${exceptions.join(",")}`;
163
+ methods.push({
164
+ name,
165
+ arity: parameters.length,
166
+ code: code?.readBytes(code.readUint32()),
167
+ signature: `${modifiers.join(" ")} ${returnType} ${className}.${name}(${parameters.join(",")})${throwsClause}`,
168
+ });
109
169
  }
110
170
  return { pool, methods };
111
171
  };
112
172
 
113
- const getClassName = (pool: PoolEntry[], index: number) => pool[pool[index].first].text;
114
-
115
173
  const getMember = (pool: PoolEntry[], index: number) => {
116
174
  const reference = pool[index];
117
175
  const nameAndType = pool[reference.second];
@@ -220,16 +278,29 @@ const getReturnedLiteral = (
220
278
  throw unsupported();
221
279
  };
222
280
 
223
- // TLC loads a class named after each module and replaces every operator the module defines
224
- // with the public static method of the same name and arity.
225
- export const overrideDefinitions = (module: Module, bytes: Uint8Array | undefined): Module => {
226
- if (!bytes) return module;
281
+ // TLC loads a class named after each module and replaces each operator of the module that the
282
+ // root module sees with the public static method of the same name and arity. It reports whether
283
+ // each method matched an operator.
284
+ export const overrideDefinitions = (
285
+ module: Module,
286
+ bytes: Uint8Array | undefined,
287
+ isRoot: boolean,
288
+ ): { module: Module; overrides: JavaOverride[] } => {
289
+ if (!bytes) return { module, overrides: [] };
227
290
  const classFile = readMethods(bytes);
291
+ const arities = new Map(
292
+ module.definitions
293
+ .filter((definition) => isRoot || !definition.isLocal)
294
+ .map((definition) => [
295
+ definition.name,
296
+ definition.kind === "Operator" ? definition.params.length : 0,
297
+ ]),
298
+ );
299
+ const isMatched = (method: JavaMethod) => arities.get(method.name) === method.arity;
228
300
  const definitions = module.definitions.map((definition): Definition => {
229
301
  if (definition.kind !== "Operator") return definition;
230
302
  const method = classFile.methods.find(
231
- (candidate) =>
232
- candidate.name === definition.name && candidate.arity === definition.params.length,
303
+ (candidate) => candidate.name === definition.name && isMatched(candidate),
233
304
  );
234
305
  if (!method) return definition;
235
306
  return {
@@ -237,5 +308,13 @@ export const overrideDefinitions = (module: Module, bytes: Uint8Array | undefine
237
308
  body: getReturnedLiteral(method, classFile.pool, module.name, definition.location),
238
309
  };
239
310
  });
240
- return { ...module, definitions };
311
+ return {
312
+ module: { ...module, definitions },
313
+ overrides: classFile.methods.map((method) => ({
314
+ module: module.name,
315
+ operator: method.name,
316
+ signature: method.signature,
317
+ isMatched: isMatched(method),
318
+ })),
319
+ };
241
320
  };
package/src/levels.ts CHANGED
@@ -7,6 +7,7 @@ import {
7
7
  type OperatorParam,
8
8
  type Proof,
9
9
  type ProofStep,
10
+ type SourceLocation,
10
11
  } from "./ast";
11
12
  import {
12
13
  createSanyError,
@@ -180,8 +181,10 @@ const bindFormals = (scope: Scope, formals: OperatorParam[]) =>
180
181
 
181
182
  const toFormals = (names: string[]) => names.map((name) => ({ name, arity: 0 }));
182
183
 
184
+ // The levels of tla2sany.semantic.LevelConstants.
185
+ export const CONSTANT_LEVEL = 0;
186
+ export const VARIABLE_LEVEL = 1;
183
187
  export const ACTION_LEVEL = 2;
184
-
185
188
  export const TEMPORAL_LEVEL = 3;
186
189
 
187
190
  // SANY reports a level error at the range of the expression it rejects.
@@ -689,6 +692,9 @@ export const createLevelAnalyzer = (context: LevelContext) => {
689
692
  ? toOperand(expression.args[0], scope)
690
693
  : { data: applyBuiltin(SET_LEVELS, expression.args, scope) },
691
694
  ]);
695
+ // SANY's semantic checker rejects the expression without computing its level.
696
+ case "Bracket":
697
+ return createData();
692
698
  case "FieldAccess":
693
699
  return applyDefined(SELECT_LEVELS, [
694
700
  toOperand(expression.record, scope),
@@ -879,8 +885,13 @@ export const analyzeAssumeProve = (
879
885
  return data;
880
886
  };
881
887
 
882
- // SANY checks the level of every definition, assumption, and theorem of a module.
883
- export const checkModuleLevels = (module: Module, context: LevelContext) => {
888
+ // SANY checks the level of every definition, assumption, theorem, and instance of a module.
889
+ export const checkModuleLevels = (
890
+ module: Module,
891
+ context: LevelContext,
892
+ instances: { location: SourceLocation; check: () => void }[],
893
+ warnings: SanyError[],
894
+ ) => {
884
895
  const analyzer = createLevelAnalyzer(context);
885
896
  const getLevel = (expressions: Expression[]) =>
886
897
  Math.max(0, ...expressions.map((expression) => analyzer.analyze(expression, new Map()).level));
@@ -951,30 +962,35 @@ export const checkModuleLevels = (module: Module, context: LevelContext) => {
951
962
  ...(step.proof?.kind === "Steps" && isTemporal() ? checkTemporalGoal(step.proof) : []),
952
963
  ];
953
964
  });
954
- collectSemanticErrors([
955
- ...module.definitions
956
- .filter((definition) => !definition.name.includes("!"))
957
- .map((definition) => ({
958
- location: definition.location,
959
- check: () => analyzer.analyzeDefinition(definition, new Map()),
965
+ const errors = collectSemanticErrors(
966
+ [
967
+ ...module.definitions
968
+ .filter((definition) => !definition.name.includes("!"))
969
+ .map((definition) => ({
970
+ location: definition.location,
971
+ check: () => analyzer.analyzeDefinition(definition, new Map()),
972
+ })),
973
+ ...module.assumptions.map((assumption) => ({
974
+ location: assumption.location,
975
+ check: () => analyzer.analyze(assumption.expression, new Map()),
976
+ })),
977
+ ...module.theorems.map((theorem) => ({
978
+ location: theorem.location,
979
+ check: () => {
980
+ const data =
981
+ theorem.statement.kind === "AssumeProve"
982
+ ? analyzeAssumeProve(theorem.statement, new Map(), analyzer)
983
+ : analyzer.analyze(theorem.statement, new Map());
984
+ const errors = [
985
+ ...checkProof(theorem.proof),
986
+ ...(data.level === TEMPORAL_LEVEL ? checkTemporalGoal(theorem.proof) : []),
987
+ ];
988
+ if (errors.length > 0) throw createSanyError(errors);
989
+ },
960
990
  })),
961
- ...module.assumptions.map((assumption) => ({
962
- location: assumption.location,
963
- check: () => analyzer.analyze(assumption.expression, new Map()),
964
- })),
965
- ...module.theorems.map((theorem) => ({
966
- location: theorem.location,
967
- check: () => {
968
- const data =
969
- theorem.statement.kind === "AssumeProve"
970
- ? analyzeAssumeProve(theorem.statement, new Map(), analyzer)
971
- : analyzer.analyze(theorem.statement, new Map());
972
- const errors = [
973
- ...checkProof(theorem.proof),
974
- ...(data.level === TEMPORAL_LEVEL ? checkTemporalGoal(theorem.proof) : []),
975
- ];
976
- if (errors.length > 0) throw createSanyError(errors);
977
- },
978
- })),
979
- ]);
991
+ ...instances,
992
+ ],
993
+ warnings,
994
+ );
995
+ if (errors.length > 0) throw createSanyError(errors, false, undefined, warnings);
980
996
  };
package/src/lexer.ts CHANGED
@@ -12,9 +12,11 @@ type TokenKind =
12
12
  | "ModuleEnd"
13
13
  | "End";
14
14
 
15
+ // spelling holds the text of an operator written as a synonym, such as (+) for \oplus.
15
16
  export interface Token {
16
17
  kind: TokenKind;
17
18
  text: string;
19
+ spelling?: string;
18
20
  line: number;
19
21
  column: number;
20
22
  endColumn: number;
@@ -335,6 +337,10 @@ const matchStep = (source: string, index: number) => {
335
337
  );
336
338
  };
337
339
 
340
+ // SANY's lexer reads @ as an identifier, which stands for the value an EXCEPT changes only alone.
341
+ export const readAtAsIdentifier = (token: Token): Token =>
342
+ token.kind === "Operator" && token.text === "@" ? { ...token, kind: "Identifier" } : token;
343
+
338
344
  const TAB_SIZE = 8;
339
345
 
340
346
  const getWidthAfter = (width: number, character: string | undefined) =>
@@ -367,9 +373,16 @@ export const tokenize = (source: string): Token[] => {
367
373
  }
368
374
  return getWidthAfter(scan.width, source[cursor.index]);
369
375
  };
370
- const pushToken = (kind: TokenKind, text: string, length: number) => {
376
+ const pushToken = (kind: TokenKind, text: string, length: number, spelling?: string) => {
371
377
  const column = getColumn();
372
- tokens.push({ kind, text, line: cursor.line, column, endColumn: column + length - 1 });
378
+ tokens.push({
379
+ kind,
380
+ text,
381
+ spelling,
382
+ line: cursor.line,
383
+ column,
384
+ endColumn: column + length - 1,
385
+ });
373
386
  cursor.index += length;
374
387
  };
375
388
  const advanceOver = (length: number) => {
@@ -442,11 +455,14 @@ export const tokenize = (source: string): Token[] => {
442
455
  const characters: string[] = [];
443
456
  advanceOver(1);
444
457
  while (source[cursor.index] !== '"') {
445
- if (cursor.index >= source.length || source[cursor.index] === "\n") {
458
+ if (cursor.index >= source.length || "\n\r".includes(source[cursor.index])) {
446
459
  throw createError("Unterminated string");
447
460
  }
448
461
  if (source[cursor.index] === "\\") {
449
462
  const escaped = source[cursor.index + 1];
463
+ if (escaped === undefined || !'ntrf\\"'.includes(escaped)) {
464
+ throw createError("Invalid escape in string");
465
+ }
450
466
  characters.push({ n: "\n", t: "\t", r: "\r", f: "\f" }[escaped] ?? escaped);
451
467
  advanceOver(2);
452
468
  continue;
@@ -479,6 +495,12 @@ export const tokenize = (source: string): Token[] => {
479
495
  pushToken(number.includes(".") ? "Decimal" : "Number", number, number.length);
480
496
  continue;
481
497
  }
498
+ // A word needs a letter to be a name, so SANY reads one without as underscores and numbers.
499
+ const part = /^(_|\d+)/.exec(word)?.[0];
500
+ if (part !== undefined && !/[A-Za-z]/.test(word)) {
501
+ pushToken(part === "_" ? "Operator" : "Number", part, part.length);
502
+ continue;
503
+ }
482
504
  if (word.startsWith("WF_") || word.startsWith("SF_")) {
483
505
  pushToken("Keyword", word.slice(0, 3), 3);
484
506
  continue;
@@ -495,7 +517,13 @@ export const tokenize = (source: string): Token[] => {
495
517
  }
496
518
  const operator = matchOperator(source, cursor.index);
497
519
  if (operator) {
498
- pushToken("Operator", operator.text, operator.length);
520
+ const spelling = source.slice(cursor.index, cursor.index + operator.length);
521
+ pushToken(
522
+ "Operator",
523
+ operator.text,
524
+ operator.length,
525
+ spelling === operator.text ? undefined : spelling,
526
+ );
499
527
  continue;
500
528
  }
501
529
  throw createError(`Unexpected character ${JSON.stringify(character)}`);