tla-ts 0.0.10-dev.28c823e4b → 0.0.10-dev.b4a856304

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist/index.cjs +1 -1
  2. package/dist/index.d.cts +0 -34
  3. package/dist/index.d.ts +0 -34
  4. package/dist/index.js +1 -1
  5. package/dist/parallel-worker.cjs +1 -1
  6. package/dist/parallel-worker.js +1 -1
  7. package/dist/prove.cjs +43 -43
  8. package/dist/prove.js +47 -47
  9. package/dist/run.cjs +30 -30
  10. package/dist/run.js +33 -33
  11. package/dist/transfer.cjs +38 -38
  12. package/dist/transfer.js +39 -39
  13. package/package.json +1 -1
  14. package/src/ast.ts +5 -0
  15. package/src/bags.ts +5 -2
  16. package/src/builtins.ts +8 -1
  17. package/src/check.ts +193 -128
  18. package/src/cli/run.ts +2 -0
  19. package/src/cli/runtime-invariants.ts +7 -8
  20. package/src/cli/shared.ts +4 -0
  21. package/src/cli/tlc-arguments.ts +30 -24
  22. package/src/cli/tlc-usage.ts +3 -2
  23. package/src/evaluate.ts +102 -26
  24. package/src/java-overrides.ts +9 -4
  25. package/src/json-reader.ts +2 -1
  26. package/src/json.ts +11 -4
  27. package/src/levels.ts +15 -7
  28. package/src/lexer.ts +83 -72
  29. package/src/liveness-checker.ts +106 -77
  30. package/src/liveness.ts +34 -10
  31. package/src/parallel.ts +33 -26
  32. package/src/parser.ts +226 -163
  33. package/src/pcal/ast.ts +65 -53
  34. package/src/pcal/parse.ts +191 -144
  35. package/src/pcal/tokenize.ts +115 -91
  36. package/src/pcal/translate.ts +98 -83
  37. package/src/pcal/validate.ts +50 -48
  38. package/src/pretty-print.ts +6 -6
  39. package/src/randomization.ts +4 -1
  40. package/src/resolve.ts +70 -12
  41. package/src/runtime.ts +18 -11
  42. package/src/sany/builtins.ts +3 -1
  43. package/src/sany/generator.ts +127 -74
  44. package/src/sany/levels.ts +9 -5
  45. package/src/sany/lexer.ts +44 -40
  46. package/src/sany/linter.ts +1 -0
  47. package/src/search.ts +1 -1
  48. package/src/simulate.ts +59 -50
  49. package/src/spec-description.ts +34 -23
  50. package/src/standard-definitions.ts +3 -0
  51. package/src/standard-modules.ts +12 -1
  52. package/src/state-graph.ts +8 -1
  53. package/src/subexpressions.ts +47 -38
  54. package/src/successors.ts +35 -14
  55. package/src/symmetry.ts +19 -4
  56. package/src/tlc/fingerprint.ts +21 -15
  57. package/src/tlc/jar-data.ts +1 -0
  58. package/src/tlc/messages.ts +21 -2
  59. package/src/tlc/order.ts +38 -20
  60. package/src/tlc/random.ts +30 -18
  61. package/src/tlc/replay.ts +33 -13
  62. package/src/tlc/summary.ts +45 -36
  63. package/src/tlc/tlc-operators.ts +14 -4
  64. package/src/tlc/tlcext-operators.ts +15 -11
  65. package/src/tlc/trace-spec.ts +4 -0
  66. package/src/tlc/value-stream.ts +13 -9
  67. package/src/tlc/version.ts +1 -0
  68. package/src/values.ts +181 -74
package/src/levels.ts CHANGED
@@ -193,6 +193,8 @@ const createLevelError = (message: string, expression: Expression) =>
193
193
 
194
194
  const LOGICAL_OPERATORS = new Set(["/\\", "\\/", "=>", "<=>"]);
195
195
 
196
+ // SANY's OpApplNode.levelCheck rejects raw actions inside temporal formulas, but not inside a LET
197
+ // or a label.
196
198
  const isOperatorApplication = (expression: Expression) =>
197
199
  expression.kind !== "Let" && expression.kind !== "Label";
198
200
 
@@ -212,6 +214,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
212
214
  const definition = context.getDefinition(name);
213
215
  if (definition) {
214
216
  const levels = analyzeDefinition(definition, new Map());
217
+ // SANY names a definition of an instance by its qualified name, as in I!Foo.
215
218
  const qualifiedName = context.getQualifiedName?.(name);
216
219
  return {
217
220
  kind: "Defined",
@@ -271,7 +274,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
271
274
  Math.max(
272
275
  operator.level,
273
276
  ...operands
274
- .filter((_operand, index) => getWeight(operator, index) === 1)
277
+ .filter((_, index) => getWeight(operator, index) === 1)
275
278
  .map((operand) => operand.data.level),
276
279
  ...ranges.map((range) => range.level),
277
280
  ),
@@ -303,7 +306,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
303
306
  const argument = operand.operator;
304
307
  if (!argument) return;
305
308
  operands.forEach((other, otherIndex) =>
306
- argument.formals.forEach((_formal, argumentIndex) => {
309
+ argument.formals.forEach((_, argumentIndex) => {
307
310
  if (!hasCondition(operandIndex, otherIndex, argumentIndex)) return;
308
311
  other.data.params.forEach((param) =>
309
312
  putConstraint(data.constraints, param, getMaxLevel(argument, argumentIndex)),
@@ -330,7 +333,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
330
333
  operands.forEach((operand, operandIndex) => {
331
334
  const declared = operand.declared;
332
335
  if (!declared) return;
333
- Array.from({ length: declared.arity }, (_value, argumentIndex) =>
336
+ Array.from({ length: declared.arity }, (_, argumentIndex) =>
334
337
  putArgConstraint(
335
338
  data.argConstraints,
336
339
  getArgumentKey(declared.name, argumentIndex),
@@ -338,7 +341,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
338
341
  ),
339
342
  );
340
343
  operands.forEach((other, otherIndex) =>
341
- Array.from({ length: declared.arity }, (_value, argumentIndex) => {
344
+ Array.from({ length: declared.arity }, (_, argumentIndex) => {
342
345
  if (hasCondition(operandIndex, otherIndex, argumentIndex)) {
343
346
  putArgConstraint(
344
347
  data.argConstraints,
@@ -391,7 +394,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
391
394
  const declared = operand.declared;
392
395
  if (!declared) return;
393
396
  operands.forEach((other, otherIndex) =>
394
- Array.from({ length: declared.arity }, (_value, argumentIndex) => {
397
+ Array.from({ length: declared.arity }, (_, argumentIndex) => {
395
398
  if (!hasCondition(operandIndex, otherIndex, argumentIndex)) return;
396
399
  other.data.params.forEach((param) =>
397
400
  data.argParams.add(getArgumentParamKey(declared.name, argumentIndex, param)),
@@ -520,6 +523,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
520
523
  return data;
521
524
  };
522
525
 
526
+ // SANY reports a level error of an operator's arguments at the application.
523
527
  const analyze = (expression: Expression, scope: Scope): LevelData => {
524
528
  try {
525
529
  return analyzeNode(expression, scope);
@@ -685,6 +689,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
685
689
  ? toOperand(expression.args[0], scope)
686
690
  : { data: applyBuiltin(SET_LEVELS, expression.args, scope) },
687
691
  ]);
692
+ // SANY's semantic checker rejects the expression without computing its level.
688
693
  case "Bracket":
689
694
  return createData();
690
695
  case "FieldAccess":
@@ -731,6 +736,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
731
736
  case "Apply":
732
737
  visitOperator(expression.operator, expression.args);
733
738
  return names;
739
+ // SANY's LetInNode and OpArgNode don't carry the non-Leibniz parameters of their bodies.
734
740
  case "Lambda":
735
741
  case "Let":
736
742
  return names;
@@ -785,13 +791,13 @@ export const createLevelAnalyzer = (context: LevelContext) => {
785
791
  minMaxLevels: formals.map((formal) =>
786
792
  Array.from(
787
793
  { length: formal.arity },
788
- (_value, index) => data.argConstraints.get(getArgumentKey(formal.name, index)) ?? 0,
794
+ (_, index) => data.argConstraints.get(getArgumentKey(formal.name, index)) ?? 0,
789
795
  ),
790
796
  ),
791
797
  conditions: new Set(
792
798
  formals.flatMap((operator, operatorIndex) =>
793
799
  formals.flatMap((param, paramIndex) =>
794
- Array.from({ length: operator.arity }, (_value, argumentIndex) => argumentIndex)
800
+ Array.from({ length: operator.arity }, (_, argumentIndex) => argumentIndex)
795
801
  .filter((argumentIndex) =>
796
802
  data.argParams.has(getArgumentParamKey(operator.name, argumentIndex, param.name)),
797
803
  )
@@ -898,6 +904,7 @@ export const checkModuleLevels = (
898
904
  ? analyzeAssumeProve(body.statement, new Map(), analyzer).level
899
905
  : getLevel([body.statement]);
900
906
  };
907
+ // The proof of a temporal goal may only assume constants.
901
908
  const checkTemporalGoal = (proof: Proof | undefined): SanyError[] =>
902
909
  proof?.kind !== "Steps"
903
910
  ? []
@@ -923,6 +930,7 @@ export const checkModuleLevels = (
923
930
  }
924
931
  return step.body.kind === "Qed" ? checkTemporalGoal(step.proof) : [];
925
932
  });
933
+ // SANY checks each step as a theorem of its own after the steps of its proof.
926
934
  const checkProof = (proof: Proof | undefined): SanyError[] =>
927
935
  proof?.kind !== "Steps"
928
936
  ? []
package/src/lexer.ts CHANGED
@@ -12,6 +12,7 @@ type TokenKind =
12
12
  | "ModuleEnd"
13
13
  | "End";
14
14
 
15
+ // spelling holds the text of an operator written as a synonym, such as (+) for \oplus.
15
16
  export interface Token {
16
17
  kind: TokenKind;
17
18
  text: string;
@@ -318,6 +319,7 @@ const NUMBER_BASES = new Map([
318
319
  ["h", 16],
319
320
  ]);
320
321
 
322
+ // A numeral token keeps its text, such as \hFF, so errors can quote it as TLC does.
321
323
  export const getNumeralValue = (text: string) =>
322
324
  text.startsWith("\\")
323
325
  ? parseInt(text.slice(2), NUMBER_BASES.get(text[1].toLowerCase()))
@@ -363,6 +365,7 @@ new Map([
363
365
  }
364
366
  });
365
367
 
368
+ // charCodeAt past the end of a source gives NaN, which is in no class.
366
369
  const isInClass = (code: number, characterClass: number) =>
367
370
  code < 128 && (CHARACTER_CLASSES[code] & characterClass) !== 0;
368
371
 
@@ -377,6 +380,7 @@ interface OperatorMatch {
377
380
  length: number;
378
381
  }
379
382
 
383
+ // In the order of SYMBOL_OPERATORS, which lists a longer operator before its prefixes.
380
384
  const SYMBOL_OPERATORS_BY_START = new Map<number, string[]>();
381
385
  SYMBOL_OPERATORS.forEach((symbol) => {
382
386
  const start = symbol.charCodeAt(0);
@@ -406,10 +410,13 @@ export const matchOperator = (source: string, index: number): OperatorMatch | un
406
410
  const code = source.charCodeAt(index);
407
411
  if (code === CODE_BACKSLASH && isInClass(source.charCodeAt(index + 1), LETTER)) {
408
412
  const word = source.slice(index + 1, skipClass(source, index + 1, LETTER));
409
- let length = word.length;
410
- while (length > 0 && !BACKSLASH_OPERATORS.has(word.slice(0, length))) length--;
411
- const operator = BACKSLASH_OPERATORS.get(word.slice(0, length));
412
- return { text: operator ?? "\\", length: operator === undefined ? 1 : length + 1 };
413
+ // Like SANY, read the longest operator that the word starts with, so S\used is S \ used.
414
+ const length = { value: word.length };
415
+ while (length.value > 0 && !BACKSLASH_OPERATORS.has(word.slice(0, length.value))) {
416
+ length.value--;
417
+ }
418
+ const operator = BACKSLASH_OPERATORS.get(word.slice(0, length.value));
419
+ return { text: operator ?? "\\", length: operator === undefined ? 1 : length.value + 1 };
413
420
  }
414
421
  const unicodeSymbol = matchUnicodeOperator(source, index);
415
422
  if (unicodeSymbol !== undefined) {
@@ -420,6 +427,7 @@ export const matchOperator = (source: string, index: number): OperatorMatch | un
420
427
  return code === CODE_BACKSLASH ? { text: "\\", length: 1 } : undefined;
421
428
  };
422
429
 
430
+ // TLC's config parser reads ℕ, ℤ, ℝ, or a Unicode operator as one token, compared as written.
423
431
  const UNICODE_NAMES = new Set(["ℕ", "ℤ", "ℝ"]);
424
432
  export const matchUnicodeToken = (source: string, index: number): OperatorMatch | undefined => {
425
433
  const token = [source.slice(index, index + 2), source[index]].find(
@@ -433,11 +441,12 @@ export const skipWhile = (
433
441
  index: number,
434
442
  isMatch: (character: string) => boolean,
435
443
  ) => {
436
- let end = index;
437
- while (end < source.length && isMatch(source[end])) end++;
438
- return end;
444
+ const end = { value: index };
445
+ while (end.value < source.length && isMatch(source[end.value])) end.value++;
446
+ return end.value;
439
447
  };
440
448
 
449
+ // SANY reads a step number as one token anywhere, and tlapm reads no name after <*> or <+>.
441
450
  const matchStep = (source: string, index: number, isTlapm: boolean) => {
442
451
  const isImplicit = source[index + 1] === "+" || source[index + 1] === "*";
443
452
  const levelEnd = isImplicit ? index + 2 : skipClass(source, index + 1, DIGIT);
@@ -451,6 +460,7 @@ const matchStep = (source: string, index: number, isTlapm: boolean) => {
451
460
  );
452
461
  };
453
462
 
463
+ // SANY's lexer reads @ as an identifier, which stands for the value an EXCEPT changes only alone.
454
464
  export const readAtAsIdentifier = (token: Token): Token =>
455
465
  token.kind === "Operator" && token.text === "@" ? { ...token, kind: "Identifier" } : token;
456
466
 
@@ -483,24 +493,24 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
483
493
  // Each name is one string object, so looking a name up in a scope compares pointers.
484
494
  const names = new Map<string, string>();
485
495
  const prefixLines = source.slice(0, headerMatch.index).split("\n");
486
- let index = headerMatch.index;
487
- let line = prefixLines.length;
488
- let lineStart = headerMatch.index - prefixLines[prefixLines.length - 1].length;
489
- let moduleDepth = 0;
490
-
491
- let scanLineStart = -1;
492
- let scanPosition = 0;
493
- let scanWidth = 0;
496
+ const cursor = {
497
+ index: headerMatch.index,
498
+ line: prefixLines.length,
499
+ lineStart: headerMatch.index - prefixLines[prefixLines.length - 1].length,
500
+ moduleDepth: 0,
501
+ };
502
+
503
+ const scan = { lineStart: -1, position: 0, width: 0 };
494
504
  const getColumn = () => {
495
- if (scanLineStart !== lineStart || scanPosition > index) {
496
- scanLineStart = lineStart;
497
- scanPosition = lineStart;
498
- scanWidth = 0;
505
+ if (scan.lineStart !== cursor.lineStart || scan.position > cursor.index) {
506
+ scan.lineStart = cursor.lineStart;
507
+ scan.position = cursor.lineStart;
508
+ scan.width = 0;
499
509
  }
500
- for (; scanPosition < index; scanPosition++) {
501
- scanWidth = getWidthAfter(scanWidth, source.charCodeAt(scanPosition));
510
+ for (; scan.position < cursor.index; scan.position++) {
511
+ scan.width = getWidthAfter(scan.width, source.charCodeAt(scan.position));
502
512
  }
503
- return getWidthAfter(scanWidth, source.charCodeAt(index));
513
+ return getWidthAfter(scan.width, source.charCodeAt(cursor.index));
504
514
  };
505
515
  const pushToken = (kind: TokenKind, text: string, length: number, spelling?: string) => {
506
516
  const column = getColumn();
@@ -508,83 +518,83 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
508
518
  kind,
509
519
  text,
510
520
  spelling,
511
- line,
521
+ line: cursor.line,
512
522
  column,
513
523
  endColumn: column + length - 1,
514
524
  });
515
- index += length;
525
+ cursor.index += length;
516
526
  };
517
527
  const advanceOver = (length: number) => {
518
528
  for (let offset = 0; offset < length; offset++) {
519
- if (source.charCodeAt(index) === CODE_NEWLINE) {
520
- line++;
521
- lineStart = index + 1;
529
+ if (source.charCodeAt(cursor.index) === CODE_NEWLINE) {
530
+ cursor.line++;
531
+ cursor.lineStart = cursor.index + 1;
522
532
  }
523
- index++;
533
+ cursor.index++;
524
534
  }
525
535
  };
526
536
  const createError = (message: string) =>
527
- new TlaSyntaxError(`${message} at line ${line}, column ${getColumn()}`, undefined, {
537
+ new TlaSyntaxError(`${message} at line ${cursor.line}, column ${getColumn()}`, undefined, {
528
538
  reason: message,
529
- line,
539
+ line: cursor.line,
530
540
  column: getColumn(),
531
- isEnd: index >= source.length,
541
+ isEnd: cursor.index >= source.length,
532
542
  });
533
543
 
534
- while (index < source.length) {
535
- const code = source.charCodeAt(index);
544
+ while (cursor.index < source.length) {
545
+ const code = source.charCodeAt(cursor.index);
536
546
 
537
547
  if (code === CODE_NEWLINE || code === CODE_SPACE || code === CODE_TAB || code === CODE_RETURN) {
538
548
  advanceOver(1);
539
549
  continue;
540
550
  }
541
- if (code === CODE_BACKSLASH && source.charCodeAt(index + 1) === CODE_STAR) {
542
- const lineEnd = skipWhile(source, index, (next) => next !== "\n" && next !== "\r");
543
- advanceOver(lineEnd - index);
551
+ if (code === CODE_BACKSLASH && source.charCodeAt(cursor.index + 1) === CODE_STAR) {
552
+ const lineEnd = skipWhile(source, cursor.index, (next) => next !== "\n" && next !== "\r");
553
+ advanceOver(lineEnd - cursor.index);
544
554
  continue;
545
555
  }
546
- if (code === CODE_OPEN_PAREN && source.charCodeAt(index + 1) === CODE_STAR) {
547
- let depth = 0;
556
+ if (code === CODE_OPEN_PAREN && source.charCodeAt(cursor.index + 1) === CODE_STAR) {
557
+ const depth = { value: 0 };
548
558
  do {
549
- if (index >= source.length) throw createError("Unterminated comment");
550
- if (source.startsWith("(*", index)) {
551
- depth++;
559
+ if (cursor.index >= source.length) throw createError("Unterminated comment");
560
+ if (source.startsWith("(*", cursor.index)) {
561
+ depth.value++;
552
562
  advanceOver(2);
553
- } else if (source.startsWith("*)", index)) {
554
- depth--;
563
+ } else if (source.startsWith("*)", cursor.index)) {
564
+ depth.value--;
555
565
  advanceOver(2);
556
566
  } else {
557
567
  advanceOver(1);
558
568
  }
559
- } while (depth > 0);
569
+ } while (depth.value > 0);
560
570
  continue;
561
571
  }
562
- if (code === CODE_DASH && source.startsWith("----", index)) {
563
- const length = skipWhile(source, index, (next) => next === "-") - index;
564
- const keyword = skipWhile(source, index + length, (next) => next === " ");
565
- if (source.startsWith("MODULE", keyword)) moduleDepth++;
572
+ if (code === CODE_DASH && source.startsWith("----", cursor.index)) {
573
+ const length = skipWhile(source, cursor.index, (next) => next === "-") - cursor.index;
574
+ const keyword = skipWhile(source, cursor.index + length, (next) => next === " ");
575
+ if (source.startsWith("MODULE", keyword)) cursor.moduleDepth++;
566
576
  pushToken("Separator", "----", length);
567
577
  continue;
568
578
  }
569
- if (code === CODE_EQUALS && source.startsWith("====", index)) {
570
- const length = skipWhile(source, index, (next) => next === "=") - index;
579
+ if (code === CODE_EQUALS && source.startsWith("====", cursor.index)) {
580
+ const length = skipWhile(source, cursor.index, (next) => next === "=") - cursor.index;
571
581
  pushToken("ModuleEnd", "====", length);
572
- moduleDepth--;
573
- if (moduleDepth <= 0) break;
582
+ cursor.moduleDepth--;
583
+ if (cursor.moduleDepth <= 0) break;
574
584
  continue;
575
585
  }
576
586
  if (code === CODE_QUOTE) {
577
- const startLine = line;
587
+ const line = cursor.line;
578
588
  const column = getColumn();
579
589
  const characters: string[] = [];
580
590
  advanceOver(1);
581
- while (source.charCodeAt(index) !== CODE_QUOTE) {
582
- const next = source.charCodeAt(index);
583
- if (index >= source.length || next === CODE_NEWLINE || next === CODE_RETURN) {
591
+ while (source.charCodeAt(cursor.index) !== CODE_QUOTE) {
592
+ const next = source.charCodeAt(cursor.index);
593
+ if (cursor.index >= source.length || next === CODE_NEWLINE || next === CODE_RETURN) {
584
594
  throw createError("Unterminated string");
585
595
  }
586
596
  if (next === CODE_BACKSLASH) {
587
- const escaped = source[index + 1];
597
+ const escaped = source[cursor.index + 1];
588
598
  if (escaped === undefined || !'ntrf\\"'.includes(escaped)) {
589
599
  throw createError("Invalid escape in string");
590
600
  }
@@ -592,38 +602,39 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
592
602
  advanceOver(2);
593
603
  continue;
594
604
  }
595
- characters.push(source[index]);
605
+ characters.push(source[cursor.index]);
596
606
  advanceOver(1);
597
607
  }
598
608
  advanceOver(1);
599
609
  tokens.push({
600
610
  kind: "String",
601
611
  text: characters.join(""),
602
- line: startLine,
612
+ line,
603
613
  column,
604
614
  endColumn: getColumn() - 1,
605
615
  });
606
616
  continue;
607
617
  }
608
- const basedLength = matchBasedNumber(source, index);
618
+ const basedLength = matchBasedNumber(source, cursor.index);
609
619
  if (basedLength > 0) {
610
- pushToken("Number", source.slice(index, index + basedLength), basedLength);
620
+ pushToken("Number", source.slice(cursor.index, cursor.index + basedLength), basedLength);
611
621
  continue;
612
622
  }
613
623
  if (isInClass(code, IDENTIFIER)) {
614
- const wordEnd = skipClass(source, index, IDENTIFIER);
615
- const word = source.slice(index, wordEnd);
616
- const digitsEnd = skipClass(source, index, DIGIT);
624
+ const wordEnd = skipClass(source, cursor.index, IDENTIFIER);
625
+ const word = source.slice(cursor.index, wordEnd);
626
+ const digitsEnd = skipClass(source, cursor.index, DIGIT);
617
627
  if (digitsEnd === wordEnd) {
618
628
  const fractionEnd =
619
629
  source.charCodeAt(wordEnd) === CODE_DOT ? skipClass(source, wordEnd + 1, DIGIT) : wordEnd;
620
630
  const isDecimal = fractionEnd > wordEnd + 1;
621
- const number = isDecimal ? source.slice(index, fractionEnd) : word;
631
+ const number = isDecimal ? source.slice(cursor.index, fractionEnd) : word;
622
632
  pushToken(isDecimal ? "Decimal" : "Number", number, number.length);
623
633
  continue;
624
634
  }
635
+ // A word needs a letter to be a name, so SANY reads one without as underscores and numbers.
625
636
  if (!/[A-Za-z]/.test(word)) {
626
- const part = code === CODE_UNDERSCORE ? "_" : source.slice(index, digitsEnd);
637
+ const part = code === CODE_UNDERSCORE ? "_" : source.slice(cursor.index, digitsEnd);
627
638
  pushToken(part === "_" ? "Operator" : "Number", part, part.length);
628
639
  continue;
629
640
  }
@@ -636,22 +647,22 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
636
647
  pushToken(KEYWORDS.has(word) ? "Keyword" : "Identifier", name, word.length);
637
648
  continue;
638
649
  }
639
- const character = source[index];
650
+ const character = source[cursor.index];
640
651
  if (code >= 128 && UNICODE_NAMES.has(character)) {
641
652
  pushToken("Identifier", names.get(character) ?? character, 1);
642
653
  names.set(character, character);
643
654
  continue;
644
655
  }
645
656
  if (code === CODE_LESS_THAN) {
646
- const stepLength = matchStep(source, index, isTlapm);
657
+ const stepLength = matchStep(source, cursor.index, isTlapm);
647
658
  if (stepLength > 0) {
648
- pushToken("Step", source.slice(index, index + stepLength), stepLength);
659
+ pushToken("Step", source.slice(cursor.index, cursor.index + stepLength), stepLength);
649
660
  continue;
650
661
  }
651
662
  }
652
- const operator = matchOperator(source, index);
663
+ const operator = matchOperator(source, cursor.index);
653
664
  if (operator) {
654
- const spelling = source.slice(index, index + operator.length);
665
+ const spelling = source.slice(cursor.index, cursor.index + operator.length);
655
666
  pushToken(
656
667
  "Operator",
657
668
  operator.text,
@@ -664,6 +675,6 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
664
675
  }
665
676
 
666
677
  const endColumn = getColumn();
667
- tokens.push({ kind: "End", text: "", line, column: endColumn, endColumn });
678
+ tokens.push({ kind: "End", text: "", line: cursor.line, column: endColumn, endColumn });
668
679
  return tokens;
669
680
  };