tla-ts 0.0.10-dev.28c823e4b → 0.0.10-dev.b4a856304
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +0 -34
- package/dist/index.d.ts +0 -34
- package/dist/index.js +1 -1
- package/dist/parallel-worker.cjs +1 -1
- package/dist/parallel-worker.js +1 -1
- package/dist/prove.cjs +43 -43
- package/dist/prove.js +47 -47
- package/dist/run.cjs +30 -30
- package/dist/run.js +33 -33
- package/dist/transfer.cjs +38 -38
- package/dist/transfer.js +39 -39
- package/package.json +1 -1
- package/src/ast.ts +5 -0
- package/src/bags.ts +5 -2
- package/src/builtins.ts +8 -1
- package/src/check.ts +193 -128
- package/src/cli/run.ts +2 -0
- package/src/cli/runtime-invariants.ts +7 -8
- package/src/cli/shared.ts +4 -0
- package/src/cli/tlc-arguments.ts +30 -24
- package/src/cli/tlc-usage.ts +3 -2
- package/src/evaluate.ts +102 -26
- package/src/java-overrides.ts +9 -4
- package/src/json-reader.ts +2 -1
- package/src/json.ts +11 -4
- package/src/levels.ts +15 -7
- package/src/lexer.ts +83 -72
- package/src/liveness-checker.ts +106 -77
- package/src/liveness.ts +34 -10
- package/src/parallel.ts +33 -26
- package/src/parser.ts +226 -163
- package/src/pcal/ast.ts +65 -53
- package/src/pcal/parse.ts +191 -144
- package/src/pcal/tokenize.ts +115 -91
- package/src/pcal/translate.ts +98 -83
- package/src/pcal/validate.ts +50 -48
- package/src/pretty-print.ts +6 -6
- package/src/randomization.ts +4 -1
- package/src/resolve.ts +70 -12
- package/src/runtime.ts +18 -11
- package/src/sany/builtins.ts +3 -1
- package/src/sany/generator.ts +127 -74
- package/src/sany/levels.ts +9 -5
- package/src/sany/lexer.ts +44 -40
- package/src/sany/linter.ts +1 -0
- package/src/search.ts +1 -1
- package/src/simulate.ts +59 -50
- package/src/spec-description.ts +34 -23
- package/src/standard-definitions.ts +3 -0
- package/src/standard-modules.ts +12 -1
- package/src/state-graph.ts +8 -1
- package/src/subexpressions.ts +47 -38
- package/src/successors.ts +35 -14
- package/src/symmetry.ts +19 -4
- package/src/tlc/fingerprint.ts +21 -15
- package/src/tlc/jar-data.ts +1 -0
- package/src/tlc/messages.ts +21 -2
- package/src/tlc/order.ts +38 -20
- package/src/tlc/random.ts +30 -18
- package/src/tlc/replay.ts +33 -13
- package/src/tlc/summary.ts +45 -36
- package/src/tlc/tlc-operators.ts +14 -4
- package/src/tlc/tlcext-operators.ts +15 -11
- package/src/tlc/trace-spec.ts +4 -0
- package/src/tlc/value-stream.ts +13 -9
- package/src/tlc/version.ts +1 -0
- package/src/values.ts +181 -74
package/src/levels.ts
CHANGED
|
@@ -193,6 +193,8 @@ const createLevelError = (message: string, expression: Expression) =>
|
|
|
193
193
|
|
|
194
194
|
const LOGICAL_OPERATORS = new Set(["/\\", "\\/", "=>", "<=>"]);
|
|
195
195
|
|
|
196
|
+
// SANY's OpApplNode.levelCheck rejects raw actions inside temporal formulas, but not inside a LET
|
|
197
|
+
// or a label.
|
|
196
198
|
const isOperatorApplication = (expression: Expression) =>
|
|
197
199
|
expression.kind !== "Let" && expression.kind !== "Label";
|
|
198
200
|
|
|
@@ -212,6 +214,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
212
214
|
const definition = context.getDefinition(name);
|
|
213
215
|
if (definition) {
|
|
214
216
|
const levels = analyzeDefinition(definition, new Map());
|
|
217
|
+
// SANY names a definition of an instance by its qualified name, as in I!Foo.
|
|
215
218
|
const qualifiedName = context.getQualifiedName?.(name);
|
|
216
219
|
return {
|
|
217
220
|
kind: "Defined",
|
|
@@ -271,7 +274,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
271
274
|
Math.max(
|
|
272
275
|
operator.level,
|
|
273
276
|
...operands
|
|
274
|
-
.filter((
|
|
277
|
+
.filter((_, index) => getWeight(operator, index) === 1)
|
|
275
278
|
.map((operand) => operand.data.level),
|
|
276
279
|
...ranges.map((range) => range.level),
|
|
277
280
|
),
|
|
@@ -303,7 +306,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
303
306
|
const argument = operand.operator;
|
|
304
307
|
if (!argument) return;
|
|
305
308
|
operands.forEach((other, otherIndex) =>
|
|
306
|
-
argument.formals.forEach((
|
|
309
|
+
argument.formals.forEach((_, argumentIndex) => {
|
|
307
310
|
if (!hasCondition(operandIndex, otherIndex, argumentIndex)) return;
|
|
308
311
|
other.data.params.forEach((param) =>
|
|
309
312
|
putConstraint(data.constraints, param, getMaxLevel(argument, argumentIndex)),
|
|
@@ -330,7 +333,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
330
333
|
operands.forEach((operand, operandIndex) => {
|
|
331
334
|
const declared = operand.declared;
|
|
332
335
|
if (!declared) return;
|
|
333
|
-
Array.from({ length: declared.arity }, (
|
|
336
|
+
Array.from({ length: declared.arity }, (_, argumentIndex) =>
|
|
334
337
|
putArgConstraint(
|
|
335
338
|
data.argConstraints,
|
|
336
339
|
getArgumentKey(declared.name, argumentIndex),
|
|
@@ -338,7 +341,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
338
341
|
),
|
|
339
342
|
);
|
|
340
343
|
operands.forEach((other, otherIndex) =>
|
|
341
|
-
Array.from({ length: declared.arity }, (
|
|
344
|
+
Array.from({ length: declared.arity }, (_, argumentIndex) => {
|
|
342
345
|
if (hasCondition(operandIndex, otherIndex, argumentIndex)) {
|
|
343
346
|
putArgConstraint(
|
|
344
347
|
data.argConstraints,
|
|
@@ -391,7 +394,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
391
394
|
const declared = operand.declared;
|
|
392
395
|
if (!declared) return;
|
|
393
396
|
operands.forEach((other, otherIndex) =>
|
|
394
|
-
Array.from({ length: declared.arity }, (
|
|
397
|
+
Array.from({ length: declared.arity }, (_, argumentIndex) => {
|
|
395
398
|
if (!hasCondition(operandIndex, otherIndex, argumentIndex)) return;
|
|
396
399
|
other.data.params.forEach((param) =>
|
|
397
400
|
data.argParams.add(getArgumentParamKey(declared.name, argumentIndex, param)),
|
|
@@ -520,6 +523,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
520
523
|
return data;
|
|
521
524
|
};
|
|
522
525
|
|
|
526
|
+
// SANY reports a level error of an operator's arguments at the application.
|
|
523
527
|
const analyze = (expression: Expression, scope: Scope): LevelData => {
|
|
524
528
|
try {
|
|
525
529
|
return analyzeNode(expression, scope);
|
|
@@ -685,6 +689,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
685
689
|
? toOperand(expression.args[0], scope)
|
|
686
690
|
: { data: applyBuiltin(SET_LEVELS, expression.args, scope) },
|
|
687
691
|
]);
|
|
692
|
+
// SANY's semantic checker rejects the expression without computing its level.
|
|
688
693
|
case "Bracket":
|
|
689
694
|
return createData();
|
|
690
695
|
case "FieldAccess":
|
|
@@ -731,6 +736,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
731
736
|
case "Apply":
|
|
732
737
|
visitOperator(expression.operator, expression.args);
|
|
733
738
|
return names;
|
|
739
|
+
// SANY's LetInNode and OpArgNode don't carry the non-Leibniz parameters of their bodies.
|
|
734
740
|
case "Lambda":
|
|
735
741
|
case "Let":
|
|
736
742
|
return names;
|
|
@@ -785,13 +791,13 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
785
791
|
minMaxLevels: formals.map((formal) =>
|
|
786
792
|
Array.from(
|
|
787
793
|
{ length: formal.arity },
|
|
788
|
-
(
|
|
794
|
+
(_, index) => data.argConstraints.get(getArgumentKey(formal.name, index)) ?? 0,
|
|
789
795
|
),
|
|
790
796
|
),
|
|
791
797
|
conditions: new Set(
|
|
792
798
|
formals.flatMap((operator, operatorIndex) =>
|
|
793
799
|
formals.flatMap((param, paramIndex) =>
|
|
794
|
-
Array.from({ length: operator.arity }, (
|
|
800
|
+
Array.from({ length: operator.arity }, (_, argumentIndex) => argumentIndex)
|
|
795
801
|
.filter((argumentIndex) =>
|
|
796
802
|
data.argParams.has(getArgumentParamKey(operator.name, argumentIndex, param.name)),
|
|
797
803
|
)
|
|
@@ -898,6 +904,7 @@ export const checkModuleLevels = (
|
|
|
898
904
|
? analyzeAssumeProve(body.statement, new Map(), analyzer).level
|
|
899
905
|
: getLevel([body.statement]);
|
|
900
906
|
};
|
|
907
|
+
// The proof of a temporal goal may only assume constants.
|
|
901
908
|
const checkTemporalGoal = (proof: Proof | undefined): SanyError[] =>
|
|
902
909
|
proof?.kind !== "Steps"
|
|
903
910
|
? []
|
|
@@ -923,6 +930,7 @@ export const checkModuleLevels = (
|
|
|
923
930
|
}
|
|
924
931
|
return step.body.kind === "Qed" ? checkTemporalGoal(step.proof) : [];
|
|
925
932
|
});
|
|
933
|
+
// SANY checks each step as a theorem of its own after the steps of its proof.
|
|
926
934
|
const checkProof = (proof: Proof | undefined): SanyError[] =>
|
|
927
935
|
proof?.kind !== "Steps"
|
|
928
936
|
? []
|
package/src/lexer.ts
CHANGED
|
@@ -12,6 +12,7 @@ type TokenKind =
|
|
|
12
12
|
| "ModuleEnd"
|
|
13
13
|
| "End";
|
|
14
14
|
|
|
15
|
+
// spelling holds the text of an operator written as a synonym, such as (+) for \oplus.
|
|
15
16
|
export interface Token {
|
|
16
17
|
kind: TokenKind;
|
|
17
18
|
text: string;
|
|
@@ -318,6 +319,7 @@ const NUMBER_BASES = new Map([
|
|
|
318
319
|
["h", 16],
|
|
319
320
|
]);
|
|
320
321
|
|
|
322
|
+
// A numeral token keeps its text, such as \hFF, so errors can quote it as TLC does.
|
|
321
323
|
export const getNumeralValue = (text: string) =>
|
|
322
324
|
text.startsWith("\\")
|
|
323
325
|
? parseInt(text.slice(2), NUMBER_BASES.get(text[1].toLowerCase()))
|
|
@@ -363,6 +365,7 @@ new Map([
|
|
|
363
365
|
}
|
|
364
366
|
});
|
|
365
367
|
|
|
368
|
+
// charCodeAt past the end of a source gives NaN, which is in no class.
|
|
366
369
|
const isInClass = (code: number, characterClass: number) =>
|
|
367
370
|
code < 128 && (CHARACTER_CLASSES[code] & characterClass) !== 0;
|
|
368
371
|
|
|
@@ -377,6 +380,7 @@ interface OperatorMatch {
|
|
|
377
380
|
length: number;
|
|
378
381
|
}
|
|
379
382
|
|
|
383
|
+
// In the order of SYMBOL_OPERATORS, which lists a longer operator before its prefixes.
|
|
380
384
|
const SYMBOL_OPERATORS_BY_START = new Map<number, string[]>();
|
|
381
385
|
SYMBOL_OPERATORS.forEach((symbol) => {
|
|
382
386
|
const start = symbol.charCodeAt(0);
|
|
@@ -406,10 +410,13 @@ export const matchOperator = (source: string, index: number): OperatorMatch | un
|
|
|
406
410
|
const code = source.charCodeAt(index);
|
|
407
411
|
if (code === CODE_BACKSLASH && isInClass(source.charCodeAt(index + 1), LETTER)) {
|
|
408
412
|
const word = source.slice(index + 1, skipClass(source, index + 1, LETTER));
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
+
// Like SANY, read the longest operator that the word starts with, so S\used is S \ used.
|
|
414
|
+
const length = { value: word.length };
|
|
415
|
+
while (length.value > 0 && !BACKSLASH_OPERATORS.has(word.slice(0, length.value))) {
|
|
416
|
+
length.value--;
|
|
417
|
+
}
|
|
418
|
+
const operator = BACKSLASH_OPERATORS.get(word.slice(0, length.value));
|
|
419
|
+
return { text: operator ?? "\\", length: operator === undefined ? 1 : length.value + 1 };
|
|
413
420
|
}
|
|
414
421
|
const unicodeSymbol = matchUnicodeOperator(source, index);
|
|
415
422
|
if (unicodeSymbol !== undefined) {
|
|
@@ -420,6 +427,7 @@ export const matchOperator = (source: string, index: number): OperatorMatch | un
|
|
|
420
427
|
return code === CODE_BACKSLASH ? { text: "\\", length: 1 } : undefined;
|
|
421
428
|
};
|
|
422
429
|
|
|
430
|
+
// TLC's config parser reads ℕ, ℤ, ℝ, or a Unicode operator as one token, compared as written.
|
|
423
431
|
const UNICODE_NAMES = new Set(["ℕ", "ℤ", "ℝ"]);
|
|
424
432
|
export const matchUnicodeToken = (source: string, index: number): OperatorMatch | undefined => {
|
|
425
433
|
const token = [source.slice(index, index + 2), source[index]].find(
|
|
@@ -433,11 +441,12 @@ export const skipWhile = (
|
|
|
433
441
|
index: number,
|
|
434
442
|
isMatch: (character: string) => boolean,
|
|
435
443
|
) => {
|
|
436
|
-
|
|
437
|
-
while (end < source.length && isMatch(source[end])) end++;
|
|
438
|
-
return end;
|
|
444
|
+
const end = { value: index };
|
|
445
|
+
while (end.value < source.length && isMatch(source[end.value])) end.value++;
|
|
446
|
+
return end.value;
|
|
439
447
|
};
|
|
440
448
|
|
|
449
|
+
// SANY reads a step number as one token anywhere, and tlapm reads no name after <*> or <+>.
|
|
441
450
|
const matchStep = (source: string, index: number, isTlapm: boolean) => {
|
|
442
451
|
const isImplicit = source[index + 1] === "+" || source[index + 1] === "*";
|
|
443
452
|
const levelEnd = isImplicit ? index + 2 : skipClass(source, index + 1, DIGIT);
|
|
@@ -451,6 +460,7 @@ const matchStep = (source: string, index: number, isTlapm: boolean) => {
|
|
|
451
460
|
);
|
|
452
461
|
};
|
|
453
462
|
|
|
463
|
+
// SANY's lexer reads @ as an identifier, which stands for the value an EXCEPT changes only alone.
|
|
454
464
|
export const readAtAsIdentifier = (token: Token): Token =>
|
|
455
465
|
token.kind === "Operator" && token.text === "@" ? { ...token, kind: "Identifier" } : token;
|
|
456
466
|
|
|
@@ -483,24 +493,24 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
483
493
|
// Each name is one string object, so looking a name up in a scope compares pointers.
|
|
484
494
|
const names = new Map<string, string>();
|
|
485
495
|
const prefixLines = source.slice(0, headerMatch.index).split("\n");
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
496
|
+
const cursor = {
|
|
497
|
+
index: headerMatch.index,
|
|
498
|
+
line: prefixLines.length,
|
|
499
|
+
lineStart: headerMatch.index - prefixLines[prefixLines.length - 1].length,
|
|
500
|
+
moduleDepth: 0,
|
|
501
|
+
};
|
|
502
|
+
|
|
503
|
+
const scan = { lineStart: -1, position: 0, width: 0 };
|
|
494
504
|
const getColumn = () => {
|
|
495
|
-
if (
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
505
|
+
if (scan.lineStart !== cursor.lineStart || scan.position > cursor.index) {
|
|
506
|
+
scan.lineStart = cursor.lineStart;
|
|
507
|
+
scan.position = cursor.lineStart;
|
|
508
|
+
scan.width = 0;
|
|
499
509
|
}
|
|
500
|
-
for (;
|
|
501
|
-
|
|
510
|
+
for (; scan.position < cursor.index; scan.position++) {
|
|
511
|
+
scan.width = getWidthAfter(scan.width, source.charCodeAt(scan.position));
|
|
502
512
|
}
|
|
503
|
-
return getWidthAfter(
|
|
513
|
+
return getWidthAfter(scan.width, source.charCodeAt(cursor.index));
|
|
504
514
|
};
|
|
505
515
|
const pushToken = (kind: TokenKind, text: string, length: number, spelling?: string) => {
|
|
506
516
|
const column = getColumn();
|
|
@@ -508,83 +518,83 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
508
518
|
kind,
|
|
509
519
|
text,
|
|
510
520
|
spelling,
|
|
511
|
-
line,
|
|
521
|
+
line: cursor.line,
|
|
512
522
|
column,
|
|
513
523
|
endColumn: column + length - 1,
|
|
514
524
|
});
|
|
515
|
-
index += length;
|
|
525
|
+
cursor.index += length;
|
|
516
526
|
};
|
|
517
527
|
const advanceOver = (length: number) => {
|
|
518
528
|
for (let offset = 0; offset < length; offset++) {
|
|
519
|
-
if (source.charCodeAt(index) === CODE_NEWLINE) {
|
|
520
|
-
line++;
|
|
521
|
-
lineStart = index + 1;
|
|
529
|
+
if (source.charCodeAt(cursor.index) === CODE_NEWLINE) {
|
|
530
|
+
cursor.line++;
|
|
531
|
+
cursor.lineStart = cursor.index + 1;
|
|
522
532
|
}
|
|
523
|
-
index++;
|
|
533
|
+
cursor.index++;
|
|
524
534
|
}
|
|
525
535
|
};
|
|
526
536
|
const createError = (message: string) =>
|
|
527
|
-
new TlaSyntaxError(`${message} at line ${line}, column ${getColumn()}`, undefined, {
|
|
537
|
+
new TlaSyntaxError(`${message} at line ${cursor.line}, column ${getColumn()}`, undefined, {
|
|
528
538
|
reason: message,
|
|
529
|
-
line,
|
|
539
|
+
line: cursor.line,
|
|
530
540
|
column: getColumn(),
|
|
531
|
-
isEnd: index >= source.length,
|
|
541
|
+
isEnd: cursor.index >= source.length,
|
|
532
542
|
});
|
|
533
543
|
|
|
534
|
-
while (index < source.length) {
|
|
535
|
-
const code = source.charCodeAt(index);
|
|
544
|
+
while (cursor.index < source.length) {
|
|
545
|
+
const code = source.charCodeAt(cursor.index);
|
|
536
546
|
|
|
537
547
|
if (code === CODE_NEWLINE || code === CODE_SPACE || code === CODE_TAB || code === CODE_RETURN) {
|
|
538
548
|
advanceOver(1);
|
|
539
549
|
continue;
|
|
540
550
|
}
|
|
541
|
-
if (code === CODE_BACKSLASH && source.charCodeAt(index + 1) === CODE_STAR) {
|
|
542
|
-
const lineEnd = skipWhile(source, index, (next) => next !== "\n" && next !== "\r");
|
|
543
|
-
advanceOver(lineEnd - index);
|
|
551
|
+
if (code === CODE_BACKSLASH && source.charCodeAt(cursor.index + 1) === CODE_STAR) {
|
|
552
|
+
const lineEnd = skipWhile(source, cursor.index, (next) => next !== "\n" && next !== "\r");
|
|
553
|
+
advanceOver(lineEnd - cursor.index);
|
|
544
554
|
continue;
|
|
545
555
|
}
|
|
546
|
-
if (code === CODE_OPEN_PAREN && source.charCodeAt(index + 1) === CODE_STAR) {
|
|
547
|
-
|
|
556
|
+
if (code === CODE_OPEN_PAREN && source.charCodeAt(cursor.index + 1) === CODE_STAR) {
|
|
557
|
+
const depth = { value: 0 };
|
|
548
558
|
do {
|
|
549
|
-
if (index >= source.length) throw createError("Unterminated comment");
|
|
550
|
-
if (source.startsWith("(*", index)) {
|
|
551
|
-
depth++;
|
|
559
|
+
if (cursor.index >= source.length) throw createError("Unterminated comment");
|
|
560
|
+
if (source.startsWith("(*", cursor.index)) {
|
|
561
|
+
depth.value++;
|
|
552
562
|
advanceOver(2);
|
|
553
|
-
} else if (source.startsWith("*)", index)) {
|
|
554
|
-
depth--;
|
|
563
|
+
} else if (source.startsWith("*)", cursor.index)) {
|
|
564
|
+
depth.value--;
|
|
555
565
|
advanceOver(2);
|
|
556
566
|
} else {
|
|
557
567
|
advanceOver(1);
|
|
558
568
|
}
|
|
559
|
-
} while (depth > 0);
|
|
569
|
+
} while (depth.value > 0);
|
|
560
570
|
continue;
|
|
561
571
|
}
|
|
562
|
-
if (code === CODE_DASH && source.startsWith("----", index)) {
|
|
563
|
-
const length = skipWhile(source, index, (next) => next === "-") - index;
|
|
564
|
-
const keyword = skipWhile(source, index + length, (next) => next === " ");
|
|
565
|
-
if (source.startsWith("MODULE", keyword)) moduleDepth++;
|
|
572
|
+
if (code === CODE_DASH && source.startsWith("----", cursor.index)) {
|
|
573
|
+
const length = skipWhile(source, cursor.index, (next) => next === "-") - cursor.index;
|
|
574
|
+
const keyword = skipWhile(source, cursor.index + length, (next) => next === " ");
|
|
575
|
+
if (source.startsWith("MODULE", keyword)) cursor.moduleDepth++;
|
|
566
576
|
pushToken("Separator", "----", length);
|
|
567
577
|
continue;
|
|
568
578
|
}
|
|
569
|
-
if (code === CODE_EQUALS && source.startsWith("====", index)) {
|
|
570
|
-
const length = skipWhile(source, index, (next) => next === "=") - index;
|
|
579
|
+
if (code === CODE_EQUALS && source.startsWith("====", cursor.index)) {
|
|
580
|
+
const length = skipWhile(source, cursor.index, (next) => next === "=") - cursor.index;
|
|
571
581
|
pushToken("ModuleEnd", "====", length);
|
|
572
|
-
moduleDepth--;
|
|
573
|
-
if (moduleDepth <= 0) break;
|
|
582
|
+
cursor.moduleDepth--;
|
|
583
|
+
if (cursor.moduleDepth <= 0) break;
|
|
574
584
|
continue;
|
|
575
585
|
}
|
|
576
586
|
if (code === CODE_QUOTE) {
|
|
577
|
-
const
|
|
587
|
+
const line = cursor.line;
|
|
578
588
|
const column = getColumn();
|
|
579
589
|
const characters: string[] = [];
|
|
580
590
|
advanceOver(1);
|
|
581
|
-
while (source.charCodeAt(index) !== CODE_QUOTE) {
|
|
582
|
-
const next = source.charCodeAt(index);
|
|
583
|
-
if (index >= source.length || next === CODE_NEWLINE || next === CODE_RETURN) {
|
|
591
|
+
while (source.charCodeAt(cursor.index) !== CODE_QUOTE) {
|
|
592
|
+
const next = source.charCodeAt(cursor.index);
|
|
593
|
+
if (cursor.index >= source.length || next === CODE_NEWLINE || next === CODE_RETURN) {
|
|
584
594
|
throw createError("Unterminated string");
|
|
585
595
|
}
|
|
586
596
|
if (next === CODE_BACKSLASH) {
|
|
587
|
-
const escaped = source[index + 1];
|
|
597
|
+
const escaped = source[cursor.index + 1];
|
|
588
598
|
if (escaped === undefined || !'ntrf\\"'.includes(escaped)) {
|
|
589
599
|
throw createError("Invalid escape in string");
|
|
590
600
|
}
|
|
@@ -592,38 +602,39 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
592
602
|
advanceOver(2);
|
|
593
603
|
continue;
|
|
594
604
|
}
|
|
595
|
-
characters.push(source[index]);
|
|
605
|
+
characters.push(source[cursor.index]);
|
|
596
606
|
advanceOver(1);
|
|
597
607
|
}
|
|
598
608
|
advanceOver(1);
|
|
599
609
|
tokens.push({
|
|
600
610
|
kind: "String",
|
|
601
611
|
text: characters.join(""),
|
|
602
|
-
line
|
|
612
|
+
line,
|
|
603
613
|
column,
|
|
604
614
|
endColumn: getColumn() - 1,
|
|
605
615
|
});
|
|
606
616
|
continue;
|
|
607
617
|
}
|
|
608
|
-
const basedLength = matchBasedNumber(source, index);
|
|
618
|
+
const basedLength = matchBasedNumber(source, cursor.index);
|
|
609
619
|
if (basedLength > 0) {
|
|
610
|
-
pushToken("Number", source.slice(index, index + basedLength), basedLength);
|
|
620
|
+
pushToken("Number", source.slice(cursor.index, cursor.index + basedLength), basedLength);
|
|
611
621
|
continue;
|
|
612
622
|
}
|
|
613
623
|
if (isInClass(code, IDENTIFIER)) {
|
|
614
|
-
const wordEnd = skipClass(source, index, IDENTIFIER);
|
|
615
|
-
const word = source.slice(index, wordEnd);
|
|
616
|
-
const digitsEnd = skipClass(source, index, DIGIT);
|
|
624
|
+
const wordEnd = skipClass(source, cursor.index, IDENTIFIER);
|
|
625
|
+
const word = source.slice(cursor.index, wordEnd);
|
|
626
|
+
const digitsEnd = skipClass(source, cursor.index, DIGIT);
|
|
617
627
|
if (digitsEnd === wordEnd) {
|
|
618
628
|
const fractionEnd =
|
|
619
629
|
source.charCodeAt(wordEnd) === CODE_DOT ? skipClass(source, wordEnd + 1, DIGIT) : wordEnd;
|
|
620
630
|
const isDecimal = fractionEnd > wordEnd + 1;
|
|
621
|
-
const number = isDecimal ? source.slice(index, fractionEnd) : word;
|
|
631
|
+
const number = isDecimal ? source.slice(cursor.index, fractionEnd) : word;
|
|
622
632
|
pushToken(isDecimal ? "Decimal" : "Number", number, number.length);
|
|
623
633
|
continue;
|
|
624
634
|
}
|
|
635
|
+
// A word needs a letter to be a name, so SANY reads one without as underscores and numbers.
|
|
625
636
|
if (!/[A-Za-z]/.test(word)) {
|
|
626
|
-
const part = code === CODE_UNDERSCORE ? "_" : source.slice(index, digitsEnd);
|
|
637
|
+
const part = code === CODE_UNDERSCORE ? "_" : source.slice(cursor.index, digitsEnd);
|
|
627
638
|
pushToken(part === "_" ? "Operator" : "Number", part, part.length);
|
|
628
639
|
continue;
|
|
629
640
|
}
|
|
@@ -636,22 +647,22 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
636
647
|
pushToken(KEYWORDS.has(word) ? "Keyword" : "Identifier", name, word.length);
|
|
637
648
|
continue;
|
|
638
649
|
}
|
|
639
|
-
const character = source[index];
|
|
650
|
+
const character = source[cursor.index];
|
|
640
651
|
if (code >= 128 && UNICODE_NAMES.has(character)) {
|
|
641
652
|
pushToken("Identifier", names.get(character) ?? character, 1);
|
|
642
653
|
names.set(character, character);
|
|
643
654
|
continue;
|
|
644
655
|
}
|
|
645
656
|
if (code === CODE_LESS_THAN) {
|
|
646
|
-
const stepLength = matchStep(source, index, isTlapm);
|
|
657
|
+
const stepLength = matchStep(source, cursor.index, isTlapm);
|
|
647
658
|
if (stepLength > 0) {
|
|
648
|
-
pushToken("Step", source.slice(index, index + stepLength), stepLength);
|
|
659
|
+
pushToken("Step", source.slice(cursor.index, cursor.index + stepLength), stepLength);
|
|
649
660
|
continue;
|
|
650
661
|
}
|
|
651
662
|
}
|
|
652
|
-
const operator = matchOperator(source, index);
|
|
663
|
+
const operator = matchOperator(source, cursor.index);
|
|
653
664
|
if (operator) {
|
|
654
|
-
const spelling = source.slice(index, index + operator.length);
|
|
665
|
+
const spelling = source.slice(cursor.index, cursor.index + operator.length);
|
|
655
666
|
pushToken(
|
|
656
667
|
"Operator",
|
|
657
668
|
operator.text,
|
|
@@ -664,6 +675,6 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
664
675
|
}
|
|
665
676
|
|
|
666
677
|
const endColumn = getColumn();
|
|
667
|
-
tokens.push({ kind: "End", text: "", line, column: endColumn, endColumn });
|
|
678
|
+
tokens.push({ kind: "End", text: "", line: cursor.line, column: endColumn, endColumn });
|
|
668
679
|
return tokens;
|
|
669
680
|
};
|