tla-ts 0.0.10-dev.b4a856304 → 0.0.10-dev.f16792fed
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +34 -0
- package/dist/index.d.ts +34 -0
- package/dist/index.js +1 -1
- package/dist/parallel-worker.cjs +1 -1
- package/dist/parallel-worker.js +1 -1
- package/dist/prove.cjs +48 -48
- package/dist/prove.js +29 -29
- package/dist/run.cjs +40 -40
- package/dist/run.js +31 -31
- package/dist/transfer.cjs +36 -36
- package/dist/transfer.js +40 -40
- package/package.json +1 -1
- package/src/ast.ts +0 -5
- package/src/bags.ts +2 -5
- package/src/builtins.ts +1 -8
- package/src/check.ts +128 -193
- package/src/cli/options.ts +1 -1
- package/src/cli/run.ts +0 -2
- package/src/cli/runtime-invariants.ts +8 -7
- package/src/cli/shared.ts +0 -4
- package/src/cli/tlc-arguments.ts +24 -30
- package/src/cli/tlc-usage.ts +2 -3
- package/src/coverage.ts +3 -8
- package/src/evaluate.ts +26 -102
- package/src/java-hash.ts +7 -0
- package/src/java-overrides.ts +4 -9
- package/src/json-reader.ts +1 -2
- package/src/json.ts +4 -11
- package/src/levels.ts +7 -15
- package/src/lexer.ts +72 -83
- package/src/liveness-checker.ts +77 -106
- package/src/liveness.ts +10 -34
- package/src/parallel.ts +26 -33
- package/src/parser.ts +163 -226
- package/src/pcal/ast.ts +53 -65
- package/src/pcal/parse.ts +144 -191
- package/src/pcal/tokenize.ts +91 -115
- package/src/pcal/translate.ts +83 -98
- package/src/pcal/validate.ts +48 -50
- package/src/pretty-print.ts +6 -6
- package/src/randomization.ts +1 -4
- package/src/resolve.ts +12 -70
- package/src/runtime.ts +11 -18
- package/src/sany/builtins.ts +1 -3
- package/src/sany/generator.ts +74 -127
- package/src/sany/levels.ts +5 -9
- package/src/sany/lexer.ts +40 -44
- package/src/sany/linter.ts +0 -1
- package/src/sany/nodes.ts +4 -10
- package/src/search.ts +1 -1
- package/src/simulate.ts +50 -59
- package/src/spec-description.ts +23 -34
- package/src/standard-definitions.ts +0 -3
- package/src/standard-modules.ts +1 -12
- package/src/state-graph.ts +3 -15
- package/src/subexpressions.ts +38 -47
- package/src/successors.ts +14 -35
- package/src/symmetry.ts +4 -19
- package/src/tlc/fingerprint.ts +15 -21
- package/src/tlc/jar-data.ts +0 -1
- package/src/tlc/messages.ts +2 -21
- package/src/tlc/order.ts +22 -47
- package/src/tlc/random.ts +18 -30
- package/src/tlc/replay.ts +13 -33
- package/src/tlc/summary.ts +36 -45
- package/src/tlc/tlc-operators.ts +4 -14
- package/src/tlc/tlcext-operators.ts +11 -15
- package/src/tlc/trace-spec.ts +4 -9
- package/src/tlc/value-stream.ts +9 -13
- package/src/tlc/version.ts +0 -1
- package/src/values.ts +74 -186
package/src/json.ts
CHANGED
|
@@ -98,12 +98,9 @@ const toJson = (value: Value): string => {
|
|
|
98
98
|
if (typeof value === "string") return toJsonString(value);
|
|
99
99
|
if (typeof value !== "object") return String(value);
|
|
100
100
|
if (value.kind === "ModelValue") return toJsonString(value.name);
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (isFunctionClass(className)) return toFunctionJson(value);
|
|
105
|
-
if (isTlcEnumerable(value)) return toArrayJson(toFiniteSet(value).elements);
|
|
106
|
-
throw createUnsupportedError(value);
|
|
101
|
+
return getJavaClassName(value) === "RecordValue"
|
|
102
|
+
? toObjectJson(toFiniteFunction(value))
|
|
103
|
+
: toJsonArray(value);
|
|
107
104
|
};
|
|
108
105
|
|
|
109
106
|
const toJsonArray = (value: Value) => {
|
|
@@ -174,8 +171,7 @@ const checkCharset = (charset: string) => {
|
|
|
174
171
|
);
|
|
175
172
|
};
|
|
176
173
|
|
|
177
|
-
// Files.newBufferedWriter
|
|
178
|
-
// start of a file that it neither truncates nor appends to.
|
|
174
|
+
// Files.newBufferedWriter writes the text over the start of a file that it neither truncates nor appends to.
|
|
179
175
|
const writeWithOptions = (files: FileSystem, path: string, names: string[], text: string) => {
|
|
180
176
|
const unknown = names.find((name) => !OPEN_OPTIONS.has(name));
|
|
181
177
|
if (unknown !== undefined) {
|
|
@@ -212,7 +208,6 @@ const writeWithOptions = (files: FileSystem, path: string, names: string[], text
|
|
|
212
208
|
);
|
|
213
209
|
};
|
|
214
210
|
|
|
215
|
-
// Java's File#getParentFile has no parent for a file name without a directory.
|
|
216
211
|
const makeParentDirectory = (files: FileSystem, path: string) => {
|
|
217
212
|
const separator = path.lastIndexOf("/");
|
|
218
213
|
if (separator !== -1) files.makeDirectory?.(path.slice(0, Math.max(separator, 1)));
|
|
@@ -289,8 +284,6 @@ export const createJsonOperators = (files: FileSystem): Builtin[] => [
|
|
|
289
284
|
),
|
|
290
285
|
];
|
|
291
286
|
|
|
292
|
-
// TLC's Json overrides IOUtils!Serialize, as the CommunityModules define it, for the NDJSON format
|
|
293
|
-
// and falls back to its definition for any other.
|
|
294
287
|
export const createIoUtilsOperators = (
|
|
295
288
|
files: FileSystem,
|
|
296
289
|
evaluateReplaced: (name: string, args: BuiltinArgument[], context: Context | undefined) => Value,
|
package/src/levels.ts
CHANGED
|
@@ -193,8 +193,6 @@ const createLevelError = (message: string, expression: Expression) =>
|
|
|
193
193
|
|
|
194
194
|
const LOGICAL_OPERATORS = new Set(["/\\", "\\/", "=>", "<=>"]);
|
|
195
195
|
|
|
196
|
-
// SANY's OpApplNode.levelCheck rejects raw actions inside temporal formulas, but not inside a LET
|
|
197
|
-
// or a label.
|
|
198
196
|
const isOperatorApplication = (expression: Expression) =>
|
|
199
197
|
expression.kind !== "Let" && expression.kind !== "Label";
|
|
200
198
|
|
|
@@ -214,7 +212,6 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
214
212
|
const definition = context.getDefinition(name);
|
|
215
213
|
if (definition) {
|
|
216
214
|
const levels = analyzeDefinition(definition, new Map());
|
|
217
|
-
// SANY names a definition of an instance by its qualified name, as in I!Foo.
|
|
218
215
|
const qualifiedName = context.getQualifiedName?.(name);
|
|
219
216
|
return {
|
|
220
217
|
kind: "Defined",
|
|
@@ -274,7 +271,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
274
271
|
Math.max(
|
|
275
272
|
operator.level,
|
|
276
273
|
...operands
|
|
277
|
-
.filter((
|
|
274
|
+
.filter((_operand, index) => getWeight(operator, index) === 1)
|
|
278
275
|
.map((operand) => operand.data.level),
|
|
279
276
|
...ranges.map((range) => range.level),
|
|
280
277
|
),
|
|
@@ -306,7 +303,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
306
303
|
const argument = operand.operator;
|
|
307
304
|
if (!argument) return;
|
|
308
305
|
operands.forEach((other, otherIndex) =>
|
|
309
|
-
argument.formals.forEach((
|
|
306
|
+
argument.formals.forEach((_formal, argumentIndex) => {
|
|
310
307
|
if (!hasCondition(operandIndex, otherIndex, argumentIndex)) return;
|
|
311
308
|
other.data.params.forEach((param) =>
|
|
312
309
|
putConstraint(data.constraints, param, getMaxLevel(argument, argumentIndex)),
|
|
@@ -333,7 +330,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
333
330
|
operands.forEach((operand, operandIndex) => {
|
|
334
331
|
const declared = operand.declared;
|
|
335
332
|
if (!declared) return;
|
|
336
|
-
Array.from({ length: declared.arity }, (
|
|
333
|
+
Array.from({ length: declared.arity }, (_value, argumentIndex) =>
|
|
337
334
|
putArgConstraint(
|
|
338
335
|
data.argConstraints,
|
|
339
336
|
getArgumentKey(declared.name, argumentIndex),
|
|
@@ -341,7 +338,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
341
338
|
),
|
|
342
339
|
);
|
|
343
340
|
operands.forEach((other, otherIndex) =>
|
|
344
|
-
Array.from({ length: declared.arity }, (
|
|
341
|
+
Array.from({ length: declared.arity }, (_value, argumentIndex) => {
|
|
345
342
|
if (hasCondition(operandIndex, otherIndex, argumentIndex)) {
|
|
346
343
|
putArgConstraint(
|
|
347
344
|
data.argConstraints,
|
|
@@ -394,7 +391,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
394
391
|
const declared = operand.declared;
|
|
395
392
|
if (!declared) return;
|
|
396
393
|
operands.forEach((other, otherIndex) =>
|
|
397
|
-
Array.from({ length: declared.arity }, (
|
|
394
|
+
Array.from({ length: declared.arity }, (_value, argumentIndex) => {
|
|
398
395
|
if (!hasCondition(operandIndex, otherIndex, argumentIndex)) return;
|
|
399
396
|
other.data.params.forEach((param) =>
|
|
400
397
|
data.argParams.add(getArgumentParamKey(declared.name, argumentIndex, param)),
|
|
@@ -523,7 +520,6 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
523
520
|
return data;
|
|
524
521
|
};
|
|
525
522
|
|
|
526
|
-
// SANY reports a level error of an operator's arguments at the application.
|
|
527
523
|
const analyze = (expression: Expression, scope: Scope): LevelData => {
|
|
528
524
|
try {
|
|
529
525
|
return analyzeNode(expression, scope);
|
|
@@ -689,7 +685,6 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
689
685
|
? toOperand(expression.args[0], scope)
|
|
690
686
|
: { data: applyBuiltin(SET_LEVELS, expression.args, scope) },
|
|
691
687
|
]);
|
|
692
|
-
// SANY's semantic checker rejects the expression without computing its level.
|
|
693
688
|
case "Bracket":
|
|
694
689
|
return createData();
|
|
695
690
|
case "FieldAccess":
|
|
@@ -736,7 +731,6 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
736
731
|
case "Apply":
|
|
737
732
|
visitOperator(expression.operator, expression.args);
|
|
738
733
|
return names;
|
|
739
|
-
// SANY's LetInNode and OpArgNode don't carry the non-Leibniz parameters of their bodies.
|
|
740
734
|
case "Lambda":
|
|
741
735
|
case "Let":
|
|
742
736
|
return names;
|
|
@@ -791,13 +785,13 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
791
785
|
minMaxLevels: formals.map((formal) =>
|
|
792
786
|
Array.from(
|
|
793
787
|
{ length: formal.arity },
|
|
794
|
-
(
|
|
788
|
+
(_value, index) => data.argConstraints.get(getArgumentKey(formal.name, index)) ?? 0,
|
|
795
789
|
),
|
|
796
790
|
),
|
|
797
791
|
conditions: new Set(
|
|
798
792
|
formals.flatMap((operator, operatorIndex) =>
|
|
799
793
|
formals.flatMap((param, paramIndex) =>
|
|
800
|
-
Array.from({ length: operator.arity }, (
|
|
794
|
+
Array.from({ length: operator.arity }, (_value, argumentIndex) => argumentIndex)
|
|
801
795
|
.filter((argumentIndex) =>
|
|
802
796
|
data.argParams.has(getArgumentParamKey(operator.name, argumentIndex, param.name)),
|
|
803
797
|
)
|
|
@@ -904,7 +898,6 @@ export const checkModuleLevels = (
|
|
|
904
898
|
? analyzeAssumeProve(body.statement, new Map(), analyzer).level
|
|
905
899
|
: getLevel([body.statement]);
|
|
906
900
|
};
|
|
907
|
-
// The proof of a temporal goal may only assume constants.
|
|
908
901
|
const checkTemporalGoal = (proof: Proof | undefined): SanyError[] =>
|
|
909
902
|
proof?.kind !== "Steps"
|
|
910
903
|
? []
|
|
@@ -930,7 +923,6 @@ export const checkModuleLevels = (
|
|
|
930
923
|
}
|
|
931
924
|
return step.body.kind === "Qed" ? checkTemporalGoal(step.proof) : [];
|
|
932
925
|
});
|
|
933
|
-
// SANY checks each step as a theorem of its own after the steps of its proof.
|
|
934
926
|
const checkProof = (proof: Proof | undefined): SanyError[] =>
|
|
935
927
|
proof?.kind !== "Steps"
|
|
936
928
|
? []
|
package/src/lexer.ts
CHANGED
|
@@ -12,7 +12,6 @@ type TokenKind =
|
|
|
12
12
|
| "ModuleEnd"
|
|
13
13
|
| "End";
|
|
14
14
|
|
|
15
|
-
// spelling holds the text of an operator written as a synonym, such as (+) for \oplus.
|
|
16
15
|
export interface Token {
|
|
17
16
|
kind: TokenKind;
|
|
18
17
|
text: string;
|
|
@@ -319,7 +318,6 @@ const NUMBER_BASES = new Map([
|
|
|
319
318
|
["h", 16],
|
|
320
319
|
]);
|
|
321
320
|
|
|
322
|
-
// A numeral token keeps its text, such as \hFF, so errors can quote it as TLC does.
|
|
323
321
|
export const getNumeralValue = (text: string) =>
|
|
324
322
|
text.startsWith("\\")
|
|
325
323
|
? parseInt(text.slice(2), NUMBER_BASES.get(text[1].toLowerCase()))
|
|
@@ -365,7 +363,6 @@ new Map([
|
|
|
365
363
|
}
|
|
366
364
|
});
|
|
367
365
|
|
|
368
|
-
// charCodeAt past the end of a source gives NaN, which is in no class.
|
|
369
366
|
const isInClass = (code: number, characterClass: number) =>
|
|
370
367
|
code < 128 && (CHARACTER_CLASSES[code] & characterClass) !== 0;
|
|
371
368
|
|
|
@@ -380,7 +377,6 @@ interface OperatorMatch {
|
|
|
380
377
|
length: number;
|
|
381
378
|
}
|
|
382
379
|
|
|
383
|
-
// In the order of SYMBOL_OPERATORS, which lists a longer operator before its prefixes.
|
|
384
380
|
const SYMBOL_OPERATORS_BY_START = new Map<number, string[]>();
|
|
385
381
|
SYMBOL_OPERATORS.forEach((symbol) => {
|
|
386
382
|
const start = symbol.charCodeAt(0);
|
|
@@ -410,13 +406,10 @@ export const matchOperator = (source: string, index: number): OperatorMatch | un
|
|
|
410
406
|
const code = source.charCodeAt(index);
|
|
411
407
|
if (code === CODE_BACKSLASH && isInClass(source.charCodeAt(index + 1), LETTER)) {
|
|
412
408
|
const word = source.slice(index + 1, skipClass(source, index + 1, LETTER));
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
}
|
|
418
|
-
const operator = BACKSLASH_OPERATORS.get(word.slice(0, length.value));
|
|
419
|
-
return { text: operator ?? "\\", length: operator === undefined ? 1 : length.value + 1 };
|
|
409
|
+
let length = word.length;
|
|
410
|
+
while (length > 0 && !BACKSLASH_OPERATORS.has(word.slice(0, length))) length--;
|
|
411
|
+
const operator = BACKSLASH_OPERATORS.get(word.slice(0, length));
|
|
412
|
+
return { text: operator ?? "\\", length: operator === undefined ? 1 : length + 1 };
|
|
420
413
|
}
|
|
421
414
|
const unicodeSymbol = matchUnicodeOperator(source, index);
|
|
422
415
|
if (unicodeSymbol !== undefined) {
|
|
@@ -427,7 +420,6 @@ export const matchOperator = (source: string, index: number): OperatorMatch | un
|
|
|
427
420
|
return code === CODE_BACKSLASH ? { text: "\\", length: 1 } : undefined;
|
|
428
421
|
};
|
|
429
422
|
|
|
430
|
-
// TLC's config parser reads ℕ, ℤ, ℝ, or a Unicode operator as one token, compared as written.
|
|
431
423
|
const UNICODE_NAMES = new Set(["ℕ", "ℤ", "ℝ"]);
|
|
432
424
|
export const matchUnicodeToken = (source: string, index: number): OperatorMatch | undefined => {
|
|
433
425
|
const token = [source.slice(index, index + 2), source[index]].find(
|
|
@@ -441,12 +433,11 @@ export const skipWhile = (
|
|
|
441
433
|
index: number,
|
|
442
434
|
isMatch: (character: string) => boolean,
|
|
443
435
|
) => {
|
|
444
|
-
|
|
445
|
-
while (end
|
|
446
|
-
return end
|
|
436
|
+
let end = index;
|
|
437
|
+
while (end < source.length && isMatch(source[end])) end++;
|
|
438
|
+
return end;
|
|
447
439
|
};
|
|
448
440
|
|
|
449
|
-
// SANY reads a step number as one token anywhere, and tlapm reads no name after <*> or <+>.
|
|
450
441
|
const matchStep = (source: string, index: number, isTlapm: boolean) => {
|
|
451
442
|
const isImplicit = source[index + 1] === "+" || source[index + 1] === "*";
|
|
452
443
|
const levelEnd = isImplicit ? index + 2 : skipClass(source, index + 1, DIGIT);
|
|
@@ -460,7 +451,6 @@ const matchStep = (source: string, index: number, isTlapm: boolean) => {
|
|
|
460
451
|
);
|
|
461
452
|
};
|
|
462
453
|
|
|
463
|
-
// SANY's lexer reads @ as an identifier, which stands for the value an EXCEPT changes only alone.
|
|
464
454
|
export const readAtAsIdentifier = (token: Token): Token =>
|
|
465
455
|
token.kind === "Operator" && token.text === "@" ? { ...token, kind: "Identifier" } : token;
|
|
466
456
|
|
|
@@ -493,24 +483,24 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
493
483
|
// Each name is one string object, so looking a name up in a scope compares pointers.
|
|
494
484
|
const names = new Map<string, string>();
|
|
495
485
|
const prefixLines = source.slice(0, headerMatch.index).split("\n");
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
486
|
+
let index = headerMatch.index;
|
|
487
|
+
let line = prefixLines.length;
|
|
488
|
+
let lineStart = headerMatch.index - prefixLines[prefixLines.length - 1].length;
|
|
489
|
+
let moduleDepth = 0;
|
|
490
|
+
|
|
491
|
+
let scanLineStart = -1;
|
|
492
|
+
let scanPosition = 0;
|
|
493
|
+
let scanWidth = 0;
|
|
504
494
|
const getColumn = () => {
|
|
505
|
-
if (
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
495
|
+
if (scanLineStart !== lineStart || scanPosition > index) {
|
|
496
|
+
scanLineStart = lineStart;
|
|
497
|
+
scanPosition = lineStart;
|
|
498
|
+
scanWidth = 0;
|
|
509
499
|
}
|
|
510
|
-
for (;
|
|
511
|
-
|
|
500
|
+
for (; scanPosition < index; scanPosition++) {
|
|
501
|
+
scanWidth = getWidthAfter(scanWidth, source.charCodeAt(scanPosition));
|
|
512
502
|
}
|
|
513
|
-
return getWidthAfter(
|
|
503
|
+
return getWidthAfter(scanWidth, source.charCodeAt(index));
|
|
514
504
|
};
|
|
515
505
|
const pushToken = (kind: TokenKind, text: string, length: number, spelling?: string) => {
|
|
516
506
|
const column = getColumn();
|
|
@@ -518,83 +508,83 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
518
508
|
kind,
|
|
519
509
|
text,
|
|
520
510
|
spelling,
|
|
521
|
-
line
|
|
511
|
+
line,
|
|
522
512
|
column,
|
|
523
513
|
endColumn: column + length - 1,
|
|
524
514
|
});
|
|
525
|
-
|
|
515
|
+
index += length;
|
|
526
516
|
};
|
|
527
517
|
const advanceOver = (length: number) => {
|
|
528
518
|
for (let offset = 0; offset < length; offset++) {
|
|
529
|
-
if (source.charCodeAt(
|
|
530
|
-
|
|
531
|
-
|
|
519
|
+
if (source.charCodeAt(index) === CODE_NEWLINE) {
|
|
520
|
+
line++;
|
|
521
|
+
lineStart = index + 1;
|
|
532
522
|
}
|
|
533
|
-
|
|
523
|
+
index++;
|
|
534
524
|
}
|
|
535
525
|
};
|
|
536
526
|
const createError = (message: string) =>
|
|
537
|
-
new TlaSyntaxError(`${message} at line ${
|
|
527
|
+
new TlaSyntaxError(`${message} at line ${line}, column ${getColumn()}`, undefined, {
|
|
538
528
|
reason: message,
|
|
539
|
-
line
|
|
529
|
+
line,
|
|
540
530
|
column: getColumn(),
|
|
541
|
-
isEnd:
|
|
531
|
+
isEnd: index >= source.length,
|
|
542
532
|
});
|
|
543
533
|
|
|
544
|
-
while (
|
|
545
|
-
const code = source.charCodeAt(
|
|
534
|
+
while (index < source.length) {
|
|
535
|
+
const code = source.charCodeAt(index);
|
|
546
536
|
|
|
547
537
|
if (code === CODE_NEWLINE || code === CODE_SPACE || code === CODE_TAB || code === CODE_RETURN) {
|
|
548
538
|
advanceOver(1);
|
|
549
539
|
continue;
|
|
550
540
|
}
|
|
551
|
-
if (code === CODE_BACKSLASH && source.charCodeAt(
|
|
552
|
-
const lineEnd = skipWhile(source,
|
|
553
|
-
advanceOver(lineEnd -
|
|
541
|
+
if (code === CODE_BACKSLASH && source.charCodeAt(index + 1) === CODE_STAR) {
|
|
542
|
+
const lineEnd = skipWhile(source, index, (next) => next !== "\n" && next !== "\r");
|
|
543
|
+
advanceOver(lineEnd - index);
|
|
554
544
|
continue;
|
|
555
545
|
}
|
|
556
|
-
if (code === CODE_OPEN_PAREN && source.charCodeAt(
|
|
557
|
-
|
|
546
|
+
if (code === CODE_OPEN_PAREN && source.charCodeAt(index + 1) === CODE_STAR) {
|
|
547
|
+
let depth = 0;
|
|
558
548
|
do {
|
|
559
|
-
if (
|
|
560
|
-
if (source.startsWith("(*",
|
|
561
|
-
depth
|
|
549
|
+
if (index >= source.length) throw createError("Unterminated comment");
|
|
550
|
+
if (source.startsWith("(*", index)) {
|
|
551
|
+
depth++;
|
|
562
552
|
advanceOver(2);
|
|
563
|
-
} else if (source.startsWith("*)",
|
|
564
|
-
depth
|
|
553
|
+
} else if (source.startsWith("*)", index)) {
|
|
554
|
+
depth--;
|
|
565
555
|
advanceOver(2);
|
|
566
556
|
} else {
|
|
567
557
|
advanceOver(1);
|
|
568
558
|
}
|
|
569
|
-
} while (depth
|
|
559
|
+
} while (depth > 0);
|
|
570
560
|
continue;
|
|
571
561
|
}
|
|
572
|
-
if (code === CODE_DASH && source.startsWith("----",
|
|
573
|
-
const length = skipWhile(source,
|
|
574
|
-
const keyword = skipWhile(source,
|
|
575
|
-
if (source.startsWith("MODULE", keyword))
|
|
562
|
+
if (code === CODE_DASH && source.startsWith("----", index)) {
|
|
563
|
+
const length = skipWhile(source, index, (next) => next === "-") - index;
|
|
564
|
+
const keyword = skipWhile(source, index + length, (next) => next === " ");
|
|
565
|
+
if (source.startsWith("MODULE", keyword)) moduleDepth++;
|
|
576
566
|
pushToken("Separator", "----", length);
|
|
577
567
|
continue;
|
|
578
568
|
}
|
|
579
|
-
if (code === CODE_EQUALS && source.startsWith("====",
|
|
580
|
-
const length = skipWhile(source,
|
|
569
|
+
if (code === CODE_EQUALS && source.startsWith("====", index)) {
|
|
570
|
+
const length = skipWhile(source, index, (next) => next === "=") - index;
|
|
581
571
|
pushToken("ModuleEnd", "====", length);
|
|
582
|
-
|
|
583
|
-
if (
|
|
572
|
+
moduleDepth--;
|
|
573
|
+
if (moduleDepth <= 0) break;
|
|
584
574
|
continue;
|
|
585
575
|
}
|
|
586
576
|
if (code === CODE_QUOTE) {
|
|
587
|
-
const
|
|
577
|
+
const startLine = line;
|
|
588
578
|
const column = getColumn();
|
|
589
579
|
const characters: string[] = [];
|
|
590
580
|
advanceOver(1);
|
|
591
|
-
while (source.charCodeAt(
|
|
592
|
-
const next = source.charCodeAt(
|
|
593
|
-
if (
|
|
581
|
+
while (source.charCodeAt(index) !== CODE_QUOTE) {
|
|
582
|
+
const next = source.charCodeAt(index);
|
|
583
|
+
if (index >= source.length || next === CODE_NEWLINE || next === CODE_RETURN) {
|
|
594
584
|
throw createError("Unterminated string");
|
|
595
585
|
}
|
|
596
586
|
if (next === CODE_BACKSLASH) {
|
|
597
|
-
const escaped = source[
|
|
587
|
+
const escaped = source[index + 1];
|
|
598
588
|
if (escaped === undefined || !'ntrf\\"'.includes(escaped)) {
|
|
599
589
|
throw createError("Invalid escape in string");
|
|
600
590
|
}
|
|
@@ -602,39 +592,38 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
602
592
|
advanceOver(2);
|
|
603
593
|
continue;
|
|
604
594
|
}
|
|
605
|
-
characters.push(source[
|
|
595
|
+
characters.push(source[index]);
|
|
606
596
|
advanceOver(1);
|
|
607
597
|
}
|
|
608
598
|
advanceOver(1);
|
|
609
599
|
tokens.push({
|
|
610
600
|
kind: "String",
|
|
611
601
|
text: characters.join(""),
|
|
612
|
-
line,
|
|
602
|
+
line: startLine,
|
|
613
603
|
column,
|
|
614
604
|
endColumn: getColumn() - 1,
|
|
615
605
|
});
|
|
616
606
|
continue;
|
|
617
607
|
}
|
|
618
|
-
const basedLength = matchBasedNumber(source,
|
|
608
|
+
const basedLength = matchBasedNumber(source, index);
|
|
619
609
|
if (basedLength > 0) {
|
|
620
|
-
pushToken("Number", source.slice(
|
|
610
|
+
pushToken("Number", source.slice(index, index + basedLength), basedLength);
|
|
621
611
|
continue;
|
|
622
612
|
}
|
|
623
613
|
if (isInClass(code, IDENTIFIER)) {
|
|
624
|
-
const wordEnd = skipClass(source,
|
|
625
|
-
const word = source.slice(
|
|
626
|
-
const digitsEnd = skipClass(source,
|
|
614
|
+
const wordEnd = skipClass(source, index, IDENTIFIER);
|
|
615
|
+
const word = source.slice(index, wordEnd);
|
|
616
|
+
const digitsEnd = skipClass(source, index, DIGIT);
|
|
627
617
|
if (digitsEnd === wordEnd) {
|
|
628
618
|
const fractionEnd =
|
|
629
619
|
source.charCodeAt(wordEnd) === CODE_DOT ? skipClass(source, wordEnd + 1, DIGIT) : wordEnd;
|
|
630
620
|
const isDecimal = fractionEnd > wordEnd + 1;
|
|
631
|
-
const number = isDecimal ? source.slice(
|
|
621
|
+
const number = isDecimal ? source.slice(index, fractionEnd) : word;
|
|
632
622
|
pushToken(isDecimal ? "Decimal" : "Number", number, number.length);
|
|
633
623
|
continue;
|
|
634
624
|
}
|
|
635
|
-
// A word needs a letter to be a name, so SANY reads one without as underscores and numbers.
|
|
636
625
|
if (!/[A-Za-z]/.test(word)) {
|
|
637
|
-
const part = code === CODE_UNDERSCORE ? "_" : source.slice(
|
|
626
|
+
const part = code === CODE_UNDERSCORE ? "_" : source.slice(index, digitsEnd);
|
|
638
627
|
pushToken(part === "_" ? "Operator" : "Number", part, part.length);
|
|
639
628
|
continue;
|
|
640
629
|
}
|
|
@@ -647,22 +636,22 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
647
636
|
pushToken(KEYWORDS.has(word) ? "Keyword" : "Identifier", name, word.length);
|
|
648
637
|
continue;
|
|
649
638
|
}
|
|
650
|
-
const character = source[
|
|
639
|
+
const character = source[index];
|
|
651
640
|
if (code >= 128 && UNICODE_NAMES.has(character)) {
|
|
652
641
|
pushToken("Identifier", names.get(character) ?? character, 1);
|
|
653
642
|
names.set(character, character);
|
|
654
643
|
continue;
|
|
655
644
|
}
|
|
656
645
|
if (code === CODE_LESS_THAN) {
|
|
657
|
-
const stepLength = matchStep(source,
|
|
646
|
+
const stepLength = matchStep(source, index, isTlapm);
|
|
658
647
|
if (stepLength > 0) {
|
|
659
|
-
pushToken("Step", source.slice(
|
|
648
|
+
pushToken("Step", source.slice(index, index + stepLength), stepLength);
|
|
660
649
|
continue;
|
|
661
650
|
}
|
|
662
651
|
}
|
|
663
|
-
const operator = matchOperator(source,
|
|
652
|
+
const operator = matchOperator(source, index);
|
|
664
653
|
if (operator) {
|
|
665
|
-
const spelling = source.slice(
|
|
654
|
+
const spelling = source.slice(index, index + operator.length);
|
|
666
655
|
pushToken(
|
|
667
656
|
"Operator",
|
|
668
657
|
operator.text,
|
|
@@ -675,6 +664,6 @@ export const tokenize = (source: string, isTlapm = false): Token[] => {
|
|
|
675
664
|
}
|
|
676
665
|
|
|
677
666
|
const endColumn = getColumn();
|
|
678
|
-
tokens.push({ kind: "End", text: "", line
|
|
667
|
+
tokens.push({ kind: "End", text: "", line, column: endColumn, endColumn });
|
|
679
668
|
return tokens;
|
|
680
669
|
};
|