tla-ts 0.0.3-dev.4c8c06c → 0.0.3-dev.4fdd845
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/dist/cli.cjs +1 -1
- package/dist/cli.js +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +157 -8
- package/dist/index.d.ts +157 -8
- package/dist/index.js +1 -1
- package/dist/parallel-worker.cjs +1 -1
- package/dist/parallel-worker.js +2 -2
- package/dist/parallel.cjs +1 -1
- package/dist/parallel.js +2 -2
- package/dist/run-cli.cjs +1351 -19
- package/dist/run-cli.js +1351 -19
- package/dist/transfer.cjs +86 -16
- package/dist/transfer.js +86 -16
- package/package.json +5 -1
- package/src/ast.ts +142 -6
- package/src/bags.ts +4 -7
- package/src/builtins.ts +29 -11
- package/src/bundled-modules.ts +3 -3
- package/src/canonical.ts +4 -3
- package/src/check.ts +1539 -235
- package/src/cli-options.ts +11 -5
- package/src/cli.ts +8 -1
- package/src/codegen.ts +77 -37
- package/src/config.ts +252 -179
- package/src/dependencies.ts +2 -2
- package/src/environment.ts +518 -137
- package/src/errors.ts +224 -3
- package/src/evaluate.ts +796 -246
- package/src/exit-codes.ts +22 -43
- package/src/index.ts +1 -0
- package/src/instances.ts +19 -16
- package/src/java-overrides.ts +104 -25
- package/src/levels.ts +191 -45
- package/src/lexer.ts +32 -4
- package/src/live-check.ts +828 -0
- package/src/liveness.ts +218 -572
- package/src/obligation-expression.ts +2 -2
- package/src/obligation-source.ts +2 -0
- package/src/parallel-worker.ts +13 -4
- package/src/parallel.ts +73 -6
- package/src/parser.ts +447 -128
- package/src/pcal-ast.ts +770 -0
- package/src/pcal-parse.ts +1523 -0
- package/src/pcal-tokenize.ts +516 -0
- package/src/pcal-translate.ts +1230 -0
- package/src/pcal-validate.ts +454 -0
- package/src/randomization.ts +24 -7
- package/src/resolve.ts +1380 -420
- package/src/run-cli.ts +1200 -143
- package/src/runtime.ts +19 -1
- package/src/sany-builtins.ts +147 -0
- package/src/sany-generator.ts +3112 -0
- package/src/sany-levels.ts +1208 -0
- package/src/sany-lexer.ts +545 -0
- package/src/sany-linter.ts +122 -0
- package/src/sany-modules.ts +1398 -0
- package/src/sany-nodes.ts +463 -0
- package/src/sany-parse-error.ts +2161 -0
- package/src/sany-report.ts +354 -0
- package/src/sany-selectors.ts +738 -0
- package/src/sany-source.ts +248 -0
- package/src/sany-xml.ts +1017 -0
- package/src/spec-description.ts +42 -11
- package/src/standard-modules.ts +200 -148
- package/src/standard-ranges.ts +116 -0
- package/src/state-graph.ts +180 -0
- package/src/subexpressions.ts +231 -171
- package/src/substitution-levels.ts +38 -17
- package/src/successors.ts +219 -107
- package/src/symmetry.ts +180 -52
- package/src/tableau.ts +54 -25
- package/src/tlc-ext.ts +42 -29
- package/src/tlc-messages.ts +329 -0
- package/src/tlc-module.ts +52 -43
- package/src/tlc-order.ts +73 -9
- package/src/tlc-random.ts +37 -9
- package/src/tlc-replay.ts +655 -0
- package/src/tlc-summary.ts +20 -5
- package/src/tlc-version.ts +18 -0
- package/src/transfer.ts +15 -3
- package/src/value-sources.ts +29 -0
- package/src/values.ts +708 -180
package/src/levels.ts
CHANGED
|
@@ -1,15 +1,23 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
import {
|
|
2
|
+
getChildren,
|
|
3
|
+
type AssumeProve,
|
|
4
|
+
type Definition,
|
|
5
|
+
type Expression,
|
|
6
|
+
type Module,
|
|
7
|
+
type OperatorParam,
|
|
8
|
+
type Proof,
|
|
9
|
+
type ProofStep,
|
|
10
|
+
type SourceLocation,
|
|
8
11
|
} from "./ast";
|
|
9
|
-
import {
|
|
10
|
-
|
|
12
|
+
import {
|
|
13
|
+
createSanyError,
|
|
14
|
+
createSemanticError,
|
|
15
|
+
formatRange,
|
|
16
|
+
TlaSyntaxError,
|
|
17
|
+
type SanyError,
|
|
18
|
+
} from "./errors";
|
|
11
19
|
import { mapChildren } from "./instances";
|
|
12
|
-
import { getBoundNames, getPatternNames } from "./resolve";
|
|
20
|
+
import { collectSemanticErrors, getBoundNames, getPatternNames } from "./resolve";
|
|
13
21
|
import { STANDARD_LEVELS } from "./standard-levels";
|
|
14
22
|
|
|
15
23
|
interface LevelData {
|
|
@@ -34,6 +42,7 @@ interface LevelContext {
|
|
|
34
42
|
constants: Set<string>;
|
|
35
43
|
variables: Set<string>;
|
|
36
44
|
getDefinition: (name: string) => Definition | undefined;
|
|
45
|
+
getQualifiedName?: (name: string) => string | undefined;
|
|
37
46
|
}
|
|
38
47
|
|
|
39
48
|
type ScopeEntry =
|
|
@@ -172,12 +181,19 @@ const bindFormals = (scope: Scope, formals: OperatorParam[]) =>
|
|
|
172
181
|
|
|
173
182
|
const toFormals = (names: string[]) => names.map((name) => ({ name, arity: 0 }));
|
|
174
183
|
|
|
184
|
+
// The levels of tla2sany.semantic.LevelConstants.
|
|
185
|
+
export const CONSTANT_LEVEL = 0;
|
|
186
|
+
export const VARIABLE_LEVEL = 1;
|
|
175
187
|
export const ACTION_LEVEL = 2;
|
|
176
|
-
|
|
177
188
|
export const TEMPORAL_LEVEL = 3;
|
|
178
189
|
|
|
179
|
-
|
|
180
|
-
|
|
190
|
+
// SANY reports a level error at the range of the expression it rejects.
|
|
191
|
+
const createLevelError = (message: string, expression: Expression) =>
|
|
192
|
+
expression.location.range
|
|
193
|
+
? createSemanticError(`${message}.`, expression.location.range)
|
|
194
|
+
: new TlaSyntaxError(
|
|
195
|
+
`${message} at line ${expression.location.line}, column ${expression.location.column}`,
|
|
196
|
+
);
|
|
181
197
|
|
|
182
198
|
const LOGICAL_OPERATORS = new Set(["/\\", "\\/", "=>", "<=>"]);
|
|
183
199
|
|
|
@@ -201,7 +217,13 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
201
217
|
if (context.variables.has(name)) return { kind: "Declared", name, level: 1, arity: 0 };
|
|
202
218
|
const definition = context.getDefinition(name);
|
|
203
219
|
if (definition) {
|
|
204
|
-
|
|
220
|
+
const levels = analyzeDefinition(definition, new Map());
|
|
221
|
+
// SANY names a definition of an instance by its qualified name, as in I!Foo.
|
|
222
|
+
const qualifiedName = context.getQualifiedName?.(name);
|
|
223
|
+
return {
|
|
224
|
+
kind: "Defined",
|
|
225
|
+
levels: qualifiedName ? { ...levels, name: qualifiedName } : levels,
|
|
226
|
+
};
|
|
205
227
|
}
|
|
206
228
|
const signature = CORE_SIGNATURES.get(name) ?? STANDARD_LEVELS.get(name);
|
|
207
229
|
if (signature) return { kind: "Defined", levels: signature };
|
|
@@ -237,13 +259,13 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
237
259
|
if (operand.operator || operand.declared) return;
|
|
238
260
|
if (operand.data.level > getMaxLevel(operator, index)) {
|
|
239
261
|
throw new TlaSyntaxError(
|
|
240
|
-
`Level error in applying operator ${operator.name}
|
|
262
|
+
`Level error in applying operator ${operator.name}:\nThe level of argument ${index + 1} exceeds the maximum level allowed by the operator`,
|
|
241
263
|
);
|
|
242
264
|
}
|
|
243
265
|
});
|
|
244
266
|
if (ranges.some((range) => range.level > ACTION_LEVEL)) {
|
|
245
267
|
throw new TlaSyntaxError(
|
|
246
|
-
`Level error in applying operator ${operator.name}
|
|
268
|
+
`Level error in applying operator ${operator.name}:\nThe level of the range for a bounded variable exceeds the maximum level allowed by the operator`,
|
|
247
269
|
);
|
|
248
270
|
}
|
|
249
271
|
const getOperand = (formal: string) => {
|
|
@@ -444,10 +466,13 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
444
466
|
? applyDeclared(resolved.name, resolved.level, operands)
|
|
445
467
|
: applyDefined(resolved.levels, operands);
|
|
446
468
|
|
|
447
|
-
const checkJunction = (operands: Operand[],
|
|
469
|
+
const checkJunction = (operands: Operand[], expression: Expression) => {
|
|
448
470
|
const levels = operands.map((operand) => operand.data.level);
|
|
449
471
|
if (levels.includes(ACTION_LEVEL) && levels.includes(TEMPORAL_LEVEL)) {
|
|
450
|
-
throw createLevelError(
|
|
472
|
+
throw createLevelError(
|
|
473
|
+
"Formula has both temporal formula and action as arguments",
|
|
474
|
+
expression,
|
|
475
|
+
);
|
|
451
476
|
}
|
|
452
477
|
};
|
|
453
478
|
|
|
@@ -456,10 +481,10 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
456
481
|
if (isLeadsTo && operands.some((operand) => operand.data.level === ACTION_LEVEL)) {
|
|
457
482
|
throw createLevelError(
|
|
458
483
|
"Action used where only temporal formula or state predicate allowed",
|
|
459
|
-
expression
|
|
484
|
+
expression,
|
|
460
485
|
);
|
|
461
486
|
}
|
|
462
|
-
if (LOGICAL_OPERATORS.has(expression.operator)) checkJunction(operands, expression
|
|
487
|
+
if (LOGICAL_OPERATORS.has(expression.operator)) checkJunction(operands, expression);
|
|
463
488
|
};
|
|
464
489
|
|
|
465
490
|
const checkTemporal = (
|
|
@@ -473,7 +498,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
473
498
|
expression.kind === "Always"
|
|
474
499
|
? "[] followed by action not of form [A]_v"
|
|
475
500
|
: "<> followed by action not of form <<A>>_v",
|
|
476
|
-
expression
|
|
501
|
+
expression,
|
|
477
502
|
);
|
|
478
503
|
};
|
|
479
504
|
|
|
@@ -501,7 +526,17 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
501
526
|
return data;
|
|
502
527
|
};
|
|
503
528
|
|
|
529
|
+
// SANY reports a level error of an operator's arguments at the application.
|
|
504
530
|
const analyze = (expression: Expression, scope: Scope): LevelData => {
|
|
531
|
+
try {
|
|
532
|
+
return analyzeNode(expression, scope);
|
|
533
|
+
} catch (error) {
|
|
534
|
+
if (!(error instanceof TlaSyntaxError) || error.sany) throw error;
|
|
535
|
+
throw createLevelError(error.message, expression);
|
|
536
|
+
}
|
|
537
|
+
};
|
|
538
|
+
|
|
539
|
+
const analyzeNode = (expression: Expression, scope: Scope): LevelData => {
|
|
505
540
|
switch (expression.kind) {
|
|
506
541
|
case "Number":
|
|
507
542
|
case "Decimal":
|
|
@@ -521,7 +556,7 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
521
556
|
case "And":
|
|
522
557
|
case "Or": {
|
|
523
558
|
const operands = toOperands(expression.operands, scope);
|
|
524
|
-
checkJunction(operands, expression
|
|
559
|
+
checkJunction(operands, expression);
|
|
525
560
|
return applyDefined(JUNCTION_LEVELS, operands);
|
|
526
561
|
}
|
|
527
562
|
case "Prime":
|
|
@@ -561,13 +596,14 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
561
596
|
expression.bounds.map((bound) => bound.set),
|
|
562
597
|
getBoundNames(expression.bounds),
|
|
563
598
|
);
|
|
564
|
-
|
|
565
|
-
data.level === TEMPORAL_LEVEL
|
|
566
|
-
|
|
567
|
-
|
|
599
|
+
const actionBound =
|
|
600
|
+
data.level === TEMPORAL_LEVEL
|
|
601
|
+
? expression.bounds.find((bound) => analyze(bound.set, scope).level === ACTION_LEVEL)
|
|
602
|
+
: undefined;
|
|
603
|
+
if (actionBound) {
|
|
568
604
|
throw createLevelError(
|
|
569
605
|
"Action-level bound of quantified temporal formula",
|
|
570
|
-
|
|
606
|
+
actionBound.set,
|
|
571
607
|
);
|
|
572
608
|
}
|
|
573
609
|
return data;
|
|
@@ -656,6 +692,9 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
656
692
|
? toOperand(expression.args[0], scope)
|
|
657
693
|
: { data: applyBuiltin(SET_LEVELS, expression.args, scope) },
|
|
658
694
|
]);
|
|
695
|
+
// SANY's semantic checker rejects the expression without computing its level.
|
|
696
|
+
case "Bracket":
|
|
697
|
+
return createData();
|
|
659
698
|
case "FieldAccess":
|
|
660
699
|
return applyDefined(SELECT_LEVELS, [
|
|
661
700
|
toOperand(expression.record, scope),
|
|
@@ -817,34 +856,141 @@ export const createLevelAnalyzer = (context: LevelContext) => {
|
|
|
817
856
|
return { analyze, analyzeDefinition, toOperand };
|
|
818
857
|
};
|
|
819
858
|
|
|
820
|
-
const analyzeAssumeProve = (
|
|
859
|
+
export const analyzeAssumeProve = (
|
|
821
860
|
assumeProve: AssumeProve,
|
|
822
861
|
scope: Scope,
|
|
823
862
|
analyzer: ReturnType<typeof createLevelAnalyzer>,
|
|
824
|
-
) => {
|
|
863
|
+
): LevelData => {
|
|
825
864
|
const inner = new Map(scope);
|
|
826
|
-
assumeProve.assumptions.
|
|
865
|
+
const parts = assumeProve.assumptions.flatMap((assumption) => {
|
|
827
866
|
if (assumption.kind === "NewSymbol") {
|
|
828
|
-
|
|
867
|
+
const set = assumption.set && analyzer.analyze(assumption.set, inner);
|
|
829
868
|
inner.set(assumption.name, { kind: "Formal", arity: assumption.arity });
|
|
830
|
-
return;
|
|
869
|
+
return set ? [set] : [];
|
|
831
870
|
}
|
|
832
|
-
|
|
833
|
-
|
|
871
|
+
return [
|
|
872
|
+
assumption.kind === "AssumeProve"
|
|
873
|
+
? analyzeAssumeProve(assumption, inner, analyzer)
|
|
874
|
+
: analyzer.analyze(assumption, inner),
|
|
875
|
+
];
|
|
834
876
|
});
|
|
835
|
-
analyzer.analyze(assumeProve.conclusion, inner);
|
|
877
|
+
parts.push(analyzer.analyze(assumeProve.conclusion, inner));
|
|
878
|
+
const data = createData(Math.max(...parts.map((part) => part.level)));
|
|
879
|
+
parts.forEach((part) => {
|
|
880
|
+
part.params.forEach((param) => data.params.add(param));
|
|
881
|
+
part.constraints.forEach((value, key) => putConstraint(data.constraints, key, value));
|
|
882
|
+
part.argConstraints.forEach((value, key) => putArgConstraint(data.argConstraints, key, value));
|
|
883
|
+
part.argParams.forEach((key) => data.argParams.add(key));
|
|
884
|
+
});
|
|
885
|
+
return data;
|
|
836
886
|
};
|
|
837
887
|
|
|
838
|
-
// SANY checks the level of every definition, assumption, and
|
|
839
|
-
export const checkModuleLevels = (
|
|
888
|
+
// SANY checks the level of every definition, assumption, theorem, and instance of a module.
|
|
889
|
+
export const checkModuleLevels = (
|
|
890
|
+
module: Module,
|
|
891
|
+
context: LevelContext,
|
|
892
|
+
instances: { location: SourceLocation; check: () => void }[],
|
|
893
|
+
warnings: SanyError[],
|
|
894
|
+
) => {
|
|
840
895
|
const analyzer = createLevelAnalyzer(context);
|
|
841
|
-
|
|
842
|
-
.
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
896
|
+
const getLevel = (expressions: Expression[]) =>
|
|
897
|
+
Math.max(0, ...expressions.map((expression) => analyzer.analyze(expression, new Map()).level));
|
|
898
|
+
const getStepLevel = (step: ProofStep) => {
|
|
899
|
+
const body = step.body;
|
|
900
|
+
if (body.kind === "Have" || body.kind === "Case") return getLevel([body.expression]);
|
|
901
|
+
if (body.kind === "Witness") return getLevel(body.expressions);
|
|
902
|
+
if (body.kind === "Take") return getLevel(body.bounds.map((bound) => bound.set));
|
|
903
|
+
if (body.kind === "Pick") {
|
|
904
|
+
return getLevel([body.body, ...body.bounds.map((bound) => bound.set)]);
|
|
905
|
+
}
|
|
906
|
+
if (body.kind !== "Assert") return 0;
|
|
907
|
+
return body.statement.kind === "AssumeProve"
|
|
908
|
+
? analyzeAssumeProve(body.statement, new Map(), analyzer).level
|
|
909
|
+
: getLevel([body.statement]);
|
|
910
|
+
};
|
|
911
|
+
// SANY's TheoremNode.LevelCheckTemporal: the proof of a temporal goal may only assume constants.
|
|
912
|
+
const checkTemporalGoal = (proof: Proof | undefined): SanyError[] =>
|
|
913
|
+
proof?.kind !== "Steps"
|
|
914
|
+
? []
|
|
915
|
+
: proof.steps.flatMap((step) => {
|
|
916
|
+
const location = formatRange(step.statement);
|
|
917
|
+
const isConstant = () => getStepLevel(step) === 0;
|
|
918
|
+
if (
|
|
919
|
+
step.body.kind === "Have" ||
|
|
920
|
+
step.body.kind === "Take" ||
|
|
921
|
+
step.body.kind === "Witness"
|
|
922
|
+
) {
|
|
923
|
+
return isConstant()
|
|
924
|
+
? []
|
|
925
|
+
: [{ location, message: "Non-constant TAKE, WITNESS, or HAVE for temporal goal." }];
|
|
926
|
+
}
|
|
927
|
+
if (step.body.kind === "Case") {
|
|
928
|
+
return [
|
|
929
|
+
...(isConstant()
|
|
930
|
+
? []
|
|
931
|
+
: [{ location, message: "Non-constant CASE for temporal goal." }]),
|
|
932
|
+
...checkTemporalGoal(step.proof),
|
|
933
|
+
];
|
|
934
|
+
}
|
|
935
|
+
return step.body.kind === "Qed" ? checkTemporalGoal(step.proof) : [];
|
|
936
|
+
});
|
|
937
|
+
// SANY checks each step as a theorem of its own after the steps of its proof.
|
|
938
|
+
const checkProof = (proof: Proof | undefined): SanyError[] =>
|
|
939
|
+
proof?.kind !== "Steps"
|
|
940
|
+
? []
|
|
941
|
+
: proof.steps.flatMap((step) => {
|
|
942
|
+
const body = step.body;
|
|
943
|
+
const isTemporal = () => getStepLevel(step) === TEMPORAL_LEVEL;
|
|
944
|
+
const pickErrors =
|
|
945
|
+
body.kind === "Pick" && body.bounds.length > 0 && isTemporal()
|
|
946
|
+
? body.bounds
|
|
947
|
+
.filter((bound) => getLevel([bound.set]) !== 0)
|
|
948
|
+
.flatMap((bound) =>
|
|
949
|
+
bound.set.location.range
|
|
950
|
+
? [
|
|
951
|
+
{
|
|
952
|
+
location: formatRange(bound.set.location.range),
|
|
953
|
+
message: "Non-constant bound of temporal PICK.",
|
|
954
|
+
},
|
|
955
|
+
]
|
|
956
|
+
: [],
|
|
957
|
+
)
|
|
958
|
+
: [];
|
|
959
|
+
return [
|
|
960
|
+
...checkProof(step.proof),
|
|
961
|
+
...pickErrors,
|
|
962
|
+
...(step.proof?.kind === "Steps" && isTemporal() ? checkTemporalGoal(step.proof) : []),
|
|
963
|
+
];
|
|
964
|
+
});
|
|
965
|
+
const errors = collectSemanticErrors(
|
|
966
|
+
[
|
|
967
|
+
...module.definitions
|
|
968
|
+
.filter((definition) => !definition.name.includes("!"))
|
|
969
|
+
.map((definition) => ({
|
|
970
|
+
location: definition.location,
|
|
971
|
+
check: () => analyzer.analyzeDefinition(definition, new Map()),
|
|
972
|
+
})),
|
|
973
|
+
...module.assumptions.map((assumption) => ({
|
|
974
|
+
location: assumption.location,
|
|
975
|
+
check: () => analyzer.analyze(assumption.expression, new Map()),
|
|
976
|
+
})),
|
|
977
|
+
...module.theorems.map((theorem) => ({
|
|
978
|
+
location: theorem.location,
|
|
979
|
+
check: () => {
|
|
980
|
+
const data =
|
|
981
|
+
theorem.statement.kind === "AssumeProve"
|
|
982
|
+
? analyzeAssumeProve(theorem.statement, new Map(), analyzer)
|
|
983
|
+
: analyzer.analyze(theorem.statement, new Map());
|
|
984
|
+
const errors = [
|
|
985
|
+
...checkProof(theorem.proof),
|
|
986
|
+
...(data.level === TEMPORAL_LEVEL ? checkTemporalGoal(theorem.proof) : []),
|
|
987
|
+
];
|
|
988
|
+
if (errors.length > 0) throw createSanyError(errors);
|
|
989
|
+
},
|
|
990
|
+
})),
|
|
991
|
+
...instances,
|
|
992
|
+
],
|
|
993
|
+
warnings,
|
|
849
994
|
);
|
|
995
|
+
if (errors.length > 0) throw createSanyError(errors, false, undefined, warnings);
|
|
850
996
|
};
|
package/src/lexer.ts
CHANGED
|
@@ -12,9 +12,11 @@ type TokenKind =
|
|
|
12
12
|
| "ModuleEnd"
|
|
13
13
|
| "End";
|
|
14
14
|
|
|
15
|
+
// spelling holds the text of an operator written as a synonym, such as (+) for \oplus.
|
|
15
16
|
export interface Token {
|
|
16
17
|
kind: TokenKind;
|
|
17
18
|
text: string;
|
|
19
|
+
spelling?: string;
|
|
18
20
|
line: number;
|
|
19
21
|
column: number;
|
|
20
22
|
endColumn: number;
|
|
@@ -335,6 +337,10 @@ const matchStep = (source: string, index: number) => {
|
|
|
335
337
|
);
|
|
336
338
|
};
|
|
337
339
|
|
|
340
|
+
// SANY's lexer reads @ as an identifier, which stands for the value an EXCEPT changes only alone.
|
|
341
|
+
export const readAtAsIdentifier = (token: Token): Token =>
|
|
342
|
+
token.kind === "Operator" && token.text === "@" ? { ...token, kind: "Identifier" } : token;
|
|
343
|
+
|
|
338
344
|
const TAB_SIZE = 8;
|
|
339
345
|
|
|
340
346
|
const getWidthAfter = (width: number, character: string | undefined) =>
|
|
@@ -367,9 +373,16 @@ export const tokenize = (source: string): Token[] => {
|
|
|
367
373
|
}
|
|
368
374
|
return getWidthAfter(scan.width, source[cursor.index]);
|
|
369
375
|
};
|
|
370
|
-
const pushToken = (kind: TokenKind, text: string, length: number) => {
|
|
376
|
+
const pushToken = (kind: TokenKind, text: string, length: number, spelling?: string) => {
|
|
371
377
|
const column = getColumn();
|
|
372
|
-
tokens.push({
|
|
378
|
+
tokens.push({
|
|
379
|
+
kind,
|
|
380
|
+
text,
|
|
381
|
+
spelling,
|
|
382
|
+
line: cursor.line,
|
|
383
|
+
column,
|
|
384
|
+
endColumn: column + length - 1,
|
|
385
|
+
});
|
|
373
386
|
cursor.index += length;
|
|
374
387
|
};
|
|
375
388
|
const advanceOver = (length: number) => {
|
|
@@ -442,11 +455,14 @@ export const tokenize = (source: string): Token[] => {
|
|
|
442
455
|
const characters: string[] = [];
|
|
443
456
|
advanceOver(1);
|
|
444
457
|
while (source[cursor.index] !== '"') {
|
|
445
|
-
if (cursor.index >= source.length || source[cursor.index]
|
|
458
|
+
if (cursor.index >= source.length || "\n\r".includes(source[cursor.index])) {
|
|
446
459
|
throw createError("Unterminated string");
|
|
447
460
|
}
|
|
448
461
|
if (source[cursor.index] === "\\") {
|
|
449
462
|
const escaped = source[cursor.index + 1];
|
|
463
|
+
if (escaped === undefined || !'ntrf\\"'.includes(escaped)) {
|
|
464
|
+
throw createError("Invalid escape in string");
|
|
465
|
+
}
|
|
450
466
|
characters.push({ n: "\n", t: "\t", r: "\r", f: "\f" }[escaped] ?? escaped);
|
|
451
467
|
advanceOver(2);
|
|
452
468
|
continue;
|
|
@@ -479,6 +495,12 @@ export const tokenize = (source: string): Token[] => {
|
|
|
479
495
|
pushToken(number.includes(".") ? "Decimal" : "Number", number, number.length);
|
|
480
496
|
continue;
|
|
481
497
|
}
|
|
498
|
+
// A word needs a letter to be a name, so SANY reads one without as underscores and numbers.
|
|
499
|
+
const part = /^(_|\d+)/.exec(word)?.[0];
|
|
500
|
+
if (part !== undefined && !/[A-Za-z]/.test(word)) {
|
|
501
|
+
pushToken(part === "_" ? "Operator" : "Number", part, part.length);
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
482
504
|
if (word.startsWith("WF_") || word.startsWith("SF_")) {
|
|
483
505
|
pushToken("Keyword", word.slice(0, 3), 3);
|
|
484
506
|
continue;
|
|
@@ -495,7 +517,13 @@ export const tokenize = (source: string): Token[] => {
|
|
|
495
517
|
}
|
|
496
518
|
const operator = matchOperator(source, cursor.index);
|
|
497
519
|
if (operator) {
|
|
498
|
-
|
|
520
|
+
const spelling = source.slice(cursor.index, cursor.index + operator.length);
|
|
521
|
+
pushToken(
|
|
522
|
+
"Operator",
|
|
523
|
+
operator.text,
|
|
524
|
+
operator.length,
|
|
525
|
+
spelling === operator.text ? undefined : spelling,
|
|
526
|
+
);
|
|
499
527
|
continue;
|
|
500
528
|
}
|
|
501
529
|
throw createError(`Unexpected character ${JSON.stringify(character)}`);
|