sommark 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/cli/commands/build.js +2 -1
- package/cli/commands/init.js +2 -6
- package/cli/commands/list.js +17 -12
- package/cli/commands/print.js +7 -2
- package/cli/helpers/transpile.js +2 -1
- package/core/errors.js +22 -9
- package/core/labels.js +3 -0
- package/core/lexer.js +207 -590
- package/core/parser.js +201 -65
- package/core/pluginManager.js +33 -23
- package/core/plugins/comment-remover.js +3 -3
- package/core/plugins/module-system.js +163 -124
- package/core/plugins/raw-content-plugin.js +15 -9
- package/core/plugins/rules-validation-plugin.js +2 -2
- package/core/plugins/sommark-format.js +92 -72
- package/core/tokenTypes.js +2 -1
- package/core/transpiler.js +70 -8
- package/coverage_test.js +21 -0
- package/helpers/utils.js +27 -0
- package/index.js +25 -16
- package/mappers/languages/html.js +5 -10
- package/package.json +1 -1
- package/v3-todo.smark +68 -70
- package/core/plugins/quote-escaper.js +0 -37
- package/format.js +0 -23
- package/unformatted.smark +0 -90
package/core/parser.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SomMark Parser
|
|
3
|
+
*/
|
|
1
4
|
import TOKEN_TYPES from "./tokenTypes.js";
|
|
2
5
|
import peek from "../helpers/peek.js";
|
|
3
6
|
import { parserError } from "./errors.js";
|
|
@@ -7,6 +10,8 @@ import {
|
|
|
7
10
|
INLINE,
|
|
8
11
|
ATBLOCK,
|
|
9
12
|
COMMENT,
|
|
13
|
+
IMPORT,
|
|
14
|
+
USE_MODULE,
|
|
10
15
|
block_id,
|
|
11
16
|
block_value,
|
|
12
17
|
inline_id,
|
|
@@ -15,6 +20,11 @@ import {
|
|
|
15
20
|
at_value,
|
|
16
21
|
end_keyword
|
|
17
22
|
} from "./labels.js";
|
|
23
|
+
import { levenshtein } from "../helpers/utils.js";
|
|
24
|
+
|
|
25
|
+
// ========================================================================== //
|
|
26
|
+
// Helper Functions //
|
|
27
|
+
// ========================================================================== //
|
|
18
28
|
|
|
19
29
|
function current_token(tokens, i) {
|
|
20
30
|
return tokens[i] || null;
|
|
@@ -38,7 +48,11 @@ function makeBlockNode() {
|
|
|
38
48
|
id: "",
|
|
39
49
|
args: [],
|
|
40
50
|
body: [],
|
|
41
|
-
depth: 0
|
|
51
|
+
depth: 0,
|
|
52
|
+
range: {
|
|
53
|
+
start: { line: 0, character: 0 },
|
|
54
|
+
end: { line: 0, character: 0 }
|
|
55
|
+
}
|
|
42
56
|
};
|
|
43
57
|
}
|
|
44
58
|
|
|
@@ -46,7 +60,11 @@ function makeTextNode() {
|
|
|
46
60
|
return {
|
|
47
61
|
type: TEXT,
|
|
48
62
|
text: "",
|
|
49
|
-
depth: 0
|
|
63
|
+
depth: 0,
|
|
64
|
+
range: {
|
|
65
|
+
start: { line: 0, character: 0 },
|
|
66
|
+
end: { line: 0, character: 0 }
|
|
67
|
+
}
|
|
50
68
|
};
|
|
51
69
|
}
|
|
52
70
|
|
|
@@ -54,7 +72,11 @@ function makeCommentNode() {
|
|
|
54
72
|
return {
|
|
55
73
|
type: COMMENT,
|
|
56
74
|
text: "",
|
|
57
|
-
depth: 0
|
|
75
|
+
depth: 0,
|
|
76
|
+
range: {
|
|
77
|
+
start: { line: 0, character: 0 },
|
|
78
|
+
end: { line: 0, character: 0 }
|
|
79
|
+
}
|
|
58
80
|
};
|
|
59
81
|
}
|
|
60
82
|
|
|
@@ -64,57 +86,82 @@ function makeInlineNode() {
|
|
|
64
86
|
value: "",
|
|
65
87
|
id: "",
|
|
66
88
|
args: [],
|
|
67
|
-
depth: 0
|
|
89
|
+
depth: 0,
|
|
90
|
+
range: {
|
|
91
|
+
start: { line: 0, character: 0 },
|
|
92
|
+
end: { line: 0, character: 0 }
|
|
93
|
+
}
|
|
68
94
|
};
|
|
69
95
|
}
|
|
70
96
|
|
|
97
|
+
// ========================================================================== //
|
|
98
|
+
// Node Creators (Factories) //
|
|
99
|
+
// ========================================================================== //
|
|
100
|
+
|
|
71
101
|
function makeAtBlockNode() {
|
|
72
102
|
return {
|
|
73
103
|
type: ATBLOCK,
|
|
74
104
|
id: "",
|
|
75
105
|
args: [],
|
|
76
106
|
content: "",
|
|
77
|
-
depth: 0
|
|
107
|
+
depth: 0,
|
|
108
|
+
range: {
|
|
109
|
+
start: { line: 0, character: 0 },
|
|
110
|
+
end: { line: 0, character: 0 }
|
|
111
|
+
}
|
|
78
112
|
};
|
|
79
113
|
}
|
|
80
114
|
|
|
115
|
+
// ========================================================================== //
|
|
116
|
+
// Parser State and Error Tracking //
|
|
117
|
+
// ========================================================================== //
|
|
118
|
+
|
|
81
119
|
let end_stack = [];
|
|
82
120
|
let tokens_stack = [];
|
|
83
|
-
let
|
|
84
|
-
start
|
|
85
|
-
end
|
|
121
|
+
let range = {
|
|
122
|
+
start: { line: 0, character: 0 },
|
|
123
|
+
end: { line: 0, character: 0 }
|
|
124
|
+
},
|
|
86
125
|
value = "";
|
|
87
126
|
|
|
88
127
|
const fallback = {
|
|
89
128
|
value: "Unknown",
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
129
|
+
range: {
|
|
130
|
+
start: { line: 0, character: 0 },
|
|
131
|
+
end: { line: 0, character: 0 }
|
|
132
|
+
},
|
|
93
133
|
tokens_stack: ["--Empty--"]
|
|
94
134
|
};
|
|
95
135
|
const updateData = (tokens, i) => {
|
|
96
136
|
if (tokens[i]) {
|
|
97
137
|
tokens_stack.push(tokens[i].value);
|
|
98
|
-
|
|
99
|
-
start = tokens[i].start;
|
|
100
|
-
end = tokens[i].end;
|
|
138
|
+
range = tokens[i].range;
|
|
101
139
|
value = tokens[i].value;
|
|
102
140
|
}
|
|
103
141
|
};
|
|
104
142
|
|
|
105
|
-
const errorMessage = (tokens, i, expectedValue, behindValue, frontText) => {
|
|
106
|
-
const current = tokens[i]
|
|
107
|
-
const errorLineNumber = current.line;
|
|
143
|
+
const errorMessage = (tokens, i, expectedValue, behindValue, frontText, filename = null) => {
|
|
144
|
+
const current = tokens[i] || fallback;
|
|
145
|
+
const errorLineNumber = current.range.start.line;
|
|
146
|
+
const errorCharNumber = current.range.start.character;
|
|
147
|
+
const source = current.source || filename;
|
|
148
|
+
const sourceLabel = source ? ` [${source}]` : "";
|
|
108
149
|
|
|
109
|
-
// Find starting index of the error line
|
|
110
150
|
let lineStartIndex = i;
|
|
111
|
-
while (
|
|
151
|
+
while (
|
|
152
|
+
lineStartIndex > 0 &&
|
|
153
|
+
tokens[lineStartIndex - 1].range.start.line === errorLineNumber &&
|
|
154
|
+
(tokens[lineStartIndex - 1].source || filename) === source
|
|
155
|
+
) {
|
|
112
156
|
lineStartIndex--;
|
|
113
157
|
}
|
|
114
158
|
|
|
115
|
-
// Find ending index of the error line
|
|
116
159
|
let lineEndIndex = i;
|
|
117
|
-
while (
|
|
160
|
+
while (
|
|
161
|
+
lineEndIndex < tokens.length - 1 &&
|
|
162
|
+
tokens[lineEndIndex + 1].range.start.line === errorLineNumber &&
|
|
163
|
+
(tokens[lineEndIndex + 1].source || filename) === source
|
|
164
|
+
) {
|
|
118
165
|
lineEndIndex++;
|
|
119
166
|
}
|
|
120
167
|
|
|
@@ -125,16 +172,19 @@ const errorMessage = (tokens, i, expectedValue, behindValue, frontText) => {
|
|
|
125
172
|
// Get content on the line before the error token
|
|
126
173
|
const tokensBeforeErrorOnLine = tokens.slice(lineStartIndex, i);
|
|
127
174
|
const contentBeforeErrorOnLine = tokensBeforeErrorOnLine.map(t => t.value).join('');
|
|
128
|
-
|
|
175
|
+
|
|
129
176
|
const pointerPadding = " ".repeat(contentBeforeErrorOnLine.length);
|
|
177
|
+
const rangeInfo = current.range.start.line === current.range.end.line
|
|
178
|
+
? `from column <$yellow:${current.range.start.character}$> to <$yellow:${current.range.end.character}$>`
|
|
179
|
+
: `from line <$yellow:${current.range.start.line + 1}$>, column <$yellow:${current.range.start.character}$> to line <$yellow:${current.range.end.line + 1}$>, column <$yellow:${current.range.end.character}$>`;
|
|
130
180
|
|
|
131
181
|
return [
|
|
132
|
-
`<$blue:{line}$><$red:Here where error occurred:$>{N}${lineContent}{N}${pointerPadding}<$yellow:^$>{N}{N}`,
|
|
133
|
-
`<$red:${frontText ? frontText : "Expected token"}
|
|
134
|
-
`
|
|
135
|
-
`{N}<$yellow:Received:$> <$blue:'${value === "\n" ? "\\n' (newline)" : value}'$>`,
|
|
136
|
-
` at line <$yellow:${current.line}$>,`,
|
|
137
|
-
`
|
|
182
|
+
`<$blue:{line}$><$red:Here where error occurred${sourceLabel}:$>{N}${lineContent}{N}${pointerPadding}<$yellow:^$>{N}{N}`,
|
|
183
|
+
`<$red:${frontText ? frontText : "Expected token"}$>${!frontText ? " <$blue:'" + expectedValue + "'$>" : ""} ${behindValue ? "after <$blue:'" + behindValue + "'$>" : ""} at line <$yellow:${current.range.start.line + 1}$>,`,
|
|
184
|
+
` ${rangeInfo}`,
|
|
185
|
+
`{N}<$yellow:Received:$> <$blue:'${current.value === "\n" ? "\\n' (newline)" : current.value}'$>`,
|
|
186
|
+
` at line <$yellow:${current.range.start.line + 1}$>,`,
|
|
187
|
+
` ${rangeInfo}{N}`,
|
|
138
188
|
"<$blue:{line}$>"
|
|
139
189
|
];
|
|
140
190
|
};
|
|
@@ -231,23 +281,34 @@ function parseSemiColon(tokens, i, afterChar = "") {
|
|
|
231
281
|
// ========================================================================== //
|
|
232
282
|
// Parse Block //
|
|
233
283
|
// ========================================================================== //
|
|
234
|
-
function parseBlock(tokens, i) {
|
|
284
|
+
function parseBlock(tokens, i, filename = null) {
|
|
235
285
|
const blockNode = makeBlockNode();
|
|
286
|
+
const openBracketToken = current_token(tokens, i);
|
|
236
287
|
// ========================================================================== //
|
|
237
288
|
// consume '[' //
|
|
238
289
|
// ========================================================================== //
|
|
239
290
|
i++;
|
|
240
291
|
updateData(tokens, i);
|
|
241
|
-
|
|
242
|
-
|
|
292
|
+
const idToken = current_token(tokens, i);
|
|
293
|
+
if (!idToken || idToken.type === TOKEN_TYPES.EOF) {
|
|
294
|
+
parserError(errorMessage(tokens, i, "Block ID", "[", "Missing Block Identifier"));
|
|
243
295
|
}
|
|
244
|
-
const id =
|
|
296
|
+
const id = idToken.value;
|
|
245
297
|
if (id.trim() === end_keyword) {
|
|
246
298
|
parserError(errorMessage(tokens, i, id, "", `'${id.trim()}' is a reserved keyword and cannot be used as an identifier.`));
|
|
247
299
|
}
|
|
248
300
|
blockNode.id = id.trim();
|
|
301
|
+
if (!blockNode.id) {
|
|
302
|
+
parserError(errorMessage(tokens, i, "Block ID", "[", "Block identifier cannot be empty"));
|
|
303
|
+
}
|
|
304
|
+
if (blockNode.id === "import") {
|
|
305
|
+
blockNode.type = IMPORT;
|
|
306
|
+
} else if (blockNode.id === "$use-module") {
|
|
307
|
+
blockNode.type = USE_MODULE;
|
|
308
|
+
}
|
|
249
309
|
validateName(blockNode.id);
|
|
250
|
-
blockNode.depth =
|
|
310
|
+
blockNode.depth = idToken.depth;
|
|
311
|
+
blockNode.range.start = openBracketToken.range.start;
|
|
251
312
|
end_stack.push(id);
|
|
252
313
|
// ========================================================================== //
|
|
253
314
|
// consume Block Identifier //
|
|
@@ -369,7 +430,7 @@ function parseBlock(tokens, i) {
|
|
|
369
430
|
peek(tokens, i, 1) &&
|
|
370
431
|
peek(tokens, i, 1).type !== TOKEN_TYPES.END_KEYWORD
|
|
371
432
|
) {
|
|
372
|
-
const [childNode, nextIndex] = parseBlock(tokens, i);
|
|
433
|
+
const [childNode, nextIndex] = parseBlock(tokens, i, filename);
|
|
373
434
|
blockNode.body.push(childNode);
|
|
374
435
|
// ========================================================================== //
|
|
375
436
|
// consume child node //
|
|
@@ -385,8 +446,16 @@ function parseBlock(tokens, i) {
|
|
|
385
446
|
// consume '[' //
|
|
386
447
|
// ========================================================================== //
|
|
387
448
|
i++;
|
|
388
|
-
|
|
389
|
-
|
|
449
|
+
const current = current_token(tokens, i);
|
|
450
|
+
if (!current || (current.type !== TOKEN_TYPES.END_KEYWORD && current.value.trim() !== end_keyword)) {
|
|
451
|
+
let extraInfo = "";
|
|
452
|
+
if (current && current.value) {
|
|
453
|
+
const dist = levenshtein(current.value.trim().toLowerCase(), "end");
|
|
454
|
+
if (dist <= 2) {
|
|
455
|
+
extraInfo = ` (Did you mean <$cyan:'[end]'$>?)`;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
parserError(errorMessage(tokens, i, "end", "[", extraInfo));
|
|
390
459
|
}
|
|
391
460
|
// ========================================================================== //
|
|
392
461
|
// consume End Keyword //
|
|
@@ -403,17 +472,19 @@ function parseBlock(tokens, i) {
|
|
|
403
472
|
// ========================================================================== //
|
|
404
473
|
// consume ']' //
|
|
405
474
|
// ========================================================================== //
|
|
475
|
+
const closeBracketToken = current_token(tokens, i);
|
|
406
476
|
i++;
|
|
407
477
|
updateData(tokens, i);
|
|
478
|
+
blockNode.range.end = closeBracketToken.range.end;
|
|
408
479
|
break;
|
|
409
480
|
} else {
|
|
410
|
-
const [childNode, nextIndex] = parseNode(tokens, i);
|
|
411
|
-
if (
|
|
412
|
-
|
|
413
|
-
|
|
481
|
+
const [childNode, nextIndex] = parseNode(tokens, i, filename);
|
|
482
|
+
if (childNode) {
|
|
483
|
+
blockNode.body.push(childNode);
|
|
484
|
+
i = nextIndex;
|
|
485
|
+
} else {
|
|
486
|
+
i++; // Should not happen with current parseNode fallback but good for safety
|
|
414
487
|
}
|
|
415
|
-
blockNode.body.push(childNode);
|
|
416
|
-
i = nextIndex;
|
|
417
488
|
}
|
|
418
489
|
}
|
|
419
490
|
return [blockNode, i];
|
|
@@ -423,6 +494,8 @@ function parseBlock(tokens, i) {
|
|
|
423
494
|
// ========================================================================== //
|
|
424
495
|
function parseInline(tokens, i) {
|
|
425
496
|
const inlineNode = makeInlineNode();
|
|
497
|
+
const openParenToken = current_token(tokens, i);
|
|
498
|
+
inlineNode.range.start = openParenToken.range.start;
|
|
426
499
|
// ========================================================================== //
|
|
427
500
|
// consume '(' //
|
|
428
501
|
// ========================================================================== //
|
|
@@ -569,8 +642,10 @@ function parseInline(tokens, i) {
|
|
|
569
642
|
// ========================================================================== //
|
|
570
643
|
// consume ')' //
|
|
571
644
|
// ========================================================================== //
|
|
645
|
+
const finalParenToken = current_token(tokens, i);
|
|
572
646
|
i++;
|
|
573
647
|
updateData(tokens, i);
|
|
648
|
+
inlineNode.range.end = finalParenToken.range.end;
|
|
574
649
|
tokens_stack.length = 0;
|
|
575
650
|
return [inlineNode, i];
|
|
576
651
|
}
|
|
@@ -579,7 +654,9 @@ function parseInline(tokens, i) {
|
|
|
579
654
|
// ========================================================================== //
|
|
580
655
|
function parseText(tokens, i, options = {}) {
|
|
581
656
|
const textNode = makeTextNode();
|
|
582
|
-
|
|
657
|
+
const startToken = current_token(tokens, i);
|
|
658
|
+
textNode.range.start = startToken.range.start;
|
|
659
|
+
textNode.depth = startToken.depth;
|
|
583
660
|
const { selectiveUnescape = false } = options;
|
|
584
661
|
|
|
585
662
|
while (i < tokens.length) {
|
|
@@ -604,14 +681,17 @@ function parseText(tokens, i, options = {}) {
|
|
|
604
681
|
} else {
|
|
605
682
|
break;
|
|
606
683
|
}
|
|
684
|
+
textNode.range.end = current_token(tokens, i - 1).range.end;
|
|
607
685
|
}
|
|
608
686
|
return [textNode, i];
|
|
609
687
|
}
|
|
610
688
|
// ========================================================================== //
|
|
611
689
|
// Parse AtBlock //
|
|
612
690
|
// ========================================================================== //
|
|
613
|
-
function parseAtBlock(tokens, i) {
|
|
691
|
+
function parseAtBlock(tokens, i, filename = null) {
|
|
614
692
|
const atBlockNode = makeAtBlockNode();
|
|
693
|
+
const openAtToken = current_token(tokens, i);
|
|
694
|
+
atBlockNode.range.start = openAtToken.range.start;
|
|
615
695
|
// ========================================================================== //
|
|
616
696
|
// consume '@_' //
|
|
617
697
|
// ========================================================================== //
|
|
@@ -712,7 +792,7 @@ function parseAtBlock(tokens, i) {
|
|
|
712
792
|
}
|
|
713
793
|
}
|
|
714
794
|
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.SEMICOLON)) {
|
|
715
|
-
parserError(errorMessage(tokens, i, ";", at_value));
|
|
795
|
+
parserError(errorMessage(tokens, i, ";", at_value, "A semicolon (;) is required after the AtBlock identifier or its arguments (e.g., '@_Table_@:' or '@_Table_@: key, val;').", filename));
|
|
716
796
|
}
|
|
717
797
|
i = parseSemiColon(tokens, i, at_value);
|
|
718
798
|
if (
|
|
@@ -754,8 +834,10 @@ function parseAtBlock(tokens, i) {
|
|
|
754
834
|
// ========================================================================== //
|
|
755
835
|
// consume '_@' //
|
|
756
836
|
// ========================================================================== //
|
|
837
|
+
const closeAtToken = current_token(tokens, i);
|
|
757
838
|
i++;
|
|
758
839
|
updateData(tokens, i);
|
|
840
|
+
atBlockNode.range.end = closeAtToken.range.end;
|
|
759
841
|
tokens_stack.length = 0;
|
|
760
842
|
return [atBlockNode, i];
|
|
761
843
|
}
|
|
@@ -764,9 +846,11 @@ function parseAtBlock(tokens, i) {
|
|
|
764
846
|
// ========================================================================== //
|
|
765
847
|
function parseCommentNode(tokens, i) {
|
|
766
848
|
const commentNode = makeCommentNode();
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
commentNode.
|
|
849
|
+
const token = current_token(tokens, i);
|
|
850
|
+
if (token && token.type === TOKEN_TYPES.COMMENT) {
|
|
851
|
+
commentNode.text = token.value;
|
|
852
|
+
commentNode.depth = token.depth;
|
|
853
|
+
commentNode.range = token.range;
|
|
770
854
|
}
|
|
771
855
|
// ========================================================================== //
|
|
772
856
|
// consume Comment '#' //
|
|
@@ -776,7 +860,11 @@ function parseCommentNode(tokens, i) {
|
|
|
776
860
|
return [commentNode, i];
|
|
777
861
|
}
|
|
778
862
|
|
|
779
|
-
|
|
863
|
+
// ========================================================================== //
|
|
864
|
+
// Main Node Dispatcher //
|
|
865
|
+
// ========================================================================== //
|
|
866
|
+
|
|
867
|
+
function parseNode(tokens, i, filename = null) {
|
|
780
868
|
if (!current_token(tokens, i) || (current_token(tokens, i) && !current_token(tokens, i).value)) {
|
|
781
869
|
return [null, i];
|
|
782
870
|
}
|
|
@@ -797,10 +885,31 @@ function parseNode(tokens, i) {
|
|
|
797
885
|
return parseBlock(tokens, i);
|
|
798
886
|
}
|
|
799
887
|
// ========================================================================== //
|
|
800
|
-
// Inline Statement
|
|
888
|
+
// Inline Statement or Text //
|
|
801
889
|
// ========================================================================== //
|
|
802
890
|
else if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.OPEN_PAREN) {
|
|
803
|
-
|
|
891
|
+
// Look ahead to see if this is an inline statement: (...) -> (...)
|
|
892
|
+
let j = i + 1;
|
|
893
|
+
let foundClose = false;
|
|
894
|
+
while (j < tokens.length) {
|
|
895
|
+
if (tokens[j].type === TOKEN_TYPES.CLOSE_PAREN) {
|
|
896
|
+
foundClose = true;
|
|
897
|
+
break;
|
|
898
|
+
}
|
|
899
|
+
// Avoid going too far if it's definitely not an inline (not matching the value part structure)
|
|
900
|
+
if (tokens[j].type === TOKEN_TYPES.OPEN_PAREN || tokens[j].type === TOKEN_TYPES.OPEN_BRACKET) break;
|
|
901
|
+
j++;
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
if (foundClose && tokens[j + 1] && tokens[j + 1].type === TOKEN_TYPES.THIN_ARROW) {
|
|
905
|
+
return parseInline(tokens, i);
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
// Treat as text if not an inline
|
|
909
|
+
const textNode = makeTextNode();
|
|
910
|
+
textNode.text = current_token(tokens, i).value;
|
|
911
|
+
textNode.range = current_token(tokens, i).range;
|
|
912
|
+
return [textNode, i + 1];
|
|
804
913
|
}
|
|
805
914
|
// ========================================================================== //
|
|
806
915
|
// Text //
|
|
@@ -815,36 +924,63 @@ function parseNode(tokens, i) {
|
|
|
815
924
|
// Atblock //
|
|
816
925
|
// ========================================================================== //
|
|
817
926
|
else if (current_token(tokens, i) && (current_token(tokens, i).type === TOKEN_TYPES.OPEN_AT)) {
|
|
818
|
-
return parseAtBlock(tokens, i);
|
|
927
|
+
return parseAtBlock(tokens, i, filename);
|
|
819
928
|
} else {
|
|
820
|
-
|
|
929
|
+
// FALLBACK: Treat any other token as TEXT to avoid infinite loops and allow literal content
|
|
930
|
+
const textNode = makeTextNode();
|
|
931
|
+
textNode.text = current_token(tokens, i).value;
|
|
932
|
+
textNode.range = current_token(tokens, i).range;
|
|
933
|
+
return [textNode, i + 1];
|
|
821
934
|
}
|
|
822
|
-
return [null, i + 1];
|
|
823
935
|
}
|
|
824
936
|
|
|
825
|
-
|
|
937
|
+
// ========================================================================== //
|
|
938
|
+
// Main Parser Entry Point //
|
|
939
|
+
// ========================================================================== //
|
|
940
|
+
|
|
941
|
+
function parser(tokens, filename = null) {
|
|
942
|
+
// Filter out structural whitespace (junk) that was emitted for highlighting purposes
|
|
943
|
+
tokens = tokens.filter(t => !t.isStructural);
|
|
826
944
|
end_stack = [];
|
|
827
945
|
tokens_stack = [];
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
946
|
+
range = {
|
|
947
|
+
start: { line: 0, character: 0 },
|
|
948
|
+
end: { line: 0, character: 0 }
|
|
949
|
+
};
|
|
831
950
|
value = "";
|
|
832
951
|
let ast = [];
|
|
833
952
|
let i = 0;
|
|
834
953
|
while (i < tokens.length) {
|
|
835
|
-
let [
|
|
836
|
-
if (
|
|
837
|
-
|
|
838
|
-
}
|
|
839
|
-
if (nodes) {
|
|
840
|
-
ast.push(nodes);
|
|
954
|
+
let [node, nextIndex] = parseNode(tokens, i, filename);
|
|
955
|
+
if (node) {
|
|
956
|
+
ast.push(node);
|
|
841
957
|
i = nextIndex;
|
|
842
958
|
} else {
|
|
843
959
|
i++;
|
|
844
960
|
}
|
|
845
961
|
}
|
|
846
962
|
if (end_stack.length !== 0) {
|
|
847
|
-
|
|
963
|
+
let extraInfo = "";
|
|
964
|
+
|
|
965
|
+
const checkTypo = (token) => {
|
|
966
|
+
if (token && token.value) {
|
|
967
|
+
const val = token.value.trim().toLowerCase();
|
|
968
|
+
if (val === "") return "";
|
|
969
|
+
const dist = levenshtein(val, "end");
|
|
970
|
+
if (dist > 0 && dist <= 2) return ` (Did you mean <$cyan:'[end]'$>?)`;
|
|
971
|
+
}
|
|
972
|
+
return "";
|
|
973
|
+
};
|
|
974
|
+
|
|
975
|
+
// Check last few tokens for a typo
|
|
976
|
+
for (let j = 1; j <= 5; j++) {
|
|
977
|
+
const token = tokens[tokens.length - j];
|
|
978
|
+
if (!token) break;
|
|
979
|
+
extraInfo = checkTypo(token);
|
|
980
|
+
if (extraInfo) break;
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
parserError(errorMessage(tokens, tokens.length - 1, "[end]", "", extraInfo ? `Missing '[end]'${extraInfo}` : "Missing '[end]'", filename));
|
|
848
984
|
}
|
|
849
985
|
return ast;
|
|
850
986
|
}
|
package/core/pluginManager.js
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plugin Manager
|
|
3
|
+
* Handles sorting by priority and running hooks at various stages.
|
|
4
|
+
*/
|
|
1
5
|
export default class PluginManager {
|
|
2
6
|
constructor(plugins = [], priority = []) {
|
|
7
|
+
// ========================================================================== //
|
|
8
|
+
// Plugin Sorting by Priority //
|
|
9
|
+
// ========================================================================== //
|
|
3
10
|
this.plugins = [...plugins].sort((a, b) => {
|
|
4
11
|
const getIndex = (p) => {
|
|
5
|
-
// ========================================================================== //
|
|
6
|
-
// Priority array can contain plugin objects or plugin names (for built-ins)
|
|
7
|
-
// ========================================================================== //
|
|
8
12
|
const index = priority.findIndex(item => {
|
|
9
13
|
if (typeof item === "string") {
|
|
10
14
|
return item === p.name;
|
|
@@ -17,29 +21,13 @@ export default class PluginManager {
|
|
|
17
21
|
const indexA = getIndex(a);
|
|
18
22
|
const indexB = getIndex(b);
|
|
19
23
|
|
|
20
|
-
|
|
21
|
-
// 1. Both have a priority index
|
|
22
|
-
// ========================================================================== //
|
|
23
|
-
if (indexA !== -1 && indexB !== -1) {
|
|
24
|
-
return indexA - indexB;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
// ========================================================================== //
|
|
28
|
-
// 2. Only A has a priority
|
|
29
|
-
// ========================================================================== //
|
|
24
|
+
if (indexA !== -1 && indexB !== -1) return indexA - indexB;
|
|
30
25
|
if (indexA !== -1) return -1;
|
|
31
|
-
|
|
32
|
-
// ========================================================================== //
|
|
33
|
-
// 3. Only B has a priority
|
|
34
|
-
// ========================================================================== //
|
|
35
26
|
if (indexB !== -1) return 1;
|
|
36
27
|
|
|
37
|
-
// ========================================================================== //
|
|
38
|
-
// 4. Neither have a priority
|
|
39
28
|
// Default rule: built-ins first, then external/user-defined
|
|
40
|
-
|
|
41
|
-
const
|
|
42
|
-
const isBuiltInB = ["module-system", "quote-escaper", "raw-content", "comment-remover", "rules-validation", "sommark-format"].includes(b.name);
|
|
29
|
+
const isBuiltInA = ["module-system", "raw-content", "comment-remover", "rules-validation", "sommark-format"].includes(a.name);
|
|
30
|
+
const isBuiltInB = ["module-system", "raw-content", "comment-remover", "rules-validation", "sommark-format"].includes(b.name);
|
|
43
31
|
|
|
44
32
|
if (isBuiltInA && !isBuiltInB) return -1;
|
|
45
33
|
if (!isBuiltInA && isBuiltInB) return 1;
|
|
@@ -48,7 +36,10 @@ export default class PluginManager {
|
|
|
48
36
|
});
|
|
49
37
|
}
|
|
50
38
|
|
|
51
|
-
|
|
39
|
+
// ========================================================================== //
|
|
40
|
+
// Preprocessing Hooks (Before Lexing) //
|
|
41
|
+
// ========================================================================== //
|
|
42
|
+
async runPreprocessor(src, scope, context) {
|
|
52
43
|
let processedSrc = src;
|
|
53
44
|
const preprocessors = this.plugins.filter(p => {
|
|
54
45
|
const types = Array.isArray(p.type) ? p.type : [p.type];
|
|
@@ -57,12 +48,16 @@ export default class PluginManager {
|
|
|
57
48
|
|
|
58
49
|
for (const plugin of preprocessors) {
|
|
59
50
|
if (typeof plugin.beforeLex === "function") {
|
|
51
|
+
plugin.context = context;
|
|
60
52
|
processedSrc = await plugin.beforeLex.call(plugin, processedSrc);
|
|
61
53
|
}
|
|
62
54
|
}
|
|
63
55
|
return processedSrc;
|
|
64
56
|
}
|
|
65
57
|
|
|
58
|
+
// ========================================================================== //
|
|
59
|
+
// Post-Lexing Hooks (Token Transformation) //
|
|
60
|
+
// ========================================================================== //
|
|
66
61
|
async runAfterLex(tokens) {
|
|
67
62
|
let processedTokens = tokens;
|
|
68
63
|
const afterLexers = this.plugins.filter(p => {
|
|
@@ -76,6 +71,9 @@ export default class PluginManager {
|
|
|
76
71
|
return processedTokens;
|
|
77
72
|
}
|
|
78
73
|
|
|
74
|
+
// ========================================================================== //
|
|
75
|
+
// AST Transformation Hooks (After Parsing) //
|
|
76
|
+
// ========================================================================== //
|
|
79
77
|
async runOnAst(ast, context = {}) {
|
|
80
78
|
let processedAst = ast;
|
|
81
79
|
const astPlugins = this.plugins.filter(p => {
|
|
@@ -89,6 +87,9 @@ export default class PluginManager {
|
|
|
89
87
|
return processedAst;
|
|
90
88
|
}
|
|
91
89
|
|
|
90
|
+
// ========================================================================== //
|
|
91
|
+
// Mapper Extensions (Tag Definitions and Rules) //
|
|
92
|
+
// ========================================================================== //
|
|
92
93
|
getMapperExtensions() {
|
|
93
94
|
const extensions = { outputs: [], rules: {} };
|
|
94
95
|
const mapperPlugins = this.plugins.filter(p => {
|
|
@@ -107,6 +108,9 @@ export default class PluginManager {
|
|
|
107
108
|
return extensions;
|
|
108
109
|
}
|
|
109
110
|
|
|
111
|
+
// ========================================================================== //
|
|
112
|
+
// Registration Hooks //
|
|
113
|
+
// ========================================================================== //
|
|
110
114
|
runRegisterHooks(sm) {
|
|
111
115
|
for (const plugin of this.plugins) {
|
|
112
116
|
const registerFn = plugin.registerOutput || plugin.register;
|
|
@@ -116,6 +120,9 @@ export default class PluginManager {
|
|
|
116
120
|
}
|
|
117
121
|
}
|
|
118
122
|
|
|
123
|
+
// ========================================================================== //
|
|
124
|
+
// Post-Processing Hooks (Final Output Transformation) //
|
|
125
|
+
// ========================================================================== //
|
|
119
126
|
async runTransformers(output) {
|
|
120
127
|
let processedOutput = output;
|
|
121
128
|
const transformers = this.plugins.filter(p => {
|
|
@@ -132,6 +139,9 @@ export default class PluginManager {
|
|
|
132
139
|
return processedOutput;
|
|
133
140
|
}
|
|
134
141
|
|
|
142
|
+
// ========================================================================== //
|
|
143
|
+
// Format-Specific Mapper Retrieval //
|
|
144
|
+
// ========================================================================== //
|
|
135
145
|
getFormatMapper(formatName) {
|
|
136
146
|
const plugin = this.plugins.find(p => p.format === formatName && p.mapper);
|
|
137
147
|
return plugin ? plugin.mapper : null;
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { COMMENT } from "../labels.js";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Comment Remover Plugin
|
|
5
|
+
* Removes all comments from the document so they don't appear in the final output.
|
|
6
6
|
*/
|
|
7
7
|
export default {
|
|
8
8
|
name: "comment-remover",
|
|
9
9
|
type: "on-ast",
|
|
10
10
|
author: "Adam-Elmi",
|
|
11
|
-
description: "Removes all
|
|
11
|
+
description: "Removes all comments from the document so they don't appear in the final output.",
|
|
12
12
|
onAst: function (ast) {
|
|
13
13
|
// ========================================================================== //
|
|
14
14
|
// Recursive function to filter out comments //
|