sommark 3.3.4 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -82
- package/assets/logo.json +28 -0
- package/assets/smark.logo.png +0 -0
- package/assets/smark.logo.svg +21 -0
- package/cli/cli.mjs +7 -17
- package/cli/commands/build.js +24 -4
- package/cli/commands/color.js +22 -26
- package/cli/commands/help.js +10 -10
- package/cli/commands/init.js +20 -31
- package/cli/commands/print.js +18 -16
- package/cli/commands/show.js +4 -0
- package/cli/commands/version.js +6 -0
- package/cli/constants.js +9 -5
- package/cli/helpers/config.js +11 -0
- package/cli/helpers/file.js +17 -6
- package/cli/helpers/transpile.js +7 -12
- package/core/errors.js +49 -25
- package/core/formats.js +7 -3
- package/core/formatter.js +215 -0
- package/core/helpers/config-loader.js +29 -74
- package/core/labels.js +21 -9
- package/core/lexer.js +491 -212
- package/core/modules.js +164 -0
- package/core/parser.js +516 -389
- package/core/tokenTypes.js +36 -1
- package/core/transpiler.js +237 -154
- package/core/validator.js +79 -0
- package/formatter/mark.js +203 -43
- package/formatter/tag.js +202 -32
- package/grammar.ebnf +57 -50
- package/helpers/colorize.js +26 -13
- package/helpers/escapeHTML.js +13 -6
- package/helpers/kebabize.js +6 -0
- package/helpers/peek.js +9 -0
- package/helpers/removeChar.js +26 -13
- package/helpers/safeDataParser.js +114 -0
- package/helpers/utils.js +140 -158
- package/index.js +198 -188
- package/mappers/languages/html.js +105 -213
- package/mappers/languages/json.js +122 -171
- package/mappers/languages/markdown.js +355 -108
- package/mappers/languages/mdx.js +76 -120
- package/mappers/languages/xml.js +114 -0
- package/mappers/mapper.js +152 -123
- package/mappers/shared/index.js +22 -0
- package/package.json +26 -6
- package/SOMMARK-SPEC.md +0 -481
- package/cli/commands/list.js +0 -124
- package/constants/html_tags.js +0 -146
- package/core/pluginManager.js +0 -149
- package/core/plugins/comment-remover.js +0 -47
- package/core/plugins/module-system.js +0 -176
- package/core/plugins/raw-content-plugin.js +0 -78
- package/core/plugins/rules-validation-plugin.js +0 -231
- package/core/plugins/sommark-format.js +0 -244
- package/coverage_test.js +0 -21
- package/debug.js +0 -15
- package/helpers/camelize.js +0 -2
- package/helpers/defaultTheme.js +0 -3
- package/test_format_fix.js +0 -42
- package/v3-todo.smark +0 -73
package/core/parser.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import TOKEN_TYPES from "./tokenTypes.js";
|
|
5
5
|
import peek from "../helpers/peek.js";
|
|
6
6
|
import { parserError } from "./errors.js";
|
|
7
|
+
import { safeDataParse } from "../helpers/safeDataParser.js";
|
|
7
8
|
import {
|
|
8
9
|
BLOCK,
|
|
9
10
|
TEXT,
|
|
@@ -13,10 +14,12 @@ import {
|
|
|
13
14
|
IMPORT,
|
|
14
15
|
USE_MODULE,
|
|
15
16
|
block_id,
|
|
17
|
+
block_key,
|
|
16
18
|
block_value,
|
|
17
19
|
inline_id,
|
|
18
|
-
|
|
20
|
+
inline_text,
|
|
19
21
|
at_id,
|
|
22
|
+
atblock_key,
|
|
20
23
|
at_value,
|
|
21
24
|
end_keyword
|
|
22
25
|
} from "./labels.js";
|
|
@@ -26,27 +29,70 @@ import { levenshtein } from "../helpers/utils.js";
|
|
|
26
29
|
// Helper Functions //
|
|
27
30
|
// ========================================================================== //
|
|
28
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Returns the token at the current position.
|
|
34
|
+
*
|
|
35
|
+
* @param {Object[]} tokens - The list of tokens.
|
|
36
|
+
* @param {number} i - The current index.
|
|
37
|
+
* @returns {Object|null} - The token or null if at the end.
|
|
38
|
+
*/
|
|
29
39
|
function current_token(tokens, i) {
|
|
30
40
|
return tokens[i] || null;
|
|
31
41
|
}
|
|
32
42
|
|
|
43
|
+
/**
|
|
44
|
+
* Skip whitespaces and comments in structural contexts.
|
|
45
|
+
*
|
|
46
|
+
* @param {Object[]} tokens - The list of tokens.
|
|
47
|
+
* @param {number} i - The current index.
|
|
48
|
+
* @returns {number} - The new index.
|
|
49
|
+
*/
|
|
50
|
+
function skipJunk(tokens, i) {
|
|
51
|
+
while (i < tokens.length) {
|
|
52
|
+
const t = tokens[i];
|
|
53
|
+
const type = t.type;
|
|
54
|
+
if (type === TOKEN_TYPES.WHITESPACE || type === TOKEN_TYPES.COMMENT) {
|
|
55
|
+
i++;
|
|
56
|
+
} else if (type === TOKEN_TYPES.TEXT && t.value.trim() === "") {
|
|
57
|
+
i++;
|
|
58
|
+
} else {
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return i;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Checks if a name is valid (using letters, numbers, and certain symbols).
|
|
67
|
+
*
|
|
68
|
+
* @param {string} id - The name to check.
|
|
69
|
+
* @param {RegExp} [keyRegex] - The rule to follow.
|
|
70
|
+
* @param {string} [name] - The type of thing we are checking.
|
|
71
|
+
* @param {string} [rule] - A human-readable version of the rule.
|
|
72
|
+
* @param {string} [ruleMessage] - The error message to show.
|
|
73
|
+
*/
|
|
33
74
|
function validateName(
|
|
34
75
|
id,
|
|
35
|
-
|
|
36
|
-
name = "Identifier"
|
|
37
|
-
rule = "(A–Z, a–z, 0–9, -, _, $)",
|
|
38
|
-
ruleMessage = "must contain only letters, numbers, hyphens, underscores, or dollar signs ($)"
|
|
76
|
+
allowColon = false,
|
|
77
|
+
name = "Identifier"
|
|
39
78
|
) {
|
|
79
|
+
const keyRegex = allowColon ? /^[a-zA-Z0-9\-_$:]+$/ : /^[a-zA-Z0-9\-_$]+$/;
|
|
80
|
+
const rule = allowColon ? "(A–Z, a–z, 0–9, -, _, $, :)" : "(A–Z, a–z, 0–9, -, _, $)";
|
|
81
|
+
const ruleMessage = allowColon
|
|
82
|
+
? "must contain only letters, numbers, hyphens, underscores, dollar signs ($), or colons (:)"
|
|
83
|
+
: "must contain only letters, numbers, hyphens, underscores, or dollar signs ($)";
|
|
84
|
+
|
|
40
85
|
if (!keyRegex.test(id)) {
|
|
41
86
|
parserError([`{line}<$red:Invalid ${name}:$><$blue: '${id}'$>{N}<$yellow:${name} ${ruleMessage}$> <$cyan: ${rule}.$>{line}`]);
|
|
42
87
|
}
|
|
43
88
|
}
|
|
44
89
|
|
|
90
|
+
/** Creates a new empty Block node. */
|
|
45
91
|
function makeBlockNode() {
|
|
46
92
|
return {
|
|
47
93
|
type: BLOCK,
|
|
48
94
|
id: "",
|
|
49
|
-
args:
|
|
95
|
+
args: {},
|
|
50
96
|
body: [],
|
|
51
97
|
depth: 0,
|
|
52
98
|
range: {
|
|
@@ -55,7 +101,7 @@ function makeBlockNode() {
|
|
|
55
101
|
}
|
|
56
102
|
};
|
|
57
103
|
}
|
|
58
|
-
|
|
104
|
+
/** Creates a new empty Text node. */
|
|
59
105
|
function makeTextNode() {
|
|
60
106
|
return {
|
|
61
107
|
type: TEXT,
|
|
@@ -67,7 +113,7 @@ function makeTextNode() {
|
|
|
67
113
|
}
|
|
68
114
|
};
|
|
69
115
|
}
|
|
70
|
-
|
|
116
|
+
/** Creates a new empty Comment node. */
|
|
71
117
|
function makeCommentNode() {
|
|
72
118
|
return {
|
|
73
119
|
type: COMMENT,
|
|
@@ -79,13 +125,13 @@ function makeCommentNode() {
|
|
|
79
125
|
}
|
|
80
126
|
};
|
|
81
127
|
}
|
|
82
|
-
|
|
128
|
+
/** Creates a new empty Inline node. */
|
|
83
129
|
function makeInlineNode() {
|
|
84
130
|
return {
|
|
85
131
|
type: INLINE,
|
|
86
132
|
value: "",
|
|
87
133
|
id: "",
|
|
88
|
-
args:
|
|
134
|
+
args: {},
|
|
89
135
|
depth: 0,
|
|
90
136
|
range: {
|
|
91
137
|
start: { line: 0, character: 0 },
|
|
@@ -95,14 +141,14 @@ function makeInlineNode() {
|
|
|
95
141
|
}
|
|
96
142
|
|
|
97
143
|
// ========================================================================== //
|
|
98
|
-
// Node Creators
|
|
144
|
+
// Node Creators //
|
|
99
145
|
// ========================================================================== //
|
|
100
|
-
|
|
146
|
+
/** Creates a new empty AtBlock node. */
|
|
101
147
|
function makeAtBlockNode() {
|
|
102
148
|
return {
|
|
103
149
|
type: ATBLOCK,
|
|
104
150
|
id: "",
|
|
105
|
-
args:
|
|
151
|
+
args: {},
|
|
106
152
|
content: "",
|
|
107
153
|
depth: 0,
|
|
108
154
|
range: {
|
|
@@ -150,6 +196,7 @@ const errorMessage = (tokens, i, expectedValue, behindValue, frontText, filename
|
|
|
150
196
|
let lineStartIndex = i;
|
|
151
197
|
while (
|
|
152
198
|
lineStartIndex > 0 &&
|
|
199
|
+
tokens[lineStartIndex - 1] &&
|
|
153
200
|
tokens[lineStartIndex - 1].range.start.line === errorLineNumber &&
|
|
154
201
|
(tokens[lineStartIndex - 1].source || filename) === source
|
|
155
202
|
) {
|
|
@@ -159,6 +206,7 @@ const errorMessage = (tokens, i, expectedValue, behindValue, frontText, filename
|
|
|
159
206
|
let lineEndIndex = i;
|
|
160
207
|
while (
|
|
161
208
|
lineEndIndex < tokens.length - 1 &&
|
|
209
|
+
tokens[lineEndIndex + 1] &&
|
|
162
210
|
tokens[lineEndIndex + 1].range.start.line === errorLineNumber &&
|
|
163
211
|
(tokens[lineEndIndex + 1].source || filename) === source
|
|
164
212
|
) {
|
|
@@ -192,43 +240,118 @@ const errorMessage = (tokens, i, expectedValue, behindValue, frontText, filename
|
|
|
192
240
|
// Parse Key //
|
|
193
241
|
// ========================================================================== //
|
|
194
242
|
function parseKey(tokens, i) {
|
|
195
|
-
let key =
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
243
|
+
let key = "";
|
|
244
|
+
if (current_token(tokens, i).type === TOKEN_TYPES.QUOTE) {
|
|
245
|
+
i++; // consume opening QUOTE
|
|
246
|
+
key = current_token(tokens, i).value;
|
|
247
|
+
i++; // consume Key
|
|
248
|
+
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.QUOTE) {
|
|
249
|
+
i++; // consume closing QUOTE
|
|
250
|
+
}
|
|
251
|
+
} else {
|
|
252
|
+
key = current_token(tokens, i).value.trim();
|
|
253
|
+
i++;
|
|
254
|
+
}
|
|
200
255
|
updateData(tokens, i);
|
|
201
256
|
return [key, i];
|
|
202
257
|
}
|
|
203
258
|
// ========================================================================== //
|
|
204
259
|
// Parse Value //
|
|
205
260
|
// ========================================================================== //
|
|
206
|
-
function parseValue(tokens, i) {
|
|
261
|
+
function parseValue(tokens, i, placeholders = {}) {
|
|
207
262
|
let val = current_token(tokens, i).value;
|
|
208
263
|
// consume Value
|
|
209
|
-
i
|
|
264
|
+
if (current_token(tokens, i).type === TOKEN_TYPES.QUOTE) {
|
|
265
|
+
i++; // consume opening QUOTE
|
|
266
|
+
val = "";
|
|
267
|
+
while (i < tokens.length && current_token(tokens, i).type !== TOKEN_TYPES.QUOTE) {
|
|
268
|
+
const token = current_token(tokens, i);
|
|
269
|
+
if (token.type === TOKEN_TYPES.PREFIX_P || token.type === TOKEN_TYPES.PREFIX_JS) {
|
|
270
|
+
const [resolvedVal, nextI] = parseValue(tokens, i, placeholders);
|
|
271
|
+
val += resolvedVal;
|
|
272
|
+
i = nextI;
|
|
273
|
+
} else {
|
|
274
|
+
val += token.value;
|
|
275
|
+
i++;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (i >= tokens.length) {
|
|
280
|
+
parserError(errorMessage(tokens, i - 1, "\"", "unclosed string", "Unclosed quote"));
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
i++; // consume closing QUOTE
|
|
284
|
+
return [val, i, true];
|
|
285
|
+
} else if (current_token(tokens, i).type === TOKEN_TYPES.PREFIX_JS) {
|
|
286
|
+
val = current_token(tokens, i).value;
|
|
287
|
+
// V4 NATIVE DATA: Strip js{ } and parse safely
|
|
288
|
+
if (val.startsWith("js{") && val.endsWith("}")) {
|
|
289
|
+
const clean = val.slice(3, -1).trim();
|
|
290
|
+
val = safeDataParse(clean);
|
|
291
|
+
}
|
|
292
|
+
i++;
|
|
293
|
+
return [val, i, false];
|
|
294
|
+
} else if (current_token(tokens, i).type === TOKEN_TYPES.PREFIX_P) {
|
|
295
|
+
val = current_token(tokens, i).value;
|
|
296
|
+
// V4 PLACEHOLDER: Strip p{ } and resolve from config
|
|
297
|
+
if (val.startsWith("p{") && val.endsWith("}")) {
|
|
298
|
+
const key = val.slice(2, -1).trim();
|
|
299
|
+
val = placeholders[key] !== undefined ? placeholders[key] : val;
|
|
300
|
+
}
|
|
301
|
+
i++;
|
|
302
|
+
return [val, i, false];
|
|
303
|
+
} else {
|
|
304
|
+
val = "";
|
|
305
|
+
while (i < tokens.length) {
|
|
306
|
+
const token = current_token(tokens, i);
|
|
307
|
+
if (!token) break;
|
|
308
|
+
|
|
309
|
+
// Stop at any structural marker or whitespace
|
|
310
|
+
if (token.type === TOKEN_TYPES.WHITESPACE ||
|
|
311
|
+
token.type === TOKEN_TYPES.COMMA ||
|
|
312
|
+
token.type === TOKEN_TYPES.CLOSE_BRACKET ||
|
|
313
|
+
token.type === TOKEN_TYPES.COLON ||
|
|
314
|
+
token.type === TOKEN_TYPES.SEMICOLON ||
|
|
315
|
+
token.type === TOKEN_TYPES.CLOSE_PAREN) break;
|
|
316
|
+
|
|
317
|
+
if (token.type === TOKEN_TYPES.ESCAPE) {
|
|
318
|
+
// Remove backslash
|
|
319
|
+
val += token.value.slice(1);
|
|
320
|
+
} else {
|
|
321
|
+
val += token.value;
|
|
322
|
+
}
|
|
323
|
+
i++;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
210
327
|
updateData(tokens, i);
|
|
211
|
-
return [val, i];
|
|
328
|
+
return [val, i, false];
|
|
212
329
|
}
|
|
213
330
|
// ========================================================================== //
|
|
214
331
|
// Parse ',' //
|
|
215
332
|
// ========================================================================== //
|
|
216
333
|
function parseComma(tokens, i, beforeChar = "") {
|
|
217
|
-
|
|
218
|
-
// consume ',' //
|
|
219
|
-
// ========================================================================== //
|
|
220
|
-
i++;
|
|
221
|
-
updateData(tokens, i);
|
|
334
|
+
i = skipJunk(tokens, i);
|
|
222
335
|
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COMMA) {
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
current_token(tokens, i)
|
|
229
|
-
current_token(tokens, i)
|
|
230
|
-
|
|
231
|
-
|
|
336
|
+
i++;
|
|
337
|
+
i = skipJunk(tokens, i);
|
|
338
|
+
updateData(tokens, i);
|
|
339
|
+
|
|
340
|
+
if (
|
|
341
|
+
!current_token(tokens, i) ||
|
|
342
|
+
(current_token(tokens, i) &&
|
|
343
|
+
current_token(tokens, i).type !== TOKEN_TYPES.VALUE &&
|
|
344
|
+
current_token(tokens, i).type !== TOKEN_TYPES.ESCAPE &&
|
|
345
|
+
current_token(tokens, i).type !== TOKEN_TYPES.IDENTIFIER &&
|
|
346
|
+
current_token(tokens, i).type !== TOKEN_TYPES.KEY &&
|
|
347
|
+
current_token(tokens, i).type !== TOKEN_TYPES.QUOTE &&
|
|
348
|
+
current_token(tokens, i).type !== TOKEN_TYPES.PREFIX_JS &&
|
|
349
|
+
current_token(tokens, i).type !== TOKEN_TYPES.PREFIX_P)
|
|
350
|
+
) {
|
|
351
|
+
parserError(errorMessage(tokens, i, "value", ","));
|
|
352
|
+
}
|
|
353
|
+
} else {
|
|
354
|
+
parserError(errorMessage(tokens, i, ",", beforeChar));
|
|
232
355
|
}
|
|
233
356
|
return i;
|
|
234
357
|
}
|
|
@@ -248,47 +371,48 @@ function parseEscape(tokens, i) {
|
|
|
248
371
|
// Parse ':' //
|
|
249
372
|
// ========================================================================== //
|
|
250
373
|
function parseColon(tokens, i, afterChar = "") {
|
|
374
|
+
i = skipJunk(tokens, i);
|
|
251
375
|
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.COLON)) {
|
|
252
376
|
parserError(errorMessage(tokens, i, ":", afterChar));
|
|
253
377
|
}
|
|
254
|
-
// ========================================================================== //
|
|
255
|
-
// consume ':' //
|
|
256
|
-
// ========================================================================== //
|
|
257
378
|
i++;
|
|
379
|
+
i = skipJunk(tokens, i);
|
|
258
380
|
updateData(tokens, i);
|
|
259
|
-
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COLON) {
|
|
260
|
-
parserError(errorMessage(tokens, i, ":", "", "Found extra"));
|
|
261
|
-
}
|
|
262
381
|
return i;
|
|
263
382
|
}
|
|
264
383
|
// ========================================================================== //
|
|
265
384
|
// Parse ';' //
|
|
266
385
|
// ========================================================================== //
|
|
267
386
|
function parseSemiColon(tokens, i, afterChar = "") {
|
|
387
|
+
i = skipJunk(tokens, i);
|
|
268
388
|
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.SEMICOLON)) {
|
|
269
389
|
parserError(errorMessage(tokens, i, ";", afterChar));
|
|
270
390
|
}
|
|
271
|
-
// ========================================================================== //
|
|
272
|
-
// consume ';' //
|
|
273
|
-
// ========================================================================== //
|
|
274
391
|
i++;
|
|
392
|
+
i = skipJunk(tokens, i);
|
|
275
393
|
updateData(tokens, i);
|
|
276
|
-
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.SEMICOLON) {
|
|
277
|
-
parserError(errorMessage(tokens, i, ";", "", "Found extra"));
|
|
278
|
-
}
|
|
279
394
|
return i;
|
|
280
395
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
396
|
+
/**
|
|
397
|
+
* Parses a standard SomMark Block ([id] ... [end]).
|
|
398
|
+
* Blocks are structural elements that can contain nested content.
|
|
399
|
+
*
|
|
400
|
+
* @param {Object[]} tokens - Token stream.
|
|
401
|
+
* @param {number} i - Initial index.
|
|
402
|
+
* @param {string|null} filename - Source filename.
|
|
403
|
+
* @param {Object} placeholders - Dynamic public API data.
|
|
404
|
+
* @returns {[Object, number]} The parsed Block node and new index.
|
|
405
|
+
*/
|
|
406
|
+
function parseBlock(tokens, i, filename = null, placeholders = {}) {
|
|
285
407
|
const blockNode = makeBlockNode();
|
|
286
408
|
const openBracketToken = current_token(tokens, i);
|
|
287
409
|
// ========================================================================== //
|
|
288
410
|
// consume '[' //
|
|
289
411
|
// ========================================================================== //
|
|
290
412
|
i++;
|
|
413
|
+
i = skipJunk(tokens, i);
|
|
291
414
|
updateData(tokens, i);
|
|
415
|
+
|
|
292
416
|
const idToken = current_token(tokens, i);
|
|
293
417
|
if (!idToken || idToken.type === TOKEN_TYPES.EOF) {
|
|
294
418
|
parserError(errorMessage(tokens, i, "Block ID", "[", "Missing Block Identifier"));
|
|
@@ -306,7 +430,7 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
306
430
|
} else if (blockNode.id === "$use-module") {
|
|
307
431
|
blockNode.type = USE_MODULE;
|
|
308
432
|
}
|
|
309
|
-
validateName(blockNode.id);
|
|
433
|
+
validateName(blockNode.id, true);
|
|
310
434
|
blockNode.depth = idToken.depth;
|
|
311
435
|
blockNode.range.start = openBracketToken.range.start;
|
|
312
436
|
end_stack.push(id);
|
|
@@ -314,19 +438,29 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
314
438
|
// consume Block Identifier //
|
|
315
439
|
// ========================================================================== //
|
|
316
440
|
i++;
|
|
441
|
+
i = skipJunk(tokens, i);
|
|
317
442
|
updateData(tokens, i);
|
|
318
443
|
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.EQUAL) {
|
|
319
444
|
// ========================================================================== //
|
|
320
445
|
// consume '=' //
|
|
321
446
|
// ========================================================================== //
|
|
322
447
|
i++;
|
|
448
|
+
i = skipJunk(tokens, i);
|
|
323
449
|
updateData(tokens, i);
|
|
450
|
+
|
|
324
451
|
if (
|
|
325
452
|
!current_token(tokens, i) ||
|
|
326
453
|
(current_token(tokens, i) &&
|
|
327
454
|
current_token(tokens, i).type !== TOKEN_TYPES.VALUE &&
|
|
328
455
|
current_token(tokens, i).type !== TOKEN_TYPES.ESCAPE &&
|
|
329
|
-
current_token(tokens, i).type !== TOKEN_TYPES.IDENTIFIER
|
|
456
|
+
current_token(tokens, i).type !== TOKEN_TYPES.IDENTIFIER &&
|
|
457
|
+
current_token(tokens, i).type !== TOKEN_TYPES.IMPORT &&
|
|
458
|
+
current_token(tokens, i).type !== TOKEN_TYPES.USE_MODULE &&
|
|
459
|
+
current_token(tokens, i).type !== TOKEN_TYPES.END_KEYWORD &&
|
|
460
|
+
current_token(tokens, i).type !== TOKEN_TYPES.KEY &&
|
|
461
|
+
current_token(tokens, i).type !== TOKEN_TYPES.QUOTE &&
|
|
462
|
+
current_token(tokens, i).type !== TOKEN_TYPES.PREFIX_JS &&
|
|
463
|
+
current_token(tokens, i).type !== TOKEN_TYPES.PREFIX_P)
|
|
330
464
|
) {
|
|
331
465
|
parserError(errorMessage(tokens, i, block_value, "="));
|
|
332
466
|
}
|
|
@@ -335,87 +469,73 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
335
469
|
// ========================================================================== //
|
|
336
470
|
let k = "";
|
|
337
471
|
let v = "";
|
|
472
|
+
let vIsQuoted = false;
|
|
473
|
+
let argIndex = 0;
|
|
338
474
|
while (i < tokens.length) {
|
|
339
|
-
|
|
475
|
+
i = skipJunk(tokens, i);
|
|
476
|
+
const token = current_token(tokens, i);
|
|
477
|
+
if (!token || token.type === TOKEN_TYPES.CLOSE_BRACKET) break;
|
|
478
|
+
|
|
479
|
+
const isQuotedKey = token.type === TOKEN_TYPES.QUOTE && peek(tokens, i, 1) && (peek(tokens, i, 1).type === TOKEN_TYPES.KEY);
|
|
480
|
+
|
|
481
|
+
if (token.type === TOKEN_TYPES.KEY || isQuotedKey) {
|
|
340
482
|
let [key, keyIndex] = parseKey(tokens, i);
|
|
341
483
|
k = key;
|
|
342
484
|
i = keyIndex;
|
|
343
|
-
|
|
344
|
-
i = parseColon(tokens, i,
|
|
345
|
-
|
|
346
|
-
parserError(errorMessage(tokens, i, block_value, ":"));
|
|
347
|
-
}
|
|
348
|
-
validateName(k);
|
|
349
|
-
continue;
|
|
350
|
-
} else if (
|
|
351
|
-
current_token(tokens, i) &&
|
|
352
|
-
(current_token(tokens, i).type === TOKEN_TYPES.VALUE || current_token(tokens, i).type === TOKEN_TYPES.ESCAPE)
|
|
353
|
-
) {
|
|
354
|
-
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.ESCAPE) {
|
|
355
|
-
let [escape_character, escapeIndex] = parseEscape(tokens, i);
|
|
356
|
-
v += escape_character;
|
|
357
|
-
i = escapeIndex;
|
|
358
|
-
} else {
|
|
359
|
-
let [value, valueIndex] = parseValue(tokens, i);
|
|
360
|
-
v += value;
|
|
361
|
-
i = valueIndex;
|
|
362
|
-
}
|
|
485
|
+
i = skipJunk(tokens, i);
|
|
486
|
+
i = parseColon(tokens, i, block_key);
|
|
487
|
+
i = skipJunk(tokens, i);
|
|
363
488
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
(
|
|
368
|
-
) {
|
|
369
|
-
if (current_token(tokens, i).type === TOKEN_TYPES.ESCAPE) {
|
|
370
|
-
let [escape_character, escapeIndex] = parseEscape(tokens, i);
|
|
371
|
-
v += escape_character;
|
|
372
|
-
i = escapeIndex;
|
|
373
|
-
} else {
|
|
374
|
-
let [value, valueIndex] = parseValue(tokens, i);
|
|
375
|
-
v += value;
|
|
376
|
-
i = valueIndex;
|
|
377
|
-
}
|
|
489
|
+
// Ensure there is a value after the colon
|
|
490
|
+
const nextToken = current_token(tokens, i);
|
|
491
|
+
if (!nextToken || nextToken.type === TOKEN_TYPES.CLOSE_BRACKET || nextToken.type === TOKEN_TYPES.COMMA) {
|
|
492
|
+
parserError(errorMessage(tokens, i, block_value, ":", "Missing value after colon"));
|
|
378
493
|
}
|
|
379
494
|
|
|
380
|
-
if
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
v = v.slice(1, -1);
|
|
384
|
-
}
|
|
385
|
-
blockNode.args.push(v);
|
|
386
|
-
if (k) {
|
|
387
|
-
blockNode.args[k] = v;
|
|
388
|
-
}
|
|
389
|
-
k = "";
|
|
390
|
-
v = "";
|
|
391
|
-
i = parseComma(tokens, i, block_value);
|
|
392
|
-
continue;
|
|
495
|
+
// Validate only if it was a plain KEY token (not from a quote)
|
|
496
|
+
if (token.type === TOKEN_TYPES.KEY) {
|
|
497
|
+
validateName(k, true);
|
|
393
498
|
}
|
|
394
|
-
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// Parse Value (handles both quoted, unquoted, and prefixes)
|
|
502
|
+
let [value, valueIndex, isQuoted] = parseValue(tokens, i, placeholders);
|
|
503
|
+
v = value;
|
|
504
|
+
vIsQuoted = isQuoted;
|
|
505
|
+
i = valueIndex;
|
|
506
|
+
|
|
507
|
+
// Store Argument
|
|
508
|
+
blockNode.args[String(argIndex++)] = v;
|
|
509
|
+
if (k) {
|
|
510
|
+
blockNode.args[k] = v;
|
|
511
|
+
}
|
|
512
|
+
k = "";
|
|
513
|
+
v = "";
|
|
514
|
+
|
|
515
|
+
i = skipJunk(tokens, i);
|
|
516
|
+
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COMMA) {
|
|
517
|
+
i = parseComma(tokens, i, block_value);
|
|
395
518
|
} else {
|
|
519
|
+
// No comma, must be end of arguments or ]
|
|
396
520
|
break;
|
|
397
521
|
}
|
|
398
522
|
}
|
|
399
523
|
if (v !== "") {
|
|
400
|
-
v
|
|
401
|
-
|
|
402
|
-
v
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
if (k) {
|
|
406
|
-
blockNode.args[k] = v;
|
|
524
|
+
if (typeof v === "string") {
|
|
525
|
+
if (!vIsQuoted) v = v.trim();
|
|
526
|
+
if (v.startsWith('"') && v.endsWith('"')) {
|
|
527
|
+
v = v.slice(1, -1);
|
|
528
|
+
}
|
|
407
529
|
}
|
|
408
530
|
}
|
|
409
531
|
}
|
|
532
|
+
|
|
533
|
+
i = skipJunk(tokens, i);
|
|
410
534
|
// ========================================================================== //
|
|
411
535
|
// Close Bracket //
|
|
412
536
|
// ========================================================================== //
|
|
413
537
|
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.CLOSE_BRACKET)) {
|
|
414
|
-
|
|
415
|
-
parserError(errorMessage(tokens, i, "]", block_value));
|
|
416
|
-
} else {
|
|
417
|
-
parserError(errorMessage(tokens, i, "]", block_id));
|
|
418
|
-
}
|
|
538
|
+
parserError(errorMessage(tokens, i, "]", block_id));
|
|
419
539
|
}
|
|
420
540
|
// ========================================================================== //
|
|
421
541
|
// consume ']' //
|
|
@@ -424,13 +544,16 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
424
544
|
updateData(tokens, i);
|
|
425
545
|
tokens_stack.length = 0;
|
|
426
546
|
while (i < tokens.length) {
|
|
547
|
+
const nextIdx = skipJunk(tokens, i + 1);
|
|
548
|
+
const nextToken = tokens[nextIdx];
|
|
427
549
|
if (
|
|
428
550
|
current_token(tokens, i) &&
|
|
429
551
|
current_token(tokens, i).type === TOKEN_TYPES.OPEN_BRACKET &&
|
|
430
|
-
|
|
431
|
-
|
|
552
|
+
nextToken &&
|
|
553
|
+
nextToken.type !== TOKEN_TYPES.END_KEYWORD &&
|
|
554
|
+
nextToken.value.trim() !== end_keyword
|
|
432
555
|
) {
|
|
433
|
-
const [childNode, nextIndex] = parseBlock(tokens, i, filename);
|
|
556
|
+
const [childNode, nextIndex] = parseBlock(tokens, i, filename, placeholders);
|
|
434
557
|
blockNode.body.push(childNode);
|
|
435
558
|
// ========================================================================== //
|
|
436
559
|
// consume child node //
|
|
@@ -439,13 +562,14 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
439
562
|
} else if (
|
|
440
563
|
current_token(tokens, i) &&
|
|
441
564
|
current_token(tokens, i).type === TOKEN_TYPES.OPEN_BRACKET &&
|
|
442
|
-
|
|
443
|
-
(
|
|
565
|
+
nextToken &&
|
|
566
|
+
(nextToken.type === TOKEN_TYPES.END_KEYWORD || nextToken.value.trim() === end_keyword)
|
|
444
567
|
) {
|
|
445
568
|
// ========================================================================== //
|
|
446
569
|
// consume '[' //
|
|
447
570
|
// ========================================================================== //
|
|
448
571
|
i++;
|
|
572
|
+
i = skipJunk(tokens, i);
|
|
449
573
|
const current = current_token(tokens, i);
|
|
450
574
|
if (!current || (current.type !== TOKEN_TYPES.END_KEYWORD && current.value.trim() !== end_keyword)) {
|
|
451
575
|
let extraInfo = "";
|
|
@@ -461,6 +585,7 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
461
585
|
// consume End Keyword //
|
|
462
586
|
// ========================================================================== //
|
|
463
587
|
i++;
|
|
588
|
+
i = skipJunk(tokens, i);
|
|
464
589
|
updateData(tokens, i);
|
|
465
590
|
if (
|
|
466
591
|
!current_token(tokens, i) ||
|
|
@@ -478,7 +603,7 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
478
603
|
blockNode.range.end = closeBracketToken.range.end;
|
|
479
604
|
break;
|
|
480
605
|
} else {
|
|
481
|
-
const [childNode, nextIndex] = parseNode(tokens, i, filename);
|
|
606
|
+
const [childNode, nextIndex] = parseNode(tokens, i, filename, placeholders);
|
|
482
607
|
if (childNode) {
|
|
483
608
|
blockNode.body.push(childNode);
|
|
484
609
|
i = nextIndex;
|
|
@@ -489,170 +614,143 @@ function parseBlock(tokens, i, filename = null) {
|
|
|
489
614
|
}
|
|
490
615
|
return [blockNode, i];
|
|
491
616
|
}
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
617
|
+
/**
|
|
618
|
+
* Parses an Inline Statement ((content) -> (id)).
|
|
619
|
+
* Inlines are fast, non-nesting formatting elements.
|
|
620
|
+
*
|
|
621
|
+
* @param {Object[]} tokens - Token stream.
|
|
622
|
+
* @param {number} i - Initial index.
|
|
623
|
+
* @param {Object} placeholders - Dynamic public API data.
|
|
624
|
+
* @returns {[Object, number]} The parsed Inline node and new index.
|
|
625
|
+
*/
|
|
626
|
+
function parseInline(tokens, i, placeholders = {}) {
|
|
496
627
|
const inlineNode = makeInlineNode();
|
|
497
628
|
const openParenToken = current_token(tokens, i);
|
|
498
629
|
inlineNode.range.start = openParenToken.range.start;
|
|
499
|
-
|
|
500
|
-
//
|
|
501
|
-
// ========================================================================== //
|
|
630
|
+
|
|
631
|
+
// consume '('
|
|
502
632
|
i++;
|
|
503
633
|
updateData(tokens, i);
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
}
|
|
634
|
+
|
|
635
|
+
// Phase 1: Content capture (Lexer provides high-level TEXT/ESCAPE tokens here)
|
|
507
636
|
while (i < tokens.length) {
|
|
508
637
|
const token = current_token(tokens, i);
|
|
509
|
-
if (!token || token.type === TOKEN_TYPES.CLOSE_PAREN)
|
|
510
|
-
|
|
511
|
-
}
|
|
638
|
+
if (!token || token.type === TOKEN_TYPES.CLOSE_PAREN) break;
|
|
639
|
+
|
|
512
640
|
if (token.type === TOKEN_TYPES.ESCAPE) {
|
|
513
641
|
inlineNode.value += token.value.slice(1);
|
|
514
642
|
} else {
|
|
515
643
|
inlineNode.value += token.value;
|
|
516
644
|
}
|
|
517
645
|
i++;
|
|
518
|
-
updateData(tokens, i);
|
|
519
646
|
}
|
|
520
|
-
|
|
521
|
-
|
|
647
|
+
|
|
648
|
+
if (!current_token(tokens, i) || current_token(tokens, i).type !== TOKEN_TYPES.CLOSE_PAREN) {
|
|
649
|
+
parserError(errorMessage(tokens, i, ")", "inline content"));
|
|
522
650
|
}
|
|
523
|
-
//
|
|
524
|
-
|
|
525
|
-
//
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
651
|
+
i++; // consume ')'
|
|
652
|
+
|
|
653
|
+
// Collapse newlines and whitespace for "inline" behavior
|
|
654
|
+
inlineNode.value = inlineNode.value.replace(/\s+/g, " ").trim();
|
|
655
|
+
|
|
656
|
+
i = skipJunk(tokens, i);
|
|
657
|
+
if (!current_token(tokens, i) || current_token(tokens, i).type !== TOKEN_TYPES.THIN_ARROW) {
|
|
529
658
|
parserError(errorMessage(tokens, i, "->", ")"));
|
|
530
659
|
}
|
|
531
|
-
//
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
i
|
|
535
|
-
updateData(tokens, i);
|
|
536
|
-
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.OPEN_PAREN)) {
|
|
660
|
+
i++; // consume '->'
|
|
661
|
+
|
|
662
|
+
i = skipJunk(tokens, i);
|
|
663
|
+
if (!current_token(tokens, i) || current_token(tokens, i).type !== TOKEN_TYPES.OPEN_PAREN) {
|
|
537
664
|
parserError(errorMessage(tokens, i, "(", "->"));
|
|
538
665
|
}
|
|
539
|
-
//
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
updateData(tokens, i);
|
|
544
|
-
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.IDENTIFIER)) {
|
|
666
|
+
i++; // consume '('
|
|
667
|
+
i = skipJunk(tokens, i);
|
|
668
|
+
const idToken = current_token(tokens, i);
|
|
669
|
+
if (!idToken || (idToken.type !== TOKEN_TYPES.IDENTIFIER && idToken.type !== TOKEN_TYPES.KEY)) {
|
|
545
670
|
parserError(errorMessage(tokens, i, inline_id, "("));
|
|
546
671
|
}
|
|
547
|
-
inlineNode.id =
|
|
548
|
-
if (inlineNode.id === end_keyword) {
|
|
549
|
-
parserError(errorMessage(tokens, i, inlineNode.id, "", `'${inlineNode.id}' is a reserved keyword and cannot be used as an identifier.`));
|
|
550
|
-
}
|
|
672
|
+
inlineNode.id = idToken.value.trim();
|
|
551
673
|
validateName(inlineNode.id);
|
|
552
|
-
|
|
553
|
-
//
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
updateData(tokens, i);
|
|
674
|
+
|
|
675
|
+
i++; // consume ID
|
|
676
|
+
i = skipJunk(tokens, i);
|
|
677
|
+
|
|
557
678
|
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COLON) {
|
|
558
|
-
i
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
parserError(errorMessage(tokens, i, inline_value, ":"));
|
|
679
|
+
i++; // consume ':'
|
|
680
|
+
i = skipJunk(tokens, i);
|
|
681
|
+
|
|
682
|
+
// Ensure there is a value after the colon
|
|
683
|
+
const nextToken = current_token(tokens, i);
|
|
684
|
+
if (!nextToken || nextToken.type === TOKEN_TYPES.CLOSE_PAREN || nextToken.type === TOKEN_TYPES.COMMA) {
|
|
685
|
+
parserError(errorMessage(tokens, i, inline_value, ":", "Missing value after colon"));
|
|
566
686
|
}
|
|
687
|
+
|
|
688
|
+
let k = "";
|
|
567
689
|
let v = "";
|
|
568
|
-
|
|
569
|
-
if (v !== "") {
|
|
570
|
-
v = v.trim();
|
|
571
|
-
if (v.startsWith('"') && v.endsWith('"')) {
|
|
572
|
-
v = v.slice(1, -1);
|
|
573
|
-
}
|
|
574
|
-
inlineNode.args.push(v);
|
|
575
|
-
v = "";
|
|
576
|
-
}
|
|
577
|
-
};
|
|
690
|
+
let argIndex = 0;
|
|
578
691
|
|
|
579
692
|
while (i < tokens.length) {
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
) {
|
|
584
|
-
if (current_token(tokens, i).type === TOKEN_TYPES.ESCAPE) {
|
|
585
|
-
// Escape Character
|
|
586
|
-
const [escape_character, escapeIndex] = parseEscape(tokens, i);
|
|
587
|
-
v += escape_character;
|
|
588
|
-
i = escapeIndex;
|
|
589
|
-
} else {
|
|
590
|
-
// Value
|
|
591
|
-
const [value, valueIndex] = parseValue(tokens, i);
|
|
592
|
-
v += value;
|
|
593
|
-
i = valueIndex;
|
|
594
|
-
}
|
|
693
|
+
i = skipJunk(tokens, i);
|
|
694
|
+
const token = current_token(tokens, i);
|
|
695
|
+
if (!token || token.type === TOKEN_TYPES.CLOSE_PAREN) break;
|
|
595
696
|
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
)
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
v += escape_character;
|
|
604
|
-
i = escapeIndex;
|
|
605
|
-
} else {
|
|
606
|
-
const [value, valueIndex] = parseValue(tokens, i);
|
|
607
|
-
v += value;
|
|
608
|
-
i = valueIndex;
|
|
609
|
-
}
|
|
610
|
-
}
|
|
697
|
+
if (token.type === TOKEN_TYPES.KEY) {
|
|
698
|
+
let [key, keyIndex] = parseKey(tokens, i);
|
|
699
|
+
k = key;
|
|
700
|
+
i = keyIndex;
|
|
701
|
+
i = skipJunk(tokens, i);
|
|
702
|
+
i = parseColon(tokens, i, "inline argument");
|
|
703
|
+
i = skipJunk(tokens, i);
|
|
611
704
|
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
// ========================================================================== //
|
|
617
|
-
i++;
|
|
618
|
-
updateData(tokens, i);
|
|
619
|
-
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COMMA) {
|
|
620
|
-
parserError(errorMessage(tokens, i, ",", "", "Found extra"));
|
|
621
|
-
}
|
|
622
|
-
if (
|
|
623
|
-
!current_token(tokens, i) ||
|
|
624
|
-
(current_token(tokens, i) &&
|
|
625
|
-
current_token(tokens, i).type !== TOKEN_TYPES.VALUE &&
|
|
626
|
-
current_token(tokens, i).type !== TOKEN_TYPES.ESCAPE)
|
|
627
|
-
) {
|
|
628
|
-
parserError(errorMessage(tokens, i, inline_value, ","));
|
|
629
|
-
}
|
|
630
|
-
continue;
|
|
705
|
+
// Ensure there is a value after the colon
|
|
706
|
+
const nextToken = current_token(tokens, i);
|
|
707
|
+
if (!nextToken || nextToken.type === TOKEN_TYPES.CLOSE_PAREN || nextToken.type === TOKEN_TYPES.COMMA) {
|
|
708
|
+
parserError(errorMessage(tokens, i, inline_value, ":", "Missing value after colon"));
|
|
631
709
|
}
|
|
632
|
-
|
|
710
|
+
validateName(k);
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
let [value, valueIndex, isQuoted] = parseValue(tokens, i, placeholders);
|
|
714
|
+
v = value;
|
|
715
|
+
i = valueIndex;
|
|
716
|
+
|
|
717
|
+
inlineNode.args[String(argIndex++)] = v;
|
|
718
|
+
if (k) {
|
|
719
|
+
inlineNode.args[k] = v;
|
|
720
|
+
}
|
|
721
|
+
k = "";
|
|
722
|
+
v = "";
|
|
723
|
+
|
|
724
|
+
i = skipJunk(tokens, i);
|
|
725
|
+
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COMMA) {
|
|
726
|
+
i = parseComma(tokens, i, "inline argument");
|
|
633
727
|
} else {
|
|
634
728
|
break;
|
|
635
729
|
}
|
|
636
730
|
}
|
|
637
|
-
pushArg();
|
|
638
731
|
}
|
|
639
|
-
|
|
640
|
-
|
|
732
|
+
|
|
733
|
+
i = skipJunk(tokens, i);
|
|
734
|
+
if (!current_token(tokens, i) || current_token(tokens, i).type !== TOKEN_TYPES.CLOSE_PAREN) {
|
|
735
|
+
parserError(errorMessage(tokens, i, ")", inlineNode.id));
|
|
641
736
|
}
|
|
642
|
-
// ========================================================================== //
|
|
643
|
-
// consume ')' //
|
|
644
|
-
// ========================================================================== //
|
|
645
737
|
const finalParenToken = current_token(tokens, i);
|
|
646
|
-
i++;
|
|
647
|
-
updateData(tokens, i);
|
|
738
|
+
i++; // consume ')'
|
|
648
739
|
inlineNode.range.end = finalParenToken.range.end;
|
|
649
|
-
|
|
740
|
+
|
|
650
741
|
return [inlineNode, i];
|
|
651
742
|
}
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
743
|
+
/**
|
|
744
|
+
* Parses a stream of text tokens into a single Text node.
|
|
745
|
+
* Handles unescaping and placeholder resolution.
|
|
746
|
+
*
|
|
747
|
+
* @param {Object[]} tokens - Token stream.
|
|
748
|
+
* @param {number} i - Initial index.
|
|
749
|
+
* @param {Object} placeholders - Dynamic public API data.
|
|
750
|
+
* @param {Object} options - Formatting options.
|
|
751
|
+
* @returns {[Object, number]} The Text node and new index.
|
|
752
|
+
*/
|
|
753
|
+
function parseText(tokens, i, placeholders = {}, options = {}) {
|
|
656
754
|
const textNode = makeTextNode();
|
|
657
755
|
const startToken = current_token(tokens, i);
|
|
658
756
|
textNode.range.start = startToken.range.start;
|
|
@@ -661,11 +759,12 @@ function parseText(tokens, i, options = {}) {
|
|
|
661
759
|
|
|
662
760
|
while (i < tokens.length) {
|
|
663
761
|
const token = current_token(tokens, i);
|
|
664
|
-
if (token
|
|
762
|
+
if (!token) break;
|
|
763
|
+
|
|
764
|
+
if (token.type === TOKEN_TYPES.TEXT || token.type === TOKEN_TYPES.WHITESPACE || token.type === TOKEN_TYPES.VALUE) {
|
|
665
765
|
textNode.text += token.value;
|
|
666
766
|
i++;
|
|
667
|
-
|
|
668
|
-
} else if (token && token.type === TOKEN_TYPES.ESCAPE) {
|
|
767
|
+
} else if (token.type === TOKEN_TYPES.ESCAPE) {
|
|
669
768
|
if (selectiveUnescape) {
|
|
670
769
|
const char = token.value.slice(1);
|
|
671
770
|
if (char === "@" || char === "_") {
|
|
@@ -677,168 +776,164 @@ function parseText(tokens, i, options = {}) {
|
|
|
677
776
|
textNode.text += token.value.slice(1); // Standard behavior: unescape all
|
|
678
777
|
}
|
|
679
778
|
i++;
|
|
680
|
-
|
|
779
|
+
} else if (token.type === TOKEN_TYPES.PREFIX_P) {
|
|
780
|
+
const val = token.value;
|
|
781
|
+
if (val.startsWith("p{") && val.endsWith("}")) {
|
|
782
|
+
const key = val.slice(2, -1).trim();
|
|
783
|
+
textNode.text += placeholders[key] !== undefined ? String(placeholders[key]) : val;
|
|
784
|
+
} else {
|
|
785
|
+
textNode.text += val;
|
|
786
|
+
}
|
|
787
|
+
i++;
|
|
681
788
|
} else {
|
|
682
789
|
break;
|
|
683
790
|
}
|
|
684
|
-
|
|
791
|
+
|
|
792
|
+
updateData(tokens, i);
|
|
793
|
+
textNode.range.end = tokens[i - 1].range.end;
|
|
685
794
|
}
|
|
686
795
|
return [textNode, i];
|
|
687
796
|
}
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
797
|
+
/**
|
|
798
|
+
* Parses an At-Block (@_id_@: args; content @_end_@).
|
|
799
|
+
* At-Blocks maintain raw content preservation.
|
|
800
|
+
*
|
|
801
|
+
* @param {Object[]} tokens - Token stream.
|
|
802
|
+
* @param {number} i - Initial index.
|
|
803
|
+
* @param {string|null} filename - Source filename.
|
|
804
|
+
* @param {Object} placeholders - Dynamic public API data.
|
|
805
|
+
* @returns {[Object, number]} The At-Block node and new index.
|
|
806
|
+
*/
|
|
807
|
+
function parseAtBlock(tokens, i, filename = null, placeholders = {}) {
|
|
692
808
|
const atBlockNode = makeAtBlockNode();
|
|
693
809
|
const openAtToken = current_token(tokens, i);
|
|
694
810
|
atBlockNode.range.start = openAtToken.range.start;
|
|
695
|
-
|
|
696
|
-
//
|
|
697
|
-
// ========================================================================== //
|
|
811
|
+
|
|
812
|
+
// consume '@_'
|
|
698
813
|
i++;
|
|
814
|
+
i = skipJunk(tokens, i);
|
|
699
815
|
updateData(tokens, i);
|
|
700
|
-
|
|
816
|
+
|
|
817
|
+
const idToken = current_token(tokens, i);
|
|
818
|
+
if (!idToken || idToken.type === TOKEN_TYPES.EOF) {
|
|
819
|
+
parserError(errorMessage(tokens, i, "AtBlock ID", "@_", "Missing AtBlock Identifier"));
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
const id = idToken.value;
|
|
701
823
|
if (id.trim() === end_keyword) {
|
|
702
824
|
parserError(errorMessage(tokens, i, id, "", `'${id.trim()}' is a reserved keyword and cannot be used as an identifier.`));
|
|
703
825
|
}
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
}
|
|
711
|
-
// ========================================================================== //
|
|
712
|
-
// consume Atblock Identifier //
|
|
713
|
-
// ========================================================================== //
|
|
826
|
+
|
|
827
|
+
atBlockNode.id = id.trim();
|
|
828
|
+
validateName(atBlockNode.id);
|
|
829
|
+
atBlockNode.depth = idToken.depth;
|
|
830
|
+
|
|
831
|
+
// consume ID
|
|
714
832
|
i++;
|
|
833
|
+
i = skipJunk(tokens, i);
|
|
715
834
|
updateData(tokens, i);
|
|
835
|
+
|
|
716
836
|
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.CLOSE_AT)) {
|
|
717
|
-
parserError(errorMessage(tokens, i, "_@",
|
|
837
|
+
parserError(errorMessage(tokens, i, "_@", "at-block identifier"));
|
|
718
838
|
}
|
|
719
|
-
//
|
|
720
|
-
// consume '_@' //
|
|
721
|
-
// ========================================================================== //
|
|
839
|
+
// consume '_@'
|
|
722
840
|
i++;
|
|
841
|
+
i = skipJunk(tokens, i);
|
|
723
842
|
updateData(tokens, i);
|
|
843
|
+
|
|
724
844
|
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COLON) {
|
|
725
|
-
//
|
|
726
|
-
// consume ':' //
|
|
727
|
-
// ========================================================================== //
|
|
845
|
+
// consume ':'
|
|
728
846
|
i++;
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
if (
|
|
734
|
-
|
|
735
|
-
(current_token(tokens, i) &&
|
|
736
|
-
current_token(tokens, i).type !== TOKEN_TYPES.IDENTIFIER &&
|
|
737
|
-
current_token(tokens, i).type !== TOKEN_TYPES.VALUE &&
|
|
738
|
-
current_token(tokens, i).type !== TOKEN_TYPES.ESCAPE)
|
|
739
|
-
) {
|
|
740
|
-
parserError(errorMessage(tokens, i, `${at_id} or ${at_value}`, ":"));
|
|
847
|
+
i = skipJunk(tokens, i);
|
|
848
|
+
|
|
849
|
+
// Ensure there is a value after the colon
|
|
850
|
+
const nextToken = current_token(tokens, i);
|
|
851
|
+
if (!nextToken || nextToken.type === TOKEN_TYPES.SEMICOLON || nextToken.type === TOKEN_TYPES.COMMA) {
|
|
852
|
+
parserError(errorMessage(tokens, i, at_value, ":", "Missing value after colon"));
|
|
741
853
|
}
|
|
854
|
+
|
|
742
855
|
let k = "";
|
|
743
856
|
let v = "";
|
|
857
|
+
let argIndex = 0;
|
|
858
|
+
|
|
744
859
|
while (i < tokens.length) {
|
|
745
|
-
|
|
860
|
+
i = skipJunk(tokens, i);
|
|
861
|
+
const token = current_token(tokens, i);
|
|
862
|
+
if (!token || token.type === TOKEN_TYPES.SEMICOLON) break;
|
|
863
|
+
|
|
864
|
+
const isQuotedKey = token.type === TOKEN_TYPES.QUOTE && peek(tokens, i, 1) && (peek(tokens, i, 1).type === TOKEN_TYPES.KEY);
|
|
865
|
+
|
|
866
|
+
if (token.type === TOKEN_TYPES.KEY || isQuotedKey) {
|
|
746
867
|
let [key, keyIndex] = parseKey(tokens, i);
|
|
747
868
|
k = key;
|
|
748
869
|
i = keyIndex;
|
|
749
|
-
i =
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
v += escape_character;
|
|
758
|
-
i = escapeIndex;
|
|
759
|
-
continue;
|
|
760
|
-
} else if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.VALUE) {
|
|
761
|
-
let [value, valueIndex] = parseValue(tokens, i);
|
|
762
|
-
v += value;
|
|
763
|
-
i = valueIndex;
|
|
764
|
-
for (let e = i; e < tokens.length; e++) {
|
|
765
|
-
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.ESCAPE) {
|
|
766
|
-
let [escape_character, escapeIndex] = parseEscape(tokens, i);
|
|
767
|
-
v += escape_character;
|
|
768
|
-
i = escapeIndex;
|
|
769
|
-
continue;
|
|
770
|
-
} else {
|
|
771
|
-
break;
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
v = v.trim();
|
|
775
|
-
if (v.startsWith('"') && v.endsWith('"')) {
|
|
776
|
-
v = v.slice(1, -1);
|
|
777
|
-
}
|
|
778
|
-
atBlockNode.args.push(v);
|
|
779
|
-
if (k) {
|
|
780
|
-
atBlockNode.args[k] = v;
|
|
870
|
+
i = skipJunk(tokens, i);
|
|
871
|
+
i = parseColon(tokens, i, "at-block argument");
|
|
872
|
+
i = skipJunk(tokens, i);
|
|
873
|
+
|
|
874
|
+
// Ensure there is a value after the colon
|
|
875
|
+
const nextToken = current_token(tokens, i);
|
|
876
|
+
if (!nextToken || nextToken.type === TOKEN_TYPES.SEMICOLON || nextToken.type === TOKEN_TYPES.COMMA) {
|
|
877
|
+
parserError(errorMessage(tokens, i, at_value, ":", "Missing value after colon"));
|
|
781
878
|
}
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
i = parseComma(tokens, i, at_value);
|
|
786
|
-
continue;
|
|
879
|
+
|
|
880
|
+
if (token.type === TOKEN_TYPES.KEY) {
|
|
881
|
+
validateName(k);
|
|
787
882
|
}
|
|
788
|
-
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
let [value, valueIndex, isQuoted] = parseValue(tokens, i, placeholders);
|
|
886
|
+
v = value;
|
|
887
|
+
i = valueIndex;
|
|
888
|
+
|
|
889
|
+
atBlockNode.args[String(argIndex++)] = v;
|
|
890
|
+
if (k) {
|
|
891
|
+
atBlockNode.args[k] = v;
|
|
892
|
+
}
|
|
893
|
+
k = "";
|
|
894
|
+
v = "";
|
|
895
|
+
|
|
896
|
+
i = skipJunk(tokens, i);
|
|
897
|
+
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.COMMA) {
|
|
898
|
+
i = parseComma(tokens, i, "at-block argument");
|
|
789
899
|
} else {
|
|
790
900
|
break;
|
|
791
901
|
}
|
|
792
902
|
}
|
|
793
903
|
}
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
parserError(errorMessage(tokens, i, "
|
|
805
|
-
}
|
|
806
|
-
if (
|
|
807
|
-
current_token(tokens, i) &&
|
|
808
|
-
(current_token(tokens, i).type === TOKEN_TYPES.TEXT || current_token(tokens, i).type === TOKEN_TYPES.ESCAPE)
|
|
809
|
-
) {
|
|
810
|
-
const [childNode, nextIndex] = parseText(tokens, i, { selectiveUnescape: true });
|
|
811
|
-
atBlockNode.content = childNode.text;
|
|
812
|
-
i = nextIndex;
|
|
813
|
-
updateData(tokens, i);
|
|
904
|
+
|
|
905
|
+
// Semicolon is ALWAYS required after ID or ARGS
|
|
906
|
+
i = parseSemiColon(tokens, i, "at-block header");
|
|
907
|
+
|
|
908
|
+
// Body Capture
|
|
909
|
+
i = skipJunk(tokens, i);
|
|
910
|
+
if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.TEXT) {
|
|
911
|
+
atBlockNode.content = current_token(tokens, i).value;
|
|
912
|
+
i++;
|
|
913
|
+
} else {
|
|
914
|
+
parserError(errorMessage(tokens, i, "content", "at-block body"));
|
|
814
915
|
}
|
|
815
|
-
|
|
816
|
-
|
|
916
|
+
|
|
917
|
+
// End Marker (@_end_@)
|
|
918
|
+
i = skipJunk(tokens, i);
|
|
919
|
+
if (!current_token(tokens, i) || current_token(tokens, i).type !== TOKEN_TYPES.OPEN_AT) {
|
|
920
|
+
parserError(errorMessage(tokens, i, "@_", "at-block content"));
|
|
817
921
|
}
|
|
818
|
-
//
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.END_KEYWORD && current_token(tokens, i).value.trim() !== end_keyword)) {
|
|
824
|
-
parserError(errorMessage(tokens, i, end_keyword, "@_"));
|
|
922
|
+
i++; // consume '@_'
|
|
923
|
+
i = skipJunk(tokens, i);
|
|
924
|
+
const endToken = current_token(tokens, i);
|
|
925
|
+
if (!endToken || (endToken.type !== TOKEN_TYPES.END_KEYWORD && endToken.value.trim() !== end_keyword)) {
|
|
926
|
+
parserError(errorMessage(tokens, i, "end", "@_"));
|
|
825
927
|
}
|
|
826
|
-
//
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
updateData(tokens, i);
|
|
831
|
-
if (!current_token(tokens, i) || (current_token(tokens, i) && current_token(tokens, i).type !== TOKEN_TYPES.CLOSE_AT)) {
|
|
832
|
-
parserError(errorMessage(tokens, i, "_@", end_keyword));
|
|
928
|
+
i++; // consume 'end'
|
|
929
|
+
i = skipJunk(tokens, i);
|
|
930
|
+
if (!current_token(tokens, i) || current_token(tokens, i).type !== TOKEN_TYPES.CLOSE_AT) {
|
|
931
|
+
parserError(errorMessage(tokens, i, "_@", "end marker"));
|
|
833
932
|
}
|
|
834
|
-
// ========================================================================== //
|
|
835
|
-
// consume '_@' //
|
|
836
|
-
// ========================================================================== //
|
|
837
933
|
const closeAtToken = current_token(tokens, i);
|
|
838
|
-
i++;
|
|
839
|
-
updateData(tokens, i);
|
|
934
|
+
i++; // consume '_@'
|
|
840
935
|
atBlockNode.range.end = closeAtToken.range.end;
|
|
841
|
-
|
|
936
|
+
|
|
842
937
|
return [atBlockNode, i];
|
|
843
938
|
}
|
|
844
939
|
// ========================================================================== //
|
|
@@ -864,7 +959,16 @@ function parseCommentNode(tokens, i) {
|
|
|
864
959
|
// Main Node Dispatcher //
|
|
865
960
|
// ========================================================================== //
|
|
866
961
|
|
|
867
|
-
|
|
962
|
+
/**
|
|
963
|
+
* Dispatches the current token to the appropriate specialized parser function.
|
|
964
|
+
*
|
|
965
|
+
* @param {Object[]} tokens - Token stream.
|
|
966
|
+
* @param {number} i - Initial index.
|
|
967
|
+
* @param {string|null} filename - Source filename.
|
|
968
|
+
* @param {Object} placeholders - Dynamic public API data.
|
|
969
|
+
* @returns {[Object, number]} The parsed node and new index.
|
|
970
|
+
*/
|
|
971
|
+
function parseNode(tokens, i, filename = null, placeholders = {}) {
|
|
868
972
|
if (!current_token(tokens, i) || (current_token(tokens, i) && !current_token(tokens, i).value)) {
|
|
869
973
|
return [null, i];
|
|
870
974
|
}
|
|
@@ -878,31 +982,38 @@ function parseNode(tokens, i, filename = null) {
|
|
|
878
982
|
// Block or Reserved Keyword //
|
|
879
983
|
// ========================================================================== //
|
|
880
984
|
else if (current_token(tokens, i) && (current_token(tokens, i).type === TOKEN_TYPES.OPEN_BRACKET)) {
|
|
881
|
-
|
|
882
|
-
if (next && (next.type === TOKEN_TYPES.END_KEYWORD || next.value.trim() === end_keyword)) {
|
|
883
|
-
parserError(errorMessage(tokens, i + 1, "Block ID", "[", `'${next.value.trim()}' is a reserved keyword and cannot be used as a start identifier.`));
|
|
884
|
-
}
|
|
885
|
-
return parseBlock(tokens, i);
|
|
985
|
+
return parseBlock(tokens, i, filename, placeholders);
|
|
886
986
|
}
|
|
887
987
|
// ========================================================================== //
|
|
888
988
|
// Inline Statement or Text //
|
|
889
989
|
// ========================================================================== //
|
|
890
990
|
else if (current_token(tokens, i) && current_token(tokens, i).type === TOKEN_TYPES.OPEN_PAREN) {
|
|
891
|
-
// Look ahead to see if this is an inline statement: (...) -> (...)
|
|
892
991
|
let j = i + 1;
|
|
893
|
-
let
|
|
992
|
+
let parenCount = 1;
|
|
993
|
+
let foundArrow = false;
|
|
894
994
|
while (j < tokens.length) {
|
|
895
|
-
|
|
896
|
-
|
|
995
|
+
const token = tokens[j];
|
|
996
|
+
if (token.type === TOKEN_TYPES.OPEN_PAREN) {
|
|
997
|
+
parenCount++;
|
|
998
|
+
} else if (token.type === TOKEN_TYPES.CLOSE_PAREN) {
|
|
999
|
+
parenCount--;
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
if (parenCount === 0) {
|
|
1003
|
+
const nextIdx = skipJunk(tokens, j + 1);
|
|
1004
|
+
if (tokens[nextIdx] && tokens[nextIdx].type === TOKEN_TYPES.THIN_ARROW) {
|
|
1005
|
+
foundArrow = true;
|
|
1006
|
+
}
|
|
897
1007
|
break;
|
|
898
1008
|
}
|
|
899
|
-
//
|
|
900
|
-
|
|
1009
|
+
// Safe-guard: If we hit a [ or @, it's highly unlikely to be an inline statement content
|
|
1010
|
+
// unless it's escaped, but lexer already handles [ and @ as structural tokens if not escaped.
|
|
1011
|
+
if (token.type === TOKEN_TYPES.OPEN_BRACKET || token.type === TOKEN_TYPES.OPEN_AT) break;
|
|
901
1012
|
j++;
|
|
902
1013
|
}
|
|
903
1014
|
|
|
904
|
-
if (
|
|
905
|
-
return parseInline(tokens, i);
|
|
1015
|
+
if (foundArrow) {
|
|
1016
|
+
return parseInline(tokens, i, placeholders);
|
|
906
1017
|
}
|
|
907
1018
|
|
|
908
1019
|
// Treat as text if not an inline
|
|
@@ -912,19 +1023,26 @@ function parseNode(tokens, i, filename = null) {
|
|
|
912
1023
|
return [textNode, i + 1];
|
|
913
1024
|
}
|
|
914
1025
|
// ========================================================================== //
|
|
915
|
-
// Text
|
|
1026
|
+
// Text or Placeholder //
|
|
1027
|
+
// ========================================================================== //
|
|
1028
|
+
// ========================================================================== //
|
|
1029
|
+
// Text or Placeholder //
|
|
916
1030
|
// ========================================================================== //
|
|
917
1031
|
else if (
|
|
918
1032
|
current_token(tokens, i) &&
|
|
919
|
-
(current_token(tokens, i).type === TOKEN_TYPES.TEXT ||
|
|
1033
|
+
(current_token(tokens, i).type === TOKEN_TYPES.TEXT ||
|
|
1034
|
+
current_token(tokens, i).type === TOKEN_TYPES.WHITESPACE ||
|
|
1035
|
+
current_token(tokens, i).type === TOKEN_TYPES.ESCAPE ||
|
|
1036
|
+
current_token(tokens, i).type === TOKEN_TYPES.VALUE ||
|
|
1037
|
+
current_token(tokens, i).type === TOKEN_TYPES.PREFIX_P)
|
|
920
1038
|
) {
|
|
921
|
-
return parseText(tokens, i);
|
|
1039
|
+
return parseText(tokens, i, placeholders);
|
|
922
1040
|
}
|
|
923
1041
|
// ========================================================================== //
|
|
924
1042
|
// Atblock //
|
|
925
1043
|
// ========================================================================== //
|
|
926
1044
|
else if (current_token(tokens, i) && (current_token(tokens, i).type === TOKEN_TYPES.OPEN_AT)) {
|
|
927
|
-
return parseAtBlock(tokens, i, filename);
|
|
1045
|
+
return parseAtBlock(tokens, i, filename, placeholders);
|
|
928
1046
|
} else {
|
|
929
1047
|
// FALLBACK: Treat any other token as TEXT to avoid infinite loops and allow literal content
|
|
930
1048
|
const textNode = makeTextNode();
|
|
@@ -938,9 +1056,18 @@ function parseNode(tokens, i, filename = null) {
|
|
|
938
1056
|
// Main Parser Entry Point //
|
|
939
1057
|
// ========================================================================== //
|
|
940
1058
|
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
1059
|
+
/**
|
|
1060
|
+
* SomMark Parser Entry Point.
|
|
1061
|
+
*
|
|
1062
|
+
* Orchestrates the recursive descent parsing of the token stream into a
|
|
1063
|
+
* hierarchical Abstract Syntax Tree (AST).
|
|
1064
|
+
*
|
|
1065
|
+
* @param {Object[]} tokens - The stream of tokens from the Lexer.
|
|
1066
|
+
* @param {string|null} [filename=null] - Source filename for error context.
|
|
1067
|
+
* @param {Object} [placeholders={}] - Global data for p{keyword} resolution.
|
|
1068
|
+
* @returns {Array<Object>} The final Abstract Syntax Tree.
|
|
1069
|
+
*/
|
|
1070
|
+
function parser(tokens, filename = null, placeholders = {}) {
|
|
944
1071
|
end_stack = [];
|
|
945
1072
|
tokens_stack = [];
|
|
946
1073
|
range = {
|
|
@@ -951,7 +1078,7 @@ function parser(tokens, filename = null) {
|
|
|
951
1078
|
let ast = [];
|
|
952
1079
|
let i = 0;
|
|
953
1080
|
while (i < tokens.length) {
|
|
954
|
-
let [node, nextIndex] = parseNode(tokens, i, filename);
|
|
1081
|
+
let [node, nextIndex] = parseNode(tokens, i, filename, placeholders);
|
|
955
1082
|
if (node) {
|
|
956
1083
|
ast.push(node);
|
|
957
1084
|
i = nextIndex;
|