toolcraft 0.0.80 → 0.0.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -225,18 +225,27 @@ Available token classes:
|
|
|
225
225
|
- `.tc-token-type`
|
|
226
226
|
- `.tc-token-variable`
|
|
227
227
|
|
|
228
|
-
The
|
|
228
|
+
The no-dependency highlighters cover these fence labels:
|
|
229
229
|
|
|
230
230
|
- ECMAScript and TypeScript: `js`, `javascript`, `mjs`, `cjs`, `es6`, `jsx`, `ts`, `typescript`, `mts`, `cts`, `tsx`
|
|
231
231
|
- Data: `json`, `jsonc`, `jsonl`, `yaml`, `yml`
|
|
232
|
-
- CSS: `css`
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
-
|
|
237
|
-
-
|
|
238
|
-
-
|
|
239
|
-
-
|
|
232
|
+
- CSS and style dialects: `css`, `scss`, `sass`, `less`, `postcss`
|
|
233
|
+
- Shell: `sh`, `bash`, `shell`, `shellscript`, `zsh`, `fish`
|
|
234
|
+
- Python: `py`, `python`
|
|
235
|
+
- SQL: `sql`, `ddl`, `dml`
|
|
236
|
+
- Markup and Markdown: `html`, `xml`, `svg`, `md`, `markdown`
|
|
237
|
+
- Line-oriented formats: `diff`, `patch`, `dockerfile`, `docker`
|
|
238
|
+
- Config formats: `ini`, `properties`, `toml`
|
|
239
|
+
- Ruby: `rb`, `ruby`
|
|
240
|
+
- Go: `go`, `golang`
|
|
241
|
+
- Java: `java`
|
|
242
|
+
- C-family: `c`, `cpp`, `c++`, `cc`, `cxx`, `cs`, `csharp`, `c#`
|
|
243
|
+
- Rust: `rs`, `rust`
|
|
244
|
+
- PHP: `php`
|
|
245
|
+
|
|
246
|
+
These language labels intentionally render as plain escaped code:
|
|
247
|
+
|
|
248
|
+
- Plain text: `text`, `txt`, `plain`, `plaintext`
|
|
240
249
|
|
|
241
250
|
Unknown fence labels also render as plain escaped code. Code text is always escaped in HTML output, even when `allowRawHtml: true` is enabled.
|
|
242
251
|
|
|
@@ -18,29 +18,29 @@ const codeLanguages = [
|
|
|
18
18
|
{ id: "jsonl", aliases: ["jsonl"], family: "data", spec: "json" },
|
|
19
19
|
{ id: "yaml", aliases: ["yaml", "yml"], family: "data", spec: "yaml" },
|
|
20
20
|
{ id: "css", aliases: ["css"], family: "style", spec: "css" },
|
|
21
|
-
{ id: "scss", aliases: ["scss"] },
|
|
22
|
-
{ id: "sass", aliases: ["sass"] },
|
|
23
|
-
{ id: "less", aliases: ["less"] },
|
|
24
|
-
{ id: "postcss", aliases: ["postcss"] },
|
|
25
|
-
{ id: "shellscript", aliases: ["sh", "bash", "shell", "shellscript", "zsh", "fish"] },
|
|
26
|
-
{ id: "python", aliases: ["py", "python"] },
|
|
27
|
-
{ id: "sql", aliases: ["sql", "ddl", "dml"] },
|
|
28
|
-
{ id: "html", aliases: ["html"] },
|
|
29
|
-
{ id: "xml", aliases: ["xml", "svg"] },
|
|
30
|
-
{ id: "markdown", aliases: ["md", "markdown"] },
|
|
31
|
-
{ id: "diff", aliases: ["diff", "patch"] },
|
|
32
|
-
{ id: "dockerfile", aliases: ["dockerfile", "docker"] },
|
|
33
|
-
{ id: "ini", aliases: ["ini", "properties"] },
|
|
34
|
-
{ id: "toml", aliases: ["toml"] },
|
|
21
|
+
{ id: "scss", aliases: ["scss"], family: "style", spec: "css" },
|
|
22
|
+
{ id: "sass", aliases: ["sass"], family: "style", spec: "css" },
|
|
23
|
+
{ id: "less", aliases: ["less"], family: "style", spec: "css" },
|
|
24
|
+
{ id: "postcss", aliases: ["postcss"], family: "style", spec: "css" },
|
|
25
|
+
{ id: "shellscript", aliases: ["sh", "bash", "shell", "shellscript", "zsh", "fish"], family: "lexical", spec: "shell" },
|
|
26
|
+
{ id: "python", aliases: ["py", "python"], family: "lexical", spec: "python" },
|
|
27
|
+
{ id: "sql", aliases: ["sql", "ddl", "dml"], family: "lexical", spec: "sql" },
|
|
28
|
+
{ id: "html", aliases: ["html"], family: "markup", spec: "html" },
|
|
29
|
+
{ id: "xml", aliases: ["xml", "svg"], family: "markup", spec: "xml" },
|
|
30
|
+
{ id: "markdown", aliases: ["md", "markdown"], family: "line", spec: "markdown" },
|
|
31
|
+
{ id: "diff", aliases: ["diff", "patch"], family: "line", spec: "diff" },
|
|
32
|
+
{ id: "dockerfile", aliases: ["dockerfile", "docker"], family: "line", spec: "dockerfile" },
|
|
33
|
+
{ id: "ini", aliases: ["ini", "properties"], family: "data", spec: "ini" },
|
|
34
|
+
{ id: "toml", aliases: ["toml"], family: "data", spec: "toml" },
|
|
35
35
|
{ id: "plaintext", aliases: ["text", "txt", "plain", "plaintext"], plain: true },
|
|
36
|
-
{ id: "ruby", aliases: ["rb", "ruby"] },
|
|
37
|
-
{ id: "go", aliases: ["go", "golang"] },
|
|
38
|
-
{ id: "java", aliases: ["java"] },
|
|
39
|
-
{ id: "c", aliases: ["c"] },
|
|
40
|
-
{ id: "cpp", aliases: ["cpp", "c++", "cc", "cxx"] },
|
|
41
|
-
{ id: "csharp", aliases: ["cs", "csharp", "c#"] },
|
|
42
|
-
{ id: "rust", aliases: ["rs", "rust"] },
|
|
43
|
-
{ id: "php", aliases: ["php"] }
|
|
36
|
+
{ id: "ruby", aliases: ["rb", "ruby"], family: "lexical", spec: "ruby" },
|
|
37
|
+
{ id: "go", aliases: ["go", "golang"], family: "lexical", spec: "go" },
|
|
38
|
+
{ id: "java", aliases: ["java"], family: "lexical", spec: "java" },
|
|
39
|
+
{ id: "c", aliases: ["c"], family: "lexical", spec: "c" },
|
|
40
|
+
{ id: "cpp", aliases: ["cpp", "c++", "cc", "cxx"], family: "lexical", spec: "cpp" },
|
|
41
|
+
{ id: "csharp", aliases: ["cs", "csharp", "c#"], family: "lexical", spec: "csharp" },
|
|
42
|
+
{ id: "rust", aliases: ["rs", "rust"], family: "lexical", spec: "rust" },
|
|
43
|
+
{ id: "php", aliases: ["php"], family: "lexical", spec: "php" }
|
|
44
44
|
];
|
|
45
45
|
const languageByAlias = new Map();
|
|
46
46
|
for (const language of codeLanguages) {
|
|
@@ -121,10 +121,122 @@ const tsTypes = new Set([
|
|
|
121
121
|
"void"
|
|
122
122
|
]);
|
|
123
123
|
const jsConstants = new Set(["true", "false", "null", "undefined", "NaN", "Infinity"]);
|
|
124
|
+
const pythonKeywords = new Set([
|
|
125
|
+
"and",
|
|
126
|
+
"as",
|
|
127
|
+
"assert",
|
|
128
|
+
"async",
|
|
129
|
+
"await",
|
|
130
|
+
"break",
|
|
131
|
+
"case",
|
|
132
|
+
"class",
|
|
133
|
+
"continue",
|
|
134
|
+
"def",
|
|
135
|
+
"del",
|
|
136
|
+
"elif",
|
|
137
|
+
"else",
|
|
138
|
+
"except",
|
|
139
|
+
"finally",
|
|
140
|
+
"for",
|
|
141
|
+
"from",
|
|
142
|
+
"global",
|
|
143
|
+
"if",
|
|
144
|
+
"import",
|
|
145
|
+
"in",
|
|
146
|
+
"is",
|
|
147
|
+
"lambda",
|
|
148
|
+
"match",
|
|
149
|
+
"nonlocal",
|
|
150
|
+
"not",
|
|
151
|
+
"or",
|
|
152
|
+
"pass",
|
|
153
|
+
"raise",
|
|
154
|
+
"return",
|
|
155
|
+
"try",
|
|
156
|
+
"while",
|
|
157
|
+
"with",
|
|
158
|
+
"yield"
|
|
159
|
+
]);
|
|
160
|
+
const pythonTypes = new Set(["Any", "Callable", "Iterable", "None", "Protocol", "Self", "TypeVar", "bool", "bytes", "dict", "float", "int", "list", "object", "set", "str", "tuple"]);
|
|
161
|
+
const pythonBooleans = new Set(["True", "False"]);
|
|
162
|
+
const pythonNulls = new Set(["None", "NotImplemented", "Ellipsis"]);
|
|
163
|
+
const shellKeywords = new Set(["case", "do", "done", "elif", "else", "esac", "fi", "for", "function", "if", "in", "select", "then", "until", "while"]);
|
|
164
|
+
const shellCommands = new Set(["awk", "cat", "cd", "chmod", "cp", "echo", "env", "export", "find", "grep", "mkdir", "mv", "printf", "pwd", "rm", "sed", "test"]);
|
|
165
|
+
const sqlKeywords = new Set([
|
|
166
|
+
"add",
|
|
167
|
+
"alter",
|
|
168
|
+
"and",
|
|
169
|
+
"as",
|
|
170
|
+
"by",
|
|
171
|
+
"case",
|
|
172
|
+
"create",
|
|
173
|
+
"delete",
|
|
174
|
+
"desc",
|
|
175
|
+
"distinct",
|
|
176
|
+
"drop",
|
|
177
|
+
"else",
|
|
178
|
+
"end",
|
|
179
|
+
"exists",
|
|
180
|
+
"from",
|
|
181
|
+
"group",
|
|
182
|
+
"having",
|
|
183
|
+
"in",
|
|
184
|
+
"insert",
|
|
185
|
+
"into",
|
|
186
|
+
"is",
|
|
187
|
+
"join",
|
|
188
|
+
"left",
|
|
189
|
+
"limit",
|
|
190
|
+
"not",
|
|
191
|
+
"null",
|
|
192
|
+
"on",
|
|
193
|
+
"or",
|
|
194
|
+
"order",
|
|
195
|
+
"outer",
|
|
196
|
+
"primary",
|
|
197
|
+
"references",
|
|
198
|
+
"right",
|
|
199
|
+
"select",
|
|
200
|
+
"set",
|
|
201
|
+
"table",
|
|
202
|
+
"then",
|
|
203
|
+
"union",
|
|
204
|
+
"update",
|
|
205
|
+
"values",
|
|
206
|
+
"when",
|
|
207
|
+
"where"
|
|
208
|
+
]);
|
|
209
|
+
const rubyKeywords = new Set(["alias", "and", "begin", "break", "case", "class", "def", "defined?", "do", "else", "elsif", "end", "ensure", "false", "for", "if", "in", "module", "next", "nil", "not", "or", "redo", "rescue", "retry", "return", "self", "super", "then", "true", "undef", "unless", "until", "when", "while", "yield"]);
|
|
210
|
+
const rubyBooleans = new Set(["true", "false"]);
|
|
211
|
+
const rubyNulls = new Set(["nil"]);
|
|
212
|
+
const goKeywords = new Set(["break", "case", "chan", "const", "continue", "default", "defer", "else", "fallthrough", "for", "func", "go", "goto", "if", "import", "interface", "map", "package", "range", "return", "select", "struct", "switch", "type", "var"]);
|
|
213
|
+
const goTypes = new Set(["any", "bool", "byte", "complex128", "complex64", "error", "float32", "float64", "int", "int16", "int32", "int64", "int8", "rune", "string", "uint", "uint16", "uint32", "uint64", "uint8", "uintptr"]);
|
|
214
|
+
const goBooleans = new Set(["true", "false"]);
|
|
215
|
+
const goNulls = new Set(["nil"]);
|
|
216
|
+
const javaKeywords = new Set(["abstract", "assert", "break", "case", "catch", "class", "const", "continue", "default", "do", "else", "enum", "extends", "final", "finally", "for", "if", "implements", "import", "instanceof", "interface", "native", "new", "package", "private", "protected", "public", "return", "static", "strictfp", "super", "switch", "synchronized", "this", "throw", "throws", "transient", "try", "volatile", "while"]);
|
|
217
|
+
const javaTypes = new Set(["Boolean", "Byte", "Character", "Double", "Float", "Integer", "Long", "Object", "Optional", "Short", "String", "Void", "boolean", "byte", "char", "double", "float", "int", "long", "short", "void"]);
|
|
218
|
+
const javaBooleans = new Set(["true", "false"]);
|
|
219
|
+
const javaNulls = new Set(["null"]);
|
|
220
|
+
const cKeywords = new Set(["auto", "break", "case", "const", "continue", "default", "do", "else", "enum", "extern", "for", "goto", "if", "inline", "register", "restrict", "return", "sizeof", "static", "struct", "switch", "typedef", "union", "volatile", "while"]);
|
|
221
|
+
const cTypes = new Set(["bool", "char", "double", "float", "int", "int16_t", "int32_t", "int64_t", "int8_t", "long", "short", "size_t", "uint16_t", "uint32_t", "uint64_t", "uint8_t", "void"]);
|
|
222
|
+
const cppKeywords = new Set([...cKeywords, "alignas", "alignof", "and", "bitand", "bitor", "catch", "class", "concept", "constexpr", "consteval", "constinit", "decltype", "delete", "explicit", "export", "friend", "mutable", "namespace", "new", "noexcept", "not", "operator", "or", "private", "protected", "public", "requires", "template", "this", "throw", "try", "typename", "using", "virtual", "xor"]);
|
|
223
|
+
const cppTypes = new Set([...cTypes, "auto", "bool", "char16_t", "char32_t", "std", "string", "wstring"]);
|
|
224
|
+
const cppBooleans = new Set(["true", "false"]);
|
|
225
|
+
const cppNulls = new Set(["nullptr", "NULL"]);
|
|
226
|
+
const csharpKeywords = new Set(["abstract", "as", "base", "break", "case", "catch", "checked", "class", "const", "continue", "default", "delegate", "do", "else", "enum", "event", "explicit", "extern", "finally", "fixed", "for", "foreach", "if", "implicit", "in", "interface", "internal", "is", "lock", "namespace", "new", "operator", "out", "override", "params", "private", "protected", "public", "readonly", "record", "ref", "return", "sealed", "sizeof", "stackalloc", "static", "struct", "switch", "this", "throw", "try", "typeof", "unchecked", "unsafe", "using", "virtual", "void", "volatile", "while"]);
|
|
227
|
+
const csharpTypes = new Set(["bool", "byte", "char", "decimal", "double", "dynamic", "float", "int", "long", "nint", "nuint", "object", "sbyte", "short", "string", "uint", "ulong", "ushort", "var"]);
|
|
228
|
+
const rustKeywords = new Set(["as", "async", "await", "box", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while"]);
|
|
229
|
+
const rustTypes = new Set(["Box", "Option", "Result", "Some", "String", "Vec", "bool", "char", "f32", "f64", "i128", "i16", "i32", "i64", "i8", "isize", "str", "u128", "u16", "u32", "u64", "u8", "usize"]);
|
|
230
|
+
const rustBooleans = new Set(["true", "false"]);
|
|
231
|
+
const rustNulls = new Set(["None"]);
|
|
232
|
+
const phpKeywords = new Set(["abstract", "and", "array", "as", "break", "callable", "case", "catch", "class", "clone", "const", "continue", "declare", "default", "do", "echo", "else", "elseif", "empty", "enddeclare", "endfor", "endforeach", "endif", "endswitch", "endwhile", "enum", "extends", "final", "finally", "fn", "for", "foreach", "function", "global", "if", "implements", "include", "instanceof", "interface", "isset", "match", "namespace", "new", "or", "private", "protected", "public", "readonly", "require", "return", "static", "switch", "throw", "trait", "try", "use", "var", "while", "xor", "yield"]);
|
|
233
|
+
const phpTypes = new Set(["array", "bool", "callable", "false", "float", "int", "iterable", "mixed", "never", "null", "object", "self", "static", "string", "true", "void"]);
|
|
124
234
|
const lexicalSpecs = {
|
|
125
235
|
javascript: {
|
|
126
236
|
keywords: jsKeywords,
|
|
127
237
|
constants: jsConstants,
|
|
238
|
+
booleans: new Set(["true", "false"]),
|
|
239
|
+
nulls: new Set(["null"]),
|
|
128
240
|
lineComments: ["//"],
|
|
129
241
|
blockComments: true,
|
|
130
242
|
stringQuotes: ['"', "'"],
|
|
@@ -135,18 +247,122 @@ const lexicalSpecs = {
|
|
|
135
247
|
keywords: tsKeywords,
|
|
136
248
|
types: tsTypes,
|
|
137
249
|
constants: jsConstants,
|
|
250
|
+
booleans: new Set(["true", "false"]),
|
|
251
|
+
nulls: new Set(["null"]),
|
|
138
252
|
lineComments: ["//"],
|
|
139
253
|
blockComments: true,
|
|
140
254
|
stringQuotes: ['"', "'"],
|
|
141
255
|
templateQuotes: true,
|
|
142
256
|
decorators: true
|
|
257
|
+
},
|
|
258
|
+
shell: {
|
|
259
|
+
keywords: shellKeywords,
|
|
260
|
+
commands: shellCommands,
|
|
261
|
+
lineComments: ["#"],
|
|
262
|
+
stringQuotes: ['"', "'"],
|
|
263
|
+
variablePrefix: "$",
|
|
264
|
+
flags: true
|
|
265
|
+
},
|
|
266
|
+
python: {
|
|
267
|
+
keywords: pythonKeywords,
|
|
268
|
+
types: pythonTypes,
|
|
269
|
+
booleans: pythonBooleans,
|
|
270
|
+
nulls: pythonNulls,
|
|
271
|
+
lineComments: ["#"],
|
|
272
|
+
stringQuotes: ['"', "'"],
|
|
273
|
+
tripleStringQuotes: true,
|
|
274
|
+
decorators: true
|
|
275
|
+
},
|
|
276
|
+
sql: {
|
|
277
|
+
keywords: sqlKeywords,
|
|
278
|
+
booleans: new Set(["true", "false"]),
|
|
279
|
+
nulls: new Set(["null"]),
|
|
280
|
+
lineComments: ["--"],
|
|
281
|
+
blockComments: true,
|
|
282
|
+
stringQuotes: ['"', "'"],
|
|
283
|
+
caseInsensitive: true
|
|
284
|
+
},
|
|
285
|
+
ruby: {
|
|
286
|
+
keywords: rubyKeywords,
|
|
287
|
+
booleans: rubyBooleans,
|
|
288
|
+
nulls: rubyNulls,
|
|
289
|
+
lineComments: ["#"],
|
|
290
|
+
stringQuotes: ['"', "'"]
|
|
291
|
+
},
|
|
292
|
+
go: {
|
|
293
|
+
keywords: goKeywords,
|
|
294
|
+
types: goTypes,
|
|
295
|
+
booleans: goBooleans,
|
|
296
|
+
nulls: goNulls,
|
|
297
|
+
lineComments: ["//"],
|
|
298
|
+
blockComments: true,
|
|
299
|
+
stringQuotes: ['"', "'", "`"]
|
|
300
|
+
},
|
|
301
|
+
java: {
|
|
302
|
+
keywords: javaKeywords,
|
|
303
|
+
types: javaTypes,
|
|
304
|
+
booleans: javaBooleans,
|
|
305
|
+
nulls: javaNulls,
|
|
306
|
+
lineComments: ["//"],
|
|
307
|
+
blockComments: true,
|
|
308
|
+
stringQuotes: ['"', "'"],
|
|
309
|
+
decorators: true
|
|
310
|
+
},
|
|
311
|
+
c: {
|
|
312
|
+
keywords: cKeywords,
|
|
313
|
+
types: cTypes,
|
|
314
|
+
booleans: cppBooleans,
|
|
315
|
+
nulls: cppNulls,
|
|
316
|
+
lineComments: ["//"],
|
|
317
|
+
blockComments: true,
|
|
318
|
+
stringQuotes: ['"', "'"]
|
|
319
|
+
},
|
|
320
|
+
cpp: {
|
|
321
|
+
keywords: cppKeywords,
|
|
322
|
+
types: cppTypes,
|
|
323
|
+
booleans: cppBooleans,
|
|
324
|
+
nulls: cppNulls,
|
|
325
|
+
lineComments: ["//"],
|
|
326
|
+
blockComments: true,
|
|
327
|
+
stringQuotes: ['"', "'"]
|
|
328
|
+
},
|
|
329
|
+
csharp: {
|
|
330
|
+
keywords: csharpKeywords,
|
|
331
|
+
types: csharpTypes,
|
|
332
|
+
booleans: cppBooleans,
|
|
333
|
+
nulls: javaNulls,
|
|
334
|
+
lineComments: ["//"],
|
|
335
|
+
blockComments: true,
|
|
336
|
+
stringQuotes: ['"', "'"],
|
|
337
|
+
decorators: true
|
|
338
|
+
},
|
|
339
|
+
rust: {
|
|
340
|
+
keywords: rustKeywords,
|
|
341
|
+
types: rustTypes,
|
|
342
|
+
booleans: rustBooleans,
|
|
343
|
+
nulls: rustNulls,
|
|
344
|
+
lineComments: ["//"],
|
|
345
|
+
blockComments: true,
|
|
346
|
+
stringQuotes: ['"', "'"],
|
|
347
|
+
rustAttributes: true
|
|
348
|
+
},
|
|
349
|
+
php: {
|
|
350
|
+
keywords: phpKeywords,
|
|
351
|
+
types: phpTypes,
|
|
352
|
+
booleans: new Set(["true", "false"]),
|
|
353
|
+
nulls: new Set(["null"]),
|
|
354
|
+
lineComments: ["//", "#"],
|
|
355
|
+
blockComments: true,
|
|
356
|
+
stringQuotes: ['"', "'"],
|
|
357
|
+
variablePrefix: "$"
|
|
143
358
|
}
|
|
144
359
|
};
|
|
145
360
|
const tokenizers = {
|
|
146
361
|
lexical: tokenizeLexical,
|
|
147
362
|
data: tokenizeData,
|
|
148
363
|
style: tokenizeStyle,
|
|
149
|
-
line: tokenizeLine
|
|
364
|
+
line: tokenizeLine,
|
|
365
|
+
markup: tokenizeMarkup
|
|
150
366
|
};
|
|
151
367
|
export function highlightCodeBlock(node) {
|
|
152
368
|
if (node.tokens !== undefined) {
|
|
@@ -196,11 +412,33 @@ function tokenizeLexical(source, language) {
|
|
|
196
412
|
index = blockCommentEnd;
|
|
197
413
|
continue;
|
|
198
414
|
}
|
|
415
|
+
if (spec.rustAttributes === true && source.startsWith("#[", index)) {
|
|
416
|
+
index = readUntil(source, index + 2, "]");
|
|
417
|
+
emitter.push("decorator", start, index);
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
199
420
|
if (spec.decorators === true && char === "@" && isIdentifierStart(source[index + 1] ?? "")) {
|
|
200
421
|
index = readIdentifier(source, index + 1);
|
|
201
422
|
emitter.push("decorator", start, index);
|
|
202
423
|
continue;
|
|
203
424
|
}
|
|
425
|
+
if (spec.variablePrefix === "$" && char === "$" && isIdentifierStart(source[index + 1] ?? "")) {
|
|
426
|
+
index = readIdentifier(source, index + 1);
|
|
427
|
+
emitter.push("variable", start, index);
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
if (spec.flags === true && char === "-" && isFlagStart(source[index + 1] ?? "")) {
|
|
431
|
+
index = readFlag(source, index + 1);
|
|
432
|
+
emitter.push("flag", start, index);
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (spec.tripleStringQuotes === true &&
|
|
436
|
+
(char === '"' || char === "'") &&
|
|
437
|
+
source.startsWith(char.repeat(3), index)) {
|
|
438
|
+
index = readTripleQuotedString(source, index, char);
|
|
439
|
+
emitter.push("string", start, index);
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
204
442
|
if ((spec.stringQuotes ?? []).includes(char)) {
|
|
205
443
|
index = readQuotedString(source, index, char);
|
|
206
444
|
emitter.push("string", start, index);
|
|
@@ -227,6 +465,9 @@ function tokenizeLexical(source, language) {
|
|
|
227
465
|
return emitter.tokens;
|
|
228
466
|
}
|
|
229
467
|
function tokenizeData(source, language) {
|
|
468
|
+
if (language.spec === "toml" || language.spec === "ini") {
|
|
469
|
+
return tokenizeConfig(source);
|
|
470
|
+
}
|
|
230
471
|
return language.spec === "yaml" ? tokenizeYaml(source) : tokenizeJsonLike(source, language.spec === "jsonc");
|
|
231
472
|
}
|
|
232
473
|
function tokenizeJsonLike(source, allowComments) {
|
|
@@ -385,8 +626,180 @@ function tokenizeStyle(source) {
|
|
|
385
626
|
}
|
|
386
627
|
return emitter.tokens;
|
|
387
628
|
}
|
|
388
|
-
function
|
|
389
|
-
|
|
629
|
+
function tokenizeConfig(source) {
|
|
630
|
+
const emitter = createEmitter(source);
|
|
631
|
+
let index = 0;
|
|
632
|
+
let atLineStart = true;
|
|
633
|
+
while (index < source.length) {
|
|
634
|
+
const start = index;
|
|
635
|
+
if (source[index] === "\n") {
|
|
636
|
+
emitter.pushPlain(index, index + 1);
|
|
637
|
+
index += 1;
|
|
638
|
+
atLineStart = true;
|
|
639
|
+
continue;
|
|
640
|
+
}
|
|
641
|
+
index = readSpacesAndTabs(source, index);
|
|
642
|
+
if (index > start) {
|
|
643
|
+
emitter.pushPlain(start, index);
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
if (source[index] === "#" || source[index] === ";") {
|
|
647
|
+
index = readUntilLineEnd(source, index);
|
|
648
|
+
emitter.push("comment", start, index);
|
|
649
|
+
atLineStart = false;
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
if (atLineStart && source[index] === "[") {
|
|
653
|
+
index = readUntil(source, index + 1, "]");
|
|
654
|
+
emitter.push("selector", start, index);
|
|
655
|
+
atLineStart = false;
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
if (atLineStart) {
|
|
659
|
+
const keyEnd = readConfigKey(source, index);
|
|
660
|
+
if (keyEnd > index) {
|
|
661
|
+
emitter.push("key", index, keyEnd);
|
|
662
|
+
index = keyEnd;
|
|
663
|
+
atLineStart = false;
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
if (source[index] === '"' || source[index] === "'") {
|
|
668
|
+
const quote = source[index];
|
|
669
|
+
index = readQuotedString(source, index, quote);
|
|
670
|
+
emitter.push("string", start, index);
|
|
671
|
+
atLineStart = false;
|
|
672
|
+
continue;
|
|
673
|
+
}
|
|
674
|
+
index = readNumber(source, index);
|
|
675
|
+
if (index > start) {
|
|
676
|
+
emitter.push("number", start, index);
|
|
677
|
+
atLineStart = false;
|
|
678
|
+
continue;
|
|
679
|
+
}
|
|
680
|
+
index = readIdentifier(source, index);
|
|
681
|
+
if (index > start) {
|
|
682
|
+
emitter.push(classifyDataWord(source.slice(start, index)), start, index);
|
|
683
|
+
atLineStart = false;
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
emitter.pushPlain(start, start + 1);
|
|
687
|
+
index = start + 1;
|
|
688
|
+
atLineStart = false;
|
|
689
|
+
}
|
|
690
|
+
return emitter.tokens;
|
|
691
|
+
}
|
|
692
|
+
function tokenizeLine(source, language) {
|
|
693
|
+
const emitter = createEmitter(source);
|
|
694
|
+
let index = 0;
|
|
695
|
+
let atLineStart = true;
|
|
696
|
+
while (index < source.length) {
|
|
697
|
+
const start = index;
|
|
698
|
+
if (source[index] === "\n") {
|
|
699
|
+
emitter.pushPlain(index, index + 1);
|
|
700
|
+
index += 1;
|
|
701
|
+
atLineStart = true;
|
|
702
|
+
continue;
|
|
703
|
+
}
|
|
704
|
+
if (atLineStart && language.spec === "diff" && (source[index] === "+" || source[index] === "-")) {
|
|
705
|
+
index = readUntilLineEnd(source, index);
|
|
706
|
+
emitter.push(source[start] === "+" ? "string" : "important", start, index);
|
|
707
|
+
atLineStart = false;
|
|
708
|
+
continue;
|
|
709
|
+
}
|
|
710
|
+
if (atLineStart && language.spec === "markdown" && source[index] === "#") {
|
|
711
|
+
index = readUntilLineEnd(source, index);
|
|
712
|
+
emitter.push("keyword", start, index);
|
|
713
|
+
atLineStart = false;
|
|
714
|
+
continue;
|
|
715
|
+
}
|
|
716
|
+
if (atLineStart && language.spec === "dockerfile") {
|
|
717
|
+
const directiveEnd = readIdentifier(source, index);
|
|
718
|
+
if (directiveEnd > index) {
|
|
719
|
+
emitter.push("directive", index, directiveEnd);
|
|
720
|
+
index = directiveEnd;
|
|
721
|
+
atLineStart = false;
|
|
722
|
+
continue;
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
if (source[index] === "#" && language.spec !== "diff") {
|
|
726
|
+
index = readUntilLineEnd(source, index);
|
|
727
|
+
emitter.push("comment", start, index);
|
|
728
|
+
atLineStart = false;
|
|
729
|
+
continue;
|
|
730
|
+
}
|
|
731
|
+
if (source[index] === '"' || source[index] === "'") {
|
|
732
|
+
const quote = source[index];
|
|
733
|
+
index = readQuotedString(source, index, quote);
|
|
734
|
+
emitter.push("string", start, index);
|
|
735
|
+
atLineStart = false;
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
index = readNumber(source, index);
|
|
739
|
+
if (index > start) {
|
|
740
|
+
emitter.push("number", start, index);
|
|
741
|
+
atLineStart = false;
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
744
|
+
emitter.pushPlain(start, start + 1);
|
|
745
|
+
index = start + 1;
|
|
746
|
+
atLineStart = false;
|
|
747
|
+
}
|
|
748
|
+
return emitter.tokens;
|
|
749
|
+
}
|
|
750
|
+
function tokenizeMarkup(source) {
|
|
751
|
+
const emitter = createEmitter(source);
|
|
752
|
+
let index = 0;
|
|
753
|
+
while (index < source.length) {
|
|
754
|
+
const start = index;
|
|
755
|
+
if (source.startsWith("<!--", index)) {
|
|
756
|
+
index = readUntil(source, index + 4, "-->");
|
|
757
|
+
emitter.push("comment", start, index);
|
|
758
|
+
continue;
|
|
759
|
+
}
|
|
760
|
+
if (source[index] === "<") {
|
|
761
|
+
emitter.push("punctuation", index, index + 1);
|
|
762
|
+
index += 1;
|
|
763
|
+
if (source[index] === "/") {
|
|
764
|
+
emitter.push("punctuation", index, index + 1);
|
|
765
|
+
index += 1;
|
|
766
|
+
}
|
|
767
|
+
const tagStart = index;
|
|
768
|
+
index = readIdentifier(source, index);
|
|
769
|
+
if (index > tagStart) {
|
|
770
|
+
emitter.push("tag", tagStart, index);
|
|
771
|
+
}
|
|
772
|
+
while (index < source.length && source[index] !== ">") {
|
|
773
|
+
const innerStart = index;
|
|
774
|
+
index = readWhitespace(source, index);
|
|
775
|
+
if (index > innerStart) {
|
|
776
|
+
emitter.pushPlain(innerStart, index);
|
|
777
|
+
continue;
|
|
778
|
+
}
|
|
779
|
+
if (source[index] === '"' || source[index] === "'") {
|
|
780
|
+
const quote = source[index];
|
|
781
|
+
index = readQuotedString(source, index, quote);
|
|
782
|
+
emitter.push("string", innerStart, index);
|
|
783
|
+
continue;
|
|
784
|
+
}
|
|
785
|
+
index = readIdentifier(source, index);
|
|
786
|
+
if (index > innerStart) {
|
|
787
|
+
emitter.push("attribute", innerStart, index);
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
790
|
+
emitter.push("punctuation", innerStart, innerStart + 1);
|
|
791
|
+
index = innerStart + 1;
|
|
792
|
+
}
|
|
793
|
+
if (source[index] === ">") {
|
|
794
|
+
emitter.push("punctuation", index, index + 1);
|
|
795
|
+
index += 1;
|
|
796
|
+
}
|
|
797
|
+
continue;
|
|
798
|
+
}
|
|
799
|
+
emitter.pushPlain(start, start + 1);
|
|
800
|
+
index = start + 1;
|
|
801
|
+
}
|
|
802
|
+
return emitter.tokens;
|
|
390
803
|
}
|
|
391
804
|
function createEmitter(source) {
|
|
392
805
|
const tokens = [];
|
|
@@ -413,19 +826,23 @@ function pushToken(tokens, source, kind, start, end) {
|
|
|
413
826
|
tokens.push({ kind, value });
|
|
414
827
|
}
|
|
415
828
|
function classifyLexicalWord(word, spec) {
|
|
416
|
-
|
|
829
|
+
const lookup = spec.caseInsensitive === true ? word.toLowerCase() : word;
|
|
830
|
+
if (spec.booleans?.has(lookup) === true || spec.booleans?.has(word) === true) {
|
|
417
831
|
return "boolean";
|
|
418
832
|
}
|
|
419
|
-
if (word ===
|
|
833
|
+
if (spec.nulls?.has(lookup) === true || spec.nulls?.has(word) === true) {
|
|
420
834
|
return "null";
|
|
421
835
|
}
|
|
422
|
-
if (spec.keywords?.has(word) === true) {
|
|
836
|
+
if (spec.keywords?.has(lookup) === true || spec.keywords?.has(word) === true) {
|
|
423
837
|
return "keyword";
|
|
424
838
|
}
|
|
425
|
-
if (spec.types?.has(word) === true) {
|
|
839
|
+
if (spec.types?.has(lookup) === true || spec.types?.has(word) === true) {
|
|
426
840
|
return "type";
|
|
427
841
|
}
|
|
428
|
-
if (spec.
|
|
842
|
+
if (spec.commands?.has(lookup) === true || spec.commands?.has(word) === true) {
|
|
843
|
+
return "command";
|
|
844
|
+
}
|
|
845
|
+
if (spec.constants?.has(lookup) === true || spec.constants?.has(word) === true) {
|
|
429
846
|
return "number";
|
|
430
847
|
}
|
|
431
848
|
return "plain";
|
|
@@ -527,6 +944,21 @@ function readQuotedString(source, index, quote) {
|
|
|
527
944
|
}
|
|
528
945
|
return index;
|
|
529
946
|
}
|
|
947
|
+
function readTripleQuotedString(source, index, quote) {
|
|
948
|
+
const marker = quote.repeat(3);
|
|
949
|
+
index += marker.length;
|
|
950
|
+
while (index < source.length) {
|
|
951
|
+
if (source.startsWith(marker, index)) {
|
|
952
|
+
return index + marker.length;
|
|
953
|
+
}
|
|
954
|
+
if (source[index] === "\\") {
|
|
955
|
+
index = Math.min(source.length, index + 2);
|
|
956
|
+
continue;
|
|
957
|
+
}
|
|
958
|
+
index += 1;
|
|
959
|
+
}
|
|
960
|
+
return index;
|
|
961
|
+
}
|
|
530
962
|
function readAnyLineComment(source, index, markers) {
|
|
531
963
|
for (const marker of markers) {
|
|
532
964
|
if (source.startsWith(marker, index)) {
|
|
@@ -554,6 +986,15 @@ function readUntilLineEnd(source, index) {
|
|
|
554
986
|
}
|
|
555
987
|
return index;
|
|
556
988
|
}
|
|
989
|
+
function readUntil(source, index, marker) {
|
|
990
|
+
while (index < source.length) {
|
|
991
|
+
if (source.startsWith(marker, index)) {
|
|
992
|
+
return index + marker.length;
|
|
993
|
+
}
|
|
994
|
+
index += 1;
|
|
995
|
+
}
|
|
996
|
+
return index;
|
|
997
|
+
}
|
|
557
998
|
function readYamlKey(source, index) {
|
|
558
999
|
const start = index;
|
|
559
1000
|
while (index < source.length) {
|
|
@@ -568,6 +1009,32 @@ function readYamlKey(source, index) {
|
|
|
568
1009
|
}
|
|
569
1010
|
return start;
|
|
570
1011
|
}
|
|
1012
|
+
function readConfigKey(source, index) {
|
|
1013
|
+
const start = index;
|
|
1014
|
+
while (index < source.length) {
|
|
1015
|
+
const char = source[index];
|
|
1016
|
+
if (char === "=" || char === ":") {
|
|
1017
|
+
return trimRightIndex(source, start, index);
|
|
1018
|
+
}
|
|
1019
|
+
if (char === "\n" || char === "#" || char === ";") {
|
|
1020
|
+
return start;
|
|
1021
|
+
}
|
|
1022
|
+
index += 1;
|
|
1023
|
+
}
|
|
1024
|
+
return start;
|
|
1025
|
+
}
|
|
1026
|
+
function readFlag(source, index) {
|
|
1027
|
+
while (index < source.length && isFlagPart(source[index])) {
|
|
1028
|
+
index += 1;
|
|
1029
|
+
}
|
|
1030
|
+
return index;
|
|
1031
|
+
}
|
|
1032
|
+
function trimRightIndex(source, start, end) {
|
|
1033
|
+
while (end > start && (source[end - 1] === " " || source[end - 1] === "\t")) {
|
|
1034
|
+
end -= 1;
|
|
1035
|
+
}
|
|
1036
|
+
return end;
|
|
1037
|
+
}
|
|
571
1038
|
function readCssColor(source, index) {
|
|
572
1039
|
let count = 0;
|
|
573
1040
|
while (index < source.length && isHex(source[index]) && count < 8) {
|
|
@@ -599,6 +1066,12 @@ function isCssNameStart(char) {
|
|
|
599
1066
|
function isCssNamePart(char) {
|
|
600
1067
|
return isCssNameStart(char) || isDigit(char);
|
|
601
1068
|
}
|
|
1069
|
+
function isFlagStart(char) {
|
|
1070
|
+
return isAlpha(char) || char === "-";
|
|
1071
|
+
}
|
|
1072
|
+
function isFlagPart(char) {
|
|
1073
|
+
return isAlpha(char) || isDigit(char) || char === "-";
|
|
1074
|
+
}
|
|
602
1075
|
function isExponentStart(char) {
|
|
603
1076
|
return isDigit(char) || char === "+" || char === "-";
|
|
604
1077
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "toolcraft",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.82",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
"postpack": "node ../../scripts/manage-bundled-workspace-deps.mjs cleanup . toolcraft-design @poe-code/frontmatter @poe-code/agent-mcp-config @poe-code/agent-human-in-loop @poe-code/task-list @poe-code/agent-defs @poe-code/config-mutations @poe-code/process-runner tiny-mcp-client mcp-oauth auth-store"
|
|
52
52
|
},
|
|
53
53
|
"dependencies": {
|
|
54
|
-
"toolcraft-schema": "0.0.
|
|
54
|
+
"toolcraft-schema": "0.0.82",
|
|
55
55
|
"commander": "^13.1.0",
|
|
56
56
|
"fast-string-width": "^3.0.2",
|
|
57
57
|
"fast-wrap-ansi": "^0.2.0",
|