sugar-high 1.2.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -42
- package/lib/core.d.ts +117 -0
- package/lib/core.js +202 -0
- package/lib/index.d.ts +39 -22
- package/lib/index.js +15 -982
- package/lib/lang.d.ts +14 -0
- package/lib/lang.js +132 -0
- package/lib/presets/lang/cpp.js +22 -0
- package/lib/presets/lang/csharp.js +21 -0
- package/lib/presets/lang/css.js +5 -0
- package/lib/presets/lang/diff.js +12 -6
- package/lib/presets/lang/dockerfile.js +5 -0
- package/lib/presets/lang/graphql.js +5 -0
- package/lib/presets/lang/hash-comment-base.js +7 -0
- package/lib/presets/lang/hcl.js +4 -0
- package/lib/presets/lang/html.js +14 -0
- package/lib/presets/lang/javascript-runtime.js +781 -0
- package/lib/presets/lang/javascript.js +6 -0
- package/lib/presets/lang/json.js +8 -0
- package/lib/presets/lang/kotlin.js +5 -0
- package/lib/presets/lang/markdown.js +15 -0
- package/lib/presets/lang/php.js +5 -0
- package/lib/presets/lang/plain-base.js +4 -0
- package/lib/presets/lang/powershell.js +5 -0
- package/lib/presets/lang/shell.js +10 -0
- package/lib/presets/lang/sql.js +31 -0
- package/lib/presets/lang/swift.js +5 -0
- package/lib/presets/lang/toml.js +5 -0
- package/lib/presets/lang/typescript.js +7 -0
- package/lib/presets/lang/yaml.js +10 -0
- package/lib/shared.js +149 -0
- package/package.json +15 -10
- package/lib/presets/index.d.ts +0 -16
- package/lib/presets/index.js +0 -7
|
@@ -0,0 +1,781 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
T_BREAK, T_CLASS as T_CLS_NUMBER, T_COMMENT, T_ENTITY, T_IDENTIFIER,
|
|
5
|
+
T_JSX_LITERALS, T_KEYWORD, T_PROPERTY, T_SIGN, T_SPACE, T_STRING,
|
|
6
|
+
} from '../../shared.js'
|
|
7
|
+
|
|
8
|
+
const JSXBrackets = new Set(['<', '>', '{', '}', '[', ']'])
|
|
9
|
+
const Keywords_Js = new Set([
|
|
10
|
+
'for', 'do', 'while', 'if', 'else', 'return', 'function', 'var', 'let', 'const',
|
|
11
|
+
'true', 'false', 'undefined', 'this', 'new', 'delete', 'typeof', 'in', 'instanceof',
|
|
12
|
+
'void', 'break', 'continue', 'switch', 'case', 'default', 'throw', 'try', 'catch',
|
|
13
|
+
'finally', 'debugger', 'with', 'yield', 'async', 'await', 'class', 'extends', 'super',
|
|
14
|
+
'import', 'export', 'from', 'static',
|
|
15
|
+
])
|
|
16
|
+
|
|
17
|
+
const Keywords_Ts = new Set([
|
|
18
|
+
...Keywords_Js,
|
|
19
|
+
'type', 'interface', 'enum', 'implements', 'readonly', 'abstract', 'declare',
|
|
20
|
+
'namespace', 'module', 'private', 'protected', 'public', 'override', 'keyof', 'infer',
|
|
21
|
+
'is', 'asserts', 'satisfies', 'as', 'unknown', 'never', 'any', 'number', 'string',
|
|
22
|
+
'boolean', 'bigint', 'symbol', 'object',
|
|
23
|
+
])
|
|
24
|
+
|
|
25
|
+
const Signs = new Set([
|
|
26
|
+
'+',
|
|
27
|
+
'-',
|
|
28
|
+
'*',
|
|
29
|
+
'/',
|
|
30
|
+
'%',
|
|
31
|
+
'=',
|
|
32
|
+
'!',
|
|
33
|
+
'&',
|
|
34
|
+
'|',
|
|
35
|
+
'^',
|
|
36
|
+
'~',
|
|
37
|
+
'!',
|
|
38
|
+
'?',
|
|
39
|
+
':',
|
|
40
|
+
'.',
|
|
41
|
+
',',
|
|
42
|
+
';',
|
|
43
|
+
`'`,
|
|
44
|
+
'"',
|
|
45
|
+
'.',
|
|
46
|
+
'(',
|
|
47
|
+
')',
|
|
48
|
+
'[',
|
|
49
|
+
']',
|
|
50
|
+
'#',
|
|
51
|
+
'@',
|
|
52
|
+
'\\',
|
|
53
|
+
...JSXBrackets,
|
|
54
|
+
])
|
|
55
|
+
|
|
56
|
+
const DefaultOptions = {
|
|
57
|
+
keywords: Keywords_Js,
|
|
58
|
+
onCommentStart: isCommentStart_Js,
|
|
59
|
+
onCommentEnd: isCommentEnd_Js,
|
|
60
|
+
jsx: true,
|
|
61
|
+
regex: true,
|
|
62
|
+
templateStrings: true,
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Apply caller overrides to the JavaScript-compatible defaults.
|
|
67
|
+
* @param {HighlightOptions | undefined} options
|
|
68
|
+
* @returns {HighlightOptions}
|
|
69
|
+
*/
|
|
70
|
+
function resolveHighlightOptions(options) {
|
|
71
|
+
return {
|
|
72
|
+
...DefaultOptions,
|
|
73
|
+
...options,
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Fast, heuristic TS detection. It intentionally prefers speed over full parsing.
|
|
79
|
+
* @param {string} code
|
|
80
|
+
* @returns {boolean}
|
|
81
|
+
*/
|
|
82
|
+
function isLikelyTypeScript(code) {
|
|
83
|
+
let tsScore = 0
|
|
84
|
+
|
|
85
|
+
// TS-only declarations and operators.
|
|
86
|
+
if (/\binterface\s+[A-Za-z_$][\w$]*/.test(code)) tsScore += 2
|
|
87
|
+
if (/\btype\s+[A-Za-z_$][\w$]*\s*=/.test(code)) tsScore += 2
|
|
88
|
+
if (/\benum\s+[A-Za-z_$][\w$]*/.test(code)) tsScore += 2
|
|
89
|
+
if (/\b(?:implements|readonly|declare|namespace|satisfies|infer|keyof|asserts)\b/.test(code)) tsScore += 2
|
|
90
|
+
|
|
91
|
+
// Common TS annotations/signatures.
|
|
92
|
+
if (/:\s*[A-Za-z_$][\w$]*(?:<[^>\n]+>)?(?:\[\])?(?=\s*[,)=;{])/m.test(code)) tsScore += 1
|
|
93
|
+
if (/\b(?:const|let|var)\s+[A-Za-z_$][\w$]*\s*:\s*/.test(code)) tsScore += 1
|
|
94
|
+
if (/\)\s*:\s*[A-Za-z_$][\w$]*(?:<[^>\n]+>)?(?:\[\])?\s*(?:=>|\{)/.test(code)) tsScore += 1
|
|
95
|
+
|
|
96
|
+
return tsScore >= 2
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Detects `<T, U = ...>(` style generic parameter lists so they are not
|
|
101
|
+
* treated as JSX tags.
|
|
102
|
+
* @param {string} code
|
|
103
|
+
* @param {number} startIndex
|
|
104
|
+
* @returns {boolean}
|
|
105
|
+
*/
|
|
106
|
+
function isTypeParameterListStart(code, startIndex) {
|
|
107
|
+
if (code[startIndex] !== '<') return false
|
|
108
|
+
|
|
109
|
+
let depth = 0
|
|
110
|
+
let sawIdentifierStart = false
|
|
111
|
+
|
|
112
|
+
for (let i = startIndex; i < code.length; i++) {
|
|
113
|
+
const ch = code[i]
|
|
114
|
+
|
|
115
|
+
if (ch === '<') {
|
|
116
|
+
depth++
|
|
117
|
+
continue
|
|
118
|
+
}
|
|
119
|
+
if (ch === '>') {
|
|
120
|
+
depth--
|
|
121
|
+
if (depth === 0) {
|
|
122
|
+
let next = i + 1
|
|
123
|
+
while (next < code.length && /\s/.test(code[next])) next++
|
|
124
|
+
if (!(sawIdentifierStart && code[next] === '(')) return false
|
|
125
|
+
|
|
126
|
+
// Focus this heuristic on generic arrow functions:
|
|
127
|
+
// const fn = <T>(arg) => ...
|
|
128
|
+
const tail = code.slice(next, next + 320)
|
|
129
|
+
return /\)\s*(?::[\s\S]{0,120}?)?=>/.test(tail)
|
|
130
|
+
}
|
|
131
|
+
continue
|
|
132
|
+
}
|
|
133
|
+
if (depth === 0) continue
|
|
134
|
+
|
|
135
|
+
if (/[$A-Za-z_]/.test(ch)) {
|
|
136
|
+
sawIdentifierStart = true
|
|
137
|
+
continue
|
|
138
|
+
}
|
|
139
|
+
if (/[\s,\.\=\?\:\|\&\[\]]/.test(ch)) continue
|
|
140
|
+
|
|
141
|
+
return false
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
return false
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
*
|
|
149
|
+
* 0 - identifier
|
|
150
|
+
* 1 - keyword
|
|
151
|
+
* 2 - string
|
|
152
|
+
* 3 - Class, number and null
|
|
153
|
+
* 4 - property
|
|
154
|
+
* 5 - entity
|
|
155
|
+
* 6 - jsx literals
|
|
156
|
+
* 7 - sign
|
|
157
|
+
* 8 - comment
|
|
158
|
+
* 9 - break
|
|
159
|
+
* 10 - space
|
|
160
|
+
*
|
|
161
|
+
*/
|
|
162
|
+
function isSpaces(str) {
|
|
163
|
+
return /^[^\S\r\n]+$/g.test(str)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function isSign(ch) {
|
|
167
|
+
return Signs.has(ch)
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function isWord(chr) {
|
|
171
|
+
return /^[\w_]+$/.test(chr) || hasUnicode(chr)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function isCls(str) {
|
|
175
|
+
const chr0 = str[0]
|
|
176
|
+
return isWord(chr0) &&
|
|
177
|
+
chr0 === chr0.toUpperCase() ||
|
|
178
|
+
str === 'null'
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function hasUnicode(s) {
|
|
182
|
+
return /[^\u0000-\u007f]/.test(s);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function isAlpha(chr) {
|
|
186
|
+
return /^[a-zA-Z]$/.test(chr)
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function isIdentifierChar(chr) {
|
|
190
|
+
return isAlpha(chr) || hasUnicode(chr)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function isIdentifier(str) {
|
|
194
|
+
return isIdentifierChar(str[0]) && (str.length === 1 || isWord(str.slice(1)))
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function isStrTemplateChr(chr) {
|
|
198
|
+
return chr === '`'
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function isSingleQuotes(chr) {
|
|
202
|
+
return chr === '"' || chr === "'"
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function isStringQuotation(chr) {
|
|
206
|
+
return isSingleQuotes(chr) || isStrTemplateChr(chr)
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** @returns {0|1|2} */
|
|
210
|
+
function isCommentStart_Js(curr, next) {
|
|
211
|
+
const str = curr + next
|
|
212
|
+
if (str === '/*') return 2
|
|
213
|
+
return str === '//' ? 1 : 0
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** @returns {0|1|2} */
|
|
217
|
+
function isCommentEnd_Js(prev, curr) {
|
|
218
|
+
return (prev + curr) === '*/'
|
|
219
|
+
? 2
|
|
220
|
+
: curr === '\n' ? 1 : 0
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function isRegexStart(str) {
|
|
224
|
+
return str[0] === '/' && !isCommentStart_Js(str[0], str[1])
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function isPropertyKey(code, quoteEnd) {
|
|
228
|
+
let i = quoteEnd + 1
|
|
229
|
+
while (i < code.length && /\s/.test(code[i])) i++
|
|
230
|
+
return code[i] === ':'
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* @param {string} code
|
|
235
|
+
* @param {HighlightOptions | undefined} options
|
|
236
|
+
* Optional `onQuote(curr, i, code)` at `code[i] === "'"`: return length to consume from `i` (>= 1),
|
|
237
|
+
* or null/undefined/below 1 for default JS single-quoted strings. No substring allocation.
|
|
238
|
+
* @return {Array<[number, string]>}
|
|
239
|
+
*/
|
|
240
|
+
function tokenize(code, options) {
|
|
241
|
+
const mergedOptions = resolveHighlightOptions(options)
|
|
242
|
+
const hasCustomKeywords = mergedOptions.keywords !== DefaultOptions.keywords
|
|
243
|
+
const isTs = typeof mergedOptions.typescript === 'boolean'
|
|
244
|
+
? mergedOptions.typescript
|
|
245
|
+
: isLikelyTypeScript(code)
|
|
246
|
+
const resolvedKeywords = hasCustomKeywords
|
|
247
|
+
? mergedOptions.keywords
|
|
248
|
+
: (isTs ? Keywords_Ts : Keywords_Js)
|
|
249
|
+
|
|
250
|
+
const {
|
|
251
|
+
onCommentStart,
|
|
252
|
+
onCommentEnd,
|
|
253
|
+
} = mergedOptions
|
|
254
|
+
|
|
255
|
+
const resolvedTypeKeywords =
|
|
256
|
+
mergedOptions.typeKeywords instanceof Set ? mergedOptions.typeKeywords : null
|
|
257
|
+
const supportsJsx = mergedOptions.jsx !== false
|
|
258
|
+
const supportsRegex = mergedOptions.regex !== false
|
|
259
|
+
const supportsTemplateStrings = mergedOptions.templateStrings !== false
|
|
260
|
+
const normalizeKeyword = mergedOptions.caseInsensitive
|
|
261
|
+
? (token) => token.toLowerCase()
|
|
262
|
+
: (token) => token
|
|
263
|
+
const isTemplateQuote = (chr) => supportsTemplateStrings && isStrTemplateChr(chr)
|
|
264
|
+
|
|
265
|
+
let current = ''
|
|
266
|
+
let type = -1
|
|
267
|
+
/** @type {[number, string]} */
|
|
268
|
+
let last = [-1, '']
|
|
269
|
+
/** @type {[number, string]} */
|
|
270
|
+
let beforeLast = [-2, '']
|
|
271
|
+
/** @type {Array<[number, string]>} */
|
|
272
|
+
const tokens = []
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* TS generics (`Map<string>`) and JSX (`<div>`) share the same `<Name …>` lexical shape. We use
|
|
276
|
+
* one tag-lexer mode (__jsxTag + __jsxStack) for both when we enter it; isTsTypeArgStart and
|
|
277
|
+
* isTypeParameterListStart only decide when *not* to enter (e.g. `foo<T>`, `<T>(x)=>`).
|
|
278
|
+
* __jsxEnter gates that mode so a latched __jsxTag (from `<` in `"a<b"`) does not run the tag
|
|
279
|
+
* lexer until we are in real JSX/TSX surface (not inside strings).
|
|
280
|
+
* @type {boolean}
|
|
281
|
+
*/
|
|
282
|
+
let __jsxEnter = false
|
|
283
|
+
/** @type {0 | 1 | 2} 0 = none; 1 = inside `<open`; 2 = inside `</close` */
|
|
284
|
+
let __jsxTag = 0
|
|
285
|
+
let __jsxExpr = false
|
|
286
|
+
|
|
287
|
+
/** Nested `<open>…</open>` depth (content between tags, including nested elements). */
|
|
288
|
+
let __jsxStack = 0
|
|
289
|
+
|
|
290
|
+
const __jsxChild = () => __jsxEnter && !__jsxExpr && !__jsxTag
|
|
291
|
+
// < __content__ >
|
|
292
|
+
const inJsxTag = () => __jsxTag && !__jsxChild()
|
|
293
|
+
// {'__content__'}
|
|
294
|
+
const inJsxLiterals = () => !__jsxTag && __jsxChild() && !__jsxExpr && __jsxStack > 0
|
|
295
|
+
|
|
296
|
+
/** @type {string | null} */
|
|
297
|
+
let __strQuote = null
|
|
298
|
+
let __strTokenStart = 0
|
|
299
|
+
let __regexQuoteStart = false
|
|
300
|
+
let __strTemplateExprStack = 0
|
|
301
|
+
let __strTemplateQuoteStack = 0
|
|
302
|
+
const inStringQuotes = () => __strQuote !== null
|
|
303
|
+
const inRegexQuotes = () => __regexQuoteStart
|
|
304
|
+
const inStrTemplateLiterals = () => (__strTemplateQuoteStack > __strTemplateExprStack)
|
|
305
|
+
const inStrTemplateExpr = () => __strTemplateQuoteStack > 0 && (__strTemplateQuoteStack === __strTemplateExprStack)
|
|
306
|
+
const inStringContent = () => inStringQuotes() || inStrTemplateLiterals()
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
*
|
|
310
|
+
* @param {string} token
|
|
311
|
+
* @returns {number}
|
|
312
|
+
*/
|
|
313
|
+
function classify(token) {
|
|
314
|
+
const isLineBreak = token === '\n'
|
|
315
|
+
// First checking if they're attributes values
|
|
316
|
+
if (inJsxTag()) {
|
|
317
|
+
if (inStringQuotes()) {
|
|
318
|
+
return T_STRING
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const [, lastToken] = last
|
|
322
|
+
if (isIdentifier(token)) {
|
|
323
|
+
// classify jsx open tag
|
|
324
|
+
if ((lastToken === '<' || lastToken === '</'))
|
|
325
|
+
return T_ENTITY
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// Then determine if they're jsx literals
|
|
329
|
+
const isJsxLiterals = inJsxLiterals()
|
|
330
|
+
if (isJsxLiterals) return T_JSX_LITERALS
|
|
331
|
+
|
|
332
|
+
// Determine strings first before other types
|
|
333
|
+
if (inStringQuotes() || inStrTemplateLiterals()) {
|
|
334
|
+
return T_STRING
|
|
335
|
+
} else if (resolvedTypeKeywords && resolvedTypeKeywords.has(normalizeKeyword(token))) {
|
|
336
|
+
return last[1] === '.' ? T_IDENTIFIER : T_CLS_NUMBER
|
|
337
|
+
} else if (resolvedKeywords.has(normalizeKeyword(token))) {
|
|
338
|
+
return last[1] === '.' ? T_IDENTIFIER : T_KEYWORD
|
|
339
|
+
} else if (isLineBreak) {
|
|
340
|
+
return T_BREAK
|
|
341
|
+
} else if (isSpaces(token)) {
|
|
342
|
+
return T_SPACE
|
|
343
|
+
} else if (token.split('').every(isSign)) {
|
|
344
|
+
return T_SIGN
|
|
345
|
+
} else if (isCls(token)) {
|
|
346
|
+
return inJsxTag() ? T_IDENTIFIER : T_CLS_NUMBER
|
|
347
|
+
} else {
|
|
348
|
+
if (isIdentifier(token)) {
|
|
349
|
+
const isLastPropDot = last[1] === '.' && isIdentifier(beforeLast[1])
|
|
350
|
+
|
|
351
|
+
if (!inStringContent() && !isLastPropDot) return T_IDENTIFIER
|
|
352
|
+
if (isLastPropDot) return T_PROPERTY
|
|
353
|
+
}
|
|
354
|
+
return T_STRING
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
*
|
|
360
|
+
* @param {number | undefined} [type_]
|
|
361
|
+
* @param {string | undefined} [token_]
|
|
362
|
+
*/
|
|
363
|
+
const append = (type_, token_) => {
|
|
364
|
+
if (token_) {
|
|
365
|
+
current = token_
|
|
366
|
+
}
|
|
367
|
+
if (current) {
|
|
368
|
+
type = typeof type_ === 'number' ? type_ : classify(current)
|
|
369
|
+
/** @type [number, string] */
|
|
370
|
+
const pair = [type, current]
|
|
371
|
+
if (type !== T_SPACE && type !== T_BREAK) {
|
|
372
|
+
beforeLast = last
|
|
373
|
+
last = pair
|
|
374
|
+
}
|
|
375
|
+
tokens.push(pair)
|
|
376
|
+
}
|
|
377
|
+
current = ''
|
|
378
|
+
}
|
|
379
|
+
for (let i = 0; i < code.length; i++) {
|
|
380
|
+
const curr = code[i]
|
|
381
|
+
const prev = code[i - 1]
|
|
382
|
+
const next = code[i + 1]
|
|
383
|
+
const p_c = prev + curr // previous and current
|
|
384
|
+
const c_n = curr + next // current and next
|
|
385
|
+
|
|
386
|
+
// onQuote(curr, i, code): length from i; end = i + len (capped).
|
|
387
|
+
if (
|
|
388
|
+
typeof mergedOptions.onQuote === 'function' &&
|
|
389
|
+
curr === "'" &&
|
|
390
|
+
!inStringQuotes() &&
|
|
391
|
+
!inJsxLiterals() &&
|
|
392
|
+
!inStrTemplateLiterals()
|
|
393
|
+
) {
|
|
394
|
+
const rawLen = mergedOptions.onQuote(curr, i, code)
|
|
395
|
+
if (
|
|
396
|
+
typeof rawLen === 'number' &&
|
|
397
|
+
rawLen >= 1 &&
|
|
398
|
+
!Number.isNaN(rawLen)
|
|
399
|
+
) {
|
|
400
|
+
const len = Math.min(rawLen, code.length - i)
|
|
401
|
+
const end = i + len
|
|
402
|
+
append()
|
|
403
|
+
current = code.slice(i, end)
|
|
404
|
+
append(T_IDENTIFIER)
|
|
405
|
+
i = end - 1
|
|
406
|
+
continue
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// Determine string quotation outside of jsx literals and template literals.
|
|
411
|
+
// Inside jsx literals or template literals, string quotation is still part of it.
|
|
412
|
+
if (isSingleQuotes(curr) && !inJsxLiterals() && !inStrTemplateLiterals()) {
|
|
413
|
+
append()
|
|
414
|
+
let isStringClose = false
|
|
415
|
+
if (prev !== `\\`) {
|
|
416
|
+
if (__strQuote && curr === __strQuote) {
|
|
417
|
+
__strQuote = null
|
|
418
|
+
isStringClose = true
|
|
419
|
+
} else if (!__strQuote) {
|
|
420
|
+
__strQuote = curr
|
|
421
|
+
__strTokenStart = tokens.length
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
append(T_STRING, curr)
|
|
426
|
+
if (mergedOptions.quotedKeys && isStringClose && isPropertyKey(code, i)) {
|
|
427
|
+
for (let tokenIndex = __strTokenStart; tokenIndex < tokens.length; tokenIndex++) {
|
|
428
|
+
tokens[tokenIndex][0] = T_PROPERTY
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
continue
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
if (!inStrTemplateLiterals()) {
|
|
435
|
+
if (prev !== '\\n' && isTemplateQuote(curr)) {
|
|
436
|
+
append()
|
|
437
|
+
append(T_STRING, curr)
|
|
438
|
+
__strTemplateQuoteStack++
|
|
439
|
+
continue
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
if (inStrTemplateLiterals()) {
|
|
444
|
+
if (prev !== '\\n' && isTemplateQuote(curr)) {
|
|
445
|
+
if (__strTemplateQuoteStack > 0) {
|
|
446
|
+
append()
|
|
447
|
+
__strTemplateQuoteStack--
|
|
448
|
+
append(T_STRING, curr)
|
|
449
|
+
continue
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
if (c_n === '${') {
|
|
454
|
+
__strTemplateExprStack++
|
|
455
|
+
append(T_STRING)
|
|
456
|
+
append(T_SIGN, c_n)
|
|
457
|
+
i++
|
|
458
|
+
continue
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
if (inStrTemplateExpr() && curr === '}') {
|
|
463
|
+
append()
|
|
464
|
+
__strTemplateExprStack--
|
|
465
|
+
append(T_SIGN, curr)
|
|
466
|
+
continue
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (__jsxChild()) {
|
|
470
|
+
if (curr === '{') {
|
|
471
|
+
append()
|
|
472
|
+
append(T_SIGN, curr)
|
|
473
|
+
__jsxExpr = true
|
|
474
|
+
continue
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
if (__jsxEnter) {
|
|
479
|
+
// <: open tag sign
|
|
480
|
+
// new '<' not inside jsx
|
|
481
|
+
if (!__jsxTag && curr === '<') {
|
|
482
|
+
append()
|
|
483
|
+
if (next === '/') {
|
|
484
|
+
// close tag
|
|
485
|
+
__jsxTag = 2
|
|
486
|
+
current = c_n
|
|
487
|
+
i++
|
|
488
|
+
} else {
|
|
489
|
+
// open tag
|
|
490
|
+
__jsxTag = 1
|
|
491
|
+
current = curr
|
|
492
|
+
}
|
|
493
|
+
append(T_SIGN)
|
|
494
|
+
continue
|
|
495
|
+
}
|
|
496
|
+
if (__jsxTag) {
|
|
497
|
+
// >: open tag close sign or closing tag closing sign
|
|
498
|
+
// and it's not `=>` or `/>`
|
|
499
|
+
// `curr` could be `>` or `/`
|
|
500
|
+
if ((curr === '>' && !'/='.includes(prev))) {
|
|
501
|
+
append()
|
|
502
|
+
if (__jsxTag === 1) {
|
|
503
|
+
__jsxTag = 0
|
|
504
|
+
__jsxStack++
|
|
505
|
+
} else {
|
|
506
|
+
__jsxTag = 0
|
|
507
|
+
__jsxEnter = false
|
|
508
|
+
}
|
|
509
|
+
append(T_SIGN, curr)
|
|
510
|
+
continue
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// >: tag self close sign or close tag sign
|
|
514
|
+
if (c_n === '/>' || c_n === '</') {
|
|
515
|
+
// if current token is not part of close tag sign, push it first
|
|
516
|
+
if (current !== '<' && current !== '/') {
|
|
517
|
+
append()
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
if (c_n === '/>') {
|
|
521
|
+
__jsxTag = 0
|
|
522
|
+
} else {
|
|
523
|
+
// is '</'
|
|
524
|
+
__jsxStack--
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
if (!__jsxStack)
|
|
528
|
+
__jsxEnter = false
|
|
529
|
+
|
|
530
|
+
current = c_n
|
|
531
|
+
i++
|
|
532
|
+
append(T_SIGN)
|
|
533
|
+
continue
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// <: open tag sign
|
|
537
|
+
if (curr === '<') {
|
|
538
|
+
append()
|
|
539
|
+
current = curr
|
|
540
|
+
append(T_SIGN)
|
|
541
|
+
continue
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// jsx property
|
|
545
|
+
// `-` in data-prop / aria-label. Consume the complete attribute as one property.
|
|
546
|
+
if (curr === '-' && current && !inStringContent() && !inJsxLiterals()) {
|
|
547
|
+
let end = i + 1
|
|
548
|
+
while (end < code.length && /[$\w-]/.test(code[end])) end++
|
|
549
|
+
append(T_PROPERTY, current + code.slice(i, end))
|
|
550
|
+
i = end - 1
|
|
551
|
+
continue
|
|
552
|
+
}
|
|
553
|
+
// `=` in property=<value>
|
|
554
|
+
if (next === '=' && !inStringContent()) {
|
|
555
|
+
// if current is not a space, ensure `prop` is a property
|
|
556
|
+
if (!isSpaces(curr)) {
|
|
557
|
+
// If there're leading spaces, append them first
|
|
558
|
+
if (isSpaces(current)) {
|
|
559
|
+
append()
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// Now check if the accumulated token is a property
|
|
563
|
+
const prop = current + curr
|
|
564
|
+
if (isIdentifier(prop)) {
|
|
565
|
+
append(T_PROPERTY, prop)
|
|
566
|
+
continue
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// if it's not in a jsx tag declaration or a string, close child if next is jsx close tag
|
|
574
|
+
if (supportsJsx && !__jsxTag && (curr === '<' && isIdentifierChar(next) || c_n === '</')) {
|
|
575
|
+
let prevNonSpace = i - 1
|
|
576
|
+
while (prevNonSpace >= 0 && /\s/.test(code[prevNonSpace])) prevNonSpace--
|
|
577
|
+
const prevChar = prevNonSpace >= 0 ? code[prevNonSpace] : ''
|
|
578
|
+
|
|
579
|
+
const [lastType, lastTok] = last
|
|
580
|
+
// Without a space before `<`, the LHS is often still in `current` (not flushed), so `last` is stale.
|
|
581
|
+
let typeArgFromPending = false
|
|
582
|
+
let jsxFromPending = false
|
|
583
|
+
if (current && !isSpaces(current)) {
|
|
584
|
+
const w = current
|
|
585
|
+
// Unflushed LHS before `<`: numbers/null/Upper (isCls), and booleans, behave like `foo<` (type
|
|
586
|
+
// args / `<`). Any other keyword (`return`, `void`, …) is JSX-friendly — no keyword allowlist.
|
|
587
|
+
if (isCls(w) || w === 'true' || w === 'false') {
|
|
588
|
+
typeArgFromPending = true
|
|
589
|
+
} else if (resolvedKeywords.has(w) && isIdentifier(w)) {
|
|
590
|
+
jsxFromPending = true
|
|
591
|
+
} else if (isIdentifier(w)) {
|
|
592
|
+
typeArgFromPending = true
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
const isTsTypeArgStart =
|
|
597
|
+
curr === '<' &&
|
|
598
|
+
/[$\w\]\)]/.test(prevChar) &&
|
|
599
|
+
(typeArgFromPending ||
|
|
600
|
+
(!jsxFromPending &&
|
|
601
|
+
(lastType === T_IDENTIFIER ||
|
|
602
|
+
lastType === T_CLS_NUMBER ||
|
|
603
|
+
(lastType === T_SIGN && (lastTok === ')' || lastTok === ']')))))
|
|
604
|
+
const isTsGenericStart = curr === '<' && isTypeParameterListStart(code, i)
|
|
605
|
+
if (!isTsTypeArgStart && !isTsGenericStart) {
|
|
606
|
+
__jsxTag = next === '/' ? 2 : 1
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
if (curr === '<' && (next === '/' || isAlpha(next))) {
|
|
610
|
+
if (
|
|
611
|
+
!isTsTypeArgStart &&
|
|
612
|
+
!isTsGenericStart &&
|
|
613
|
+
!inStringContent() &&
|
|
614
|
+
!inJsxLiterals() &&
|
|
615
|
+
!inRegexQuotes()
|
|
616
|
+
) {
|
|
617
|
+
__jsxEnter = true
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
const isQuotationChar = isSingleQuotes(curr) || isTemplateQuote(curr)
|
|
623
|
+
const isStringTemplateLiterals = inStrTemplateLiterals()
|
|
624
|
+
const isRegexChar = supportsRegex && !__jsxEnter && isRegexStart(c_n)
|
|
625
|
+
const isJsxLiterals = inJsxLiterals()
|
|
626
|
+
|
|
627
|
+
// string quotation
|
|
628
|
+
if (isQuotationChar || isStringTemplateLiterals || isSingleQuotes(__strQuote)) {
|
|
629
|
+
current += curr
|
|
630
|
+
} else if (isRegexChar) {
|
|
631
|
+
append()
|
|
632
|
+
const [lastType, lastToken] = last
|
|
633
|
+
// Special cases that are not considered as regex:
|
|
634
|
+
// * (expr1) / expr2: `)` before `/` operator is still in expression
|
|
635
|
+
// * <non comment start>/ expr: non comment start before `/` is not regex
|
|
636
|
+
if (
|
|
637
|
+
isRegexChar &&
|
|
638
|
+
lastType !== -1 &&
|
|
639
|
+
!(
|
|
640
|
+
(lastType === T_SIGN && ')' !== lastToken) ||
|
|
641
|
+
lastType === T_COMMENT
|
|
642
|
+
)
|
|
643
|
+
) {
|
|
644
|
+
current = curr
|
|
645
|
+
append()
|
|
646
|
+
continue
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
__regexQuoteStart = true
|
|
650
|
+
const start = i++
|
|
651
|
+
|
|
652
|
+
// end of line of end of file
|
|
653
|
+
const isEof = () => i >= code.length
|
|
654
|
+
const isEol = () => isEof() || code[i] === '\n'
|
|
655
|
+
|
|
656
|
+
let foundClose = false
|
|
657
|
+
|
|
658
|
+
// `/` is literal inside regex character classes, e.g. `[/]`.
|
|
659
|
+
let inCharClass = false
|
|
660
|
+
|
|
661
|
+
// traverse to find closing regex slash
|
|
662
|
+
for (; !isEol(); i++) {
|
|
663
|
+
const ch = code[i]
|
|
664
|
+
const escaped = code[i - 1] === '\\'
|
|
665
|
+
if (!escaped && ch === '[') inCharClass = true
|
|
666
|
+
if (!escaped && ch === ']') inCharClass = false
|
|
667
|
+
if (ch === '/' && !inCharClass && !escaped) {
|
|
668
|
+
foundClose = true
|
|
669
|
+
// end of regex, append regex flags
|
|
670
|
+
while (start !== i && /^[a-z]$/.test(code[i + 1]) && !isEol()) {
|
|
671
|
+
i++
|
|
672
|
+
}
|
|
673
|
+
break
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
__regexQuoteStart = false
|
|
677
|
+
|
|
678
|
+
if (start !== i && foundClose) {
|
|
679
|
+
// If current line is fully closed with string quotes or regex slashes,
|
|
680
|
+
// add them to tokens
|
|
681
|
+
current = code.slice(start, i + 1)
|
|
682
|
+
append(T_STRING)
|
|
683
|
+
} else {
|
|
684
|
+
// If it doesn't match any of the above, just leave it as operator and move on
|
|
685
|
+
current = curr
|
|
686
|
+
append()
|
|
687
|
+
i = start
|
|
688
|
+
}
|
|
689
|
+
} else if (onCommentStart(curr, next, i, code)) {
|
|
690
|
+
append()
|
|
691
|
+
const start = i
|
|
692
|
+
const startCommentType = onCommentStart(curr, next, i, code)
|
|
693
|
+
|
|
694
|
+
// just match the comment, commentType === true
|
|
695
|
+
// inline comment, commentType === 1
|
|
696
|
+
// block comment, commentType === 2
|
|
697
|
+
if (startCommentType) {
|
|
698
|
+
for (; i < code.length; i++) {
|
|
699
|
+
const endCommentType = onCommentEnd(code[i - 1], code[i], i, code)
|
|
700
|
+
if (endCommentType == startCommentType) break
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
current = code.slice(start, i + 1)
|
|
704
|
+
append(T_COMMENT)
|
|
705
|
+
} else if (curr === ' ' || curr === '\n') {
|
|
706
|
+
if (
|
|
707
|
+
curr === ' ' &&
|
|
708
|
+
(
|
|
709
|
+
(isSpaces(current) || !current) ||
|
|
710
|
+
isJsxLiterals
|
|
711
|
+
)
|
|
712
|
+
) {
|
|
713
|
+
let end = i + 1
|
|
714
|
+
while (code[end] === ' ') end++
|
|
715
|
+
current += code.slice(i, end)
|
|
716
|
+
i = end - 1
|
|
717
|
+
if (code[end] === '<') {
|
|
718
|
+
append()
|
|
719
|
+
}
|
|
720
|
+
} else {
|
|
721
|
+
append()
|
|
722
|
+
current = curr
|
|
723
|
+
append()
|
|
724
|
+
}
|
|
725
|
+
} else {
|
|
726
|
+
if (__jsxExpr && curr === '}') {
|
|
727
|
+
append()
|
|
728
|
+
current = curr
|
|
729
|
+
append()
|
|
730
|
+
__jsxExpr = false
|
|
731
|
+
} else if (
|
|
732
|
+
// it's jsx literals and is not a jsx bracket
|
|
733
|
+
(isJsxLiterals && !JSXBrackets.has(curr)) ||
|
|
734
|
+
// it's template literal content (including quotes)
|
|
735
|
+
inStrTemplateLiterals() ||
|
|
736
|
+
// same type char as previous one in current token
|
|
737
|
+
((isWord(curr) === isWord(current[current.length - 1]) || __jsxChild()) && !Signs.has(curr))
|
|
738
|
+
) {
|
|
739
|
+
current += curr
|
|
740
|
+
} else {
|
|
741
|
+
if (p_c === '</') {
|
|
742
|
+
current = p_c
|
|
743
|
+
}
|
|
744
|
+
append()
|
|
745
|
+
|
|
746
|
+
if (p_c !== '</') {
|
|
747
|
+
current = curr
|
|
748
|
+
|
|
749
|
+
}
|
|
750
|
+
if ((c_n === '</' || c_n === '/>')) {
|
|
751
|
+
current = c_n
|
|
752
|
+
append()
|
|
753
|
+
i++
|
|
754
|
+
}
|
|
755
|
+
else if (JSXBrackets.has(curr)) append()
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
append()
|
|
761
|
+
|
|
762
|
+
return tokens
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
export { tokenize }
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* @typedef {Object} HighlightOptions
|
|
769
|
+
* @property {Set<string>} [keywords]
|
|
770
|
+
* @property {Set<string>} [typeKeywords]
|
|
771
|
+
* @property {(curr: string, next: string, index: number, code: string) => number | boolean} [onCommentStart]
|
|
772
|
+
* @property {(prev: string, curr: string, index: number, code: string) => number | boolean} [onCommentEnd]
|
|
773
|
+
* @property {(curr: string, i: number, code: string) => number | null | undefined} [onQuote]
|
|
774
|
+
* @property {boolean} [quotedKeys]
|
|
775
|
+
* @property {boolean} [jsx] Whether JSX tag parsing is enabled.
|
|
776
|
+
* @property {boolean} [regex] Whether JavaScript-style regular expressions are enabled.
|
|
777
|
+
* @property {boolean} [templateStrings] Whether JavaScript template strings are enabled.
|
|
778
|
+
* @property {boolean} [caseInsensitive] Whether keyword matching ignores case.
|
|
779
|
+
* @property {boolean} [typescript] Override heuristic TypeScript detection.
|
|
780
|
+
* @property {(line: import('../../core.js').AnnotateLine) => void} [annotateLine]
|
|
781
|
+
*/
|