svelte-highlight 7.22.0 → 7.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AnsiOutput.svelte +6 -30
- package/FileTabs.svelte +3 -1
- package/HighlightEditable.svelte +115 -143
- package/HighlightStream.svelte +44 -48
- package/HighlightStyle.svelte +9 -21
- package/HighlightVirtual.svelte +82 -4
- package/HighlightVirtual.svelte.d.ts +19 -2
- package/MarkdownStream.svelte +16 -9
- package/README.md +27 -7
- package/ansi-color.js +109 -22
- package/ansi-output.js +93 -0
- package/ansi.d.ts +18 -0
- package/ansi.js +85 -24
- package/css-walk.js +9 -12
- package/editable-css-paint.js +118 -0
- package/editable-dom-paint.js +217 -146
- package/editable-line-dom.js +241 -0
- package/engine.d.ts +28 -6
- package/engine.js +707 -144
- package/fence.d.ts +5 -5
- package/fence.js +196 -56
- package/highlight-theme.js +10 -4
- package/incremental-tokenize.js +95 -17
- package/package.json +1 -1
- package/preprocessor.js +16 -2
- package/scoped.js +15 -13
- package/search.d.ts +6 -4
- package/search.js +427 -85
- package/stream-final-highlight.js +52 -0
- package/stream-highlighted.js +17 -0
- package/stream-preview.js +7 -2
- package/stream-regenerate.js +262 -0
- package/style-registry.js +60 -0
- package/text-diff.js +66 -18
- package/tokenized-document.d.ts +8 -0
- package/tokenized-document.js +60 -15
- package/worker.js +14 -2
package/engine.js
CHANGED
|
@@ -69,7 +69,23 @@ import {
|
|
|
69
69
|
* earliest guard-passing match at or after the position `this.pos` had when
|
|
70
70
|
* it was computed (or `null` if none exists in `this.code` as of `codeLen`
|
|
71
71
|
* characters). See `Tokenizer#nextMatch`'s cache-hit checks.
|
|
72
|
-
*
|
|
72
|
+
*
|
|
73
|
+
* A windowed tokenizer (see `Tokenizer#windowed`) also sets `until` and
|
|
74
|
+
* `level`. With `match: null`, `until` is the end of the window it scanned:
|
|
75
|
+
* no guard-passing match starts before it. `until` is `Infinity` once a scan
|
|
76
|
+
* reached the end of the code. `level` picks the window size for the next
|
|
77
|
+
* scan (see `scanWindowed`).
|
|
78
|
+
* @typedef {{
|
|
79
|
+
* codeLen: number,
|
|
80
|
+
* match: RegExpExecArray | null,
|
|
81
|
+
* until?: number,
|
|
82
|
+
* level?: number,
|
|
83
|
+
* }} MatchCache
|
|
84
|
+
*/
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* A windowed tokenizer's `MatchCache`, which always has `until`/`level`.
|
|
88
|
+
* @typedef {Required<MatchCache>} WindowCache
|
|
73
89
|
*/
|
|
74
90
|
|
|
75
91
|
/**
|
|
@@ -109,7 +125,10 @@ import {
|
|
|
109
125
|
*/
|
|
110
126
|
|
|
111
127
|
/**
|
|
112
|
-
*
|
|
128
|
+
* Which kind of pattern produced `nextMatch`'s winner. Its `matchData` is
|
|
129
|
+
* the rule's state index for "begin", the frame depth for "end", and null
|
|
130
|
+
* for "illegal".
|
|
131
|
+
* @typedef {"begin" | "end" | "illegal"} MatchKind
|
|
113
132
|
*/
|
|
114
133
|
|
|
115
134
|
const MAX_KEYWORD_HITS = 7;
|
|
@@ -122,6 +141,54 @@ const MAX_KEYWORD_HITS = 7;
|
|
|
122
141
|
*/
|
|
123
142
|
const DETECT_SAMPLE_LIMIT = 8000;
|
|
124
143
|
|
|
144
|
+
/**
|
|
145
|
+
* Window sizes for `Tokenizer#scanWindowed`: `WINDOW_BASE << level` match
|
|
146
|
+
* start positions, for `level` in `[0, WINDOW_LEVELS)`. Past the last level
|
|
147
|
+
* the scan runs to the end of the code, as an unwindowed scan does. A
|
|
148
|
+
* window scan reads more slowly per character than a plain `exec` (which
|
|
149
|
+
* can skip ahead for a literal first character), so the windows stay small
|
|
150
|
+
* and an edit that never re-converges soon falls back to plain scans (see
|
|
151
|
+
* bench/incremental.bench.ts).
|
|
152
|
+
*/
|
|
153
|
+
const WINDOW_BASE = 256;
|
|
154
|
+
const WINDOW_LEVELS = 5;
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Window regexes by source regex and level (see `windowRegExp`).
|
|
158
|
+
* @type {WeakMap<RegExp, RegExp[]>}
|
|
159
|
+
*/
|
|
160
|
+
const windowRegExps = new WeakMap();
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* A sticky regex that, run at `lastIndex = from`, matches the shortest run
|
|
164
|
+
* of up to `(WINDOW_BASE << level) - 1` characters after which `re` matches.
|
|
165
|
+
* So `from + match[0].length` is the first position in the window where an
|
|
166
|
+
* attempt of `re` succeeds, with the same result an unbounded `re.exec`
|
|
167
|
+
* from `from` would find there: the lookahead sees the whole code, so
|
|
168
|
+
* anchors, `\b`, and lookarounds behave exactly as in `re`. The prefix is
|
|
169
|
+
* non-capturing, so `re`'s group numbers and backreferences don't shift.
|
|
170
|
+
* @param {RegExp} re
|
|
171
|
+
* @param {number} level
|
|
172
|
+
* @returns {RegExp}
|
|
173
|
+
*/
|
|
174
|
+
function windowRegExp(re, level) {
|
|
175
|
+
let byLevel = windowRegExps.get(re);
|
|
176
|
+
if (byLevel === undefined) {
|
|
177
|
+
byLevel = [];
|
|
178
|
+
windowRegExps.set(re, byLevel);
|
|
179
|
+
}
|
|
180
|
+
let windowRe = byLevel[level];
|
|
181
|
+
if (windowRe === undefined) {
|
|
182
|
+
const span = (WINDOW_BASE << level) - 1;
|
|
183
|
+
windowRe = new RegExp(
|
|
184
|
+
`[\\s\\S]{0,${span}}?(?=(?:${re.source}))`,
|
|
185
|
+
`${re.flags.replace("g", "")}y`,
|
|
186
|
+
);
|
|
187
|
+
byLevel[level] = windowRe;
|
|
188
|
+
}
|
|
189
|
+
return windowRe;
|
|
190
|
+
}
|
|
191
|
+
|
|
125
192
|
// Tokenizer#isTrulyOpeningTag lookaheads.
|
|
126
193
|
const XML_TAG_DEFAULT_PARAM_RE = /^\s*=/;
|
|
127
194
|
const XML_TAG_EXTENDS_CONSTRAINT_RE = /^\s+extends\s+/;
|
|
@@ -150,8 +217,11 @@ export class UnknownLanguageError extends Error {
|
|
|
150
217
|
}
|
|
151
218
|
|
|
152
219
|
/**
|
|
153
|
-
* Thrown by `Tokenizer#run` once a parse exceeds 500,000 iterations
|
|
154
|
-
*
|
|
220
|
+
* Thrown by `Tokenizer#run` once a parse exceeds both 500,000 iterations
|
|
221
|
+
* and three iterations per character consumed, guarding against a grammar
|
|
222
|
+
* bug that never advances `pos`. The per-character term keeps large inputs
|
|
223
|
+
* legal: real grammars take well under one iteration per character, and a
|
|
224
|
+
* flat cap rejected plain JS past ~80k lines.
|
|
155
225
|
*/
|
|
156
226
|
export class TokenizerLoopError extends Error {
|
|
157
227
|
/**
|
|
@@ -169,26 +239,64 @@ export class TokenizerLoopError extends Error {
|
|
|
169
239
|
/** Scope name prefix marking a sub-language boundary (`language:css`). */
|
|
170
240
|
const LANGUAGE_SCOPE_PREFIX = "language:";
|
|
171
241
|
|
|
172
|
-
const HTML_ESCAPE_RE = /[&<>"']/g;
|
|
173
|
-
/** @type {Record<string, string>} */
|
|
174
|
-
const HTML_ESCAPE_MAP = {
|
|
175
|
-
"&": "&",
|
|
176
|
-
"<": "<",
|
|
177
|
-
">": ">",
|
|
178
|
-
'"': """,
|
|
179
|
-
"'": "'",
|
|
180
|
-
};
|
|
181
|
-
|
|
182
242
|
/**
|
|
183
243
|
* Five sequential `.replace()` passes (one per character) previously cost
|
|
184
244
|
* ~87% of renderHtml's time, since every TEXT event runs through this. A
|
|
185
|
-
* single
|
|
186
|
-
* that to a fraction
|
|
245
|
+
* single pass that returns `value` untouched when there's nothing to escape
|
|
246
|
+
* cuts that to a fraction. It scans char codes rather than calling a regex:
|
|
247
|
+
* most TEXT values are a few characters long, where a regex call's fixed
|
|
248
|
+
* cost dominated (see bench/render.bench.ts).
|
|
187
249
|
* @param {string} value
|
|
188
250
|
*/
|
|
189
251
|
export function escapeHtml(value) {
|
|
190
|
-
|
|
191
|
-
|
|
252
|
+
for (let i = 0; i < value.length; i++) {
|
|
253
|
+
const code = value.charCodeAt(i);
|
|
254
|
+
// `>` (62) is the highest code point that needs escaping.
|
|
255
|
+
if (code <= 62 && htmlEscape(code) !== null)
|
|
256
|
+
return escapeHtmlFrom(value, i);
|
|
257
|
+
}
|
|
258
|
+
return value;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* The entity for an escaped char code, or null if it needs no escaping.
|
|
263
|
+
* @param {number} code
|
|
264
|
+
* @returns {string | null}
|
|
265
|
+
*/
|
|
266
|
+
function htmlEscape(code) {
|
|
267
|
+
switch (code) {
|
|
268
|
+
case 38:
|
|
269
|
+
return "&";
|
|
270
|
+
case 60:
|
|
271
|
+
return "<";
|
|
272
|
+
case 62:
|
|
273
|
+
return ">";
|
|
274
|
+
case 34:
|
|
275
|
+
return """;
|
|
276
|
+
case 39:
|
|
277
|
+
return "'";
|
|
278
|
+
default:
|
|
279
|
+
return null;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* `escapeHtml`'s slow path, from the first index `start` that needs escaping.
|
|
285
|
+
* @param {string} value
|
|
286
|
+
* @param {number} start
|
|
287
|
+
*/
|
|
288
|
+
function escapeHtmlFrom(value, start) {
|
|
289
|
+
let out = value.slice(0, start);
|
|
290
|
+
let last = start;
|
|
291
|
+
for (let i = start; i < value.length; i++) {
|
|
292
|
+
const code = value.charCodeAt(i);
|
|
293
|
+
if (code > 62) continue;
|
|
294
|
+
const entity = htmlEscape(code);
|
|
295
|
+
if (entity === null) continue;
|
|
296
|
+
out += value.slice(last, i) + entity;
|
|
297
|
+
last = i + 1;
|
|
298
|
+
}
|
|
299
|
+
return out + value.slice(last);
|
|
192
300
|
}
|
|
193
301
|
|
|
194
302
|
/**
|
|
@@ -236,6 +344,14 @@ class Tokenizer {
|
|
|
236
344
|
this.openScopes = 0;
|
|
237
345
|
/** @type {Record<string, number>} */
|
|
238
346
|
this.kwHits = Object.create(null);
|
|
347
|
+
/**
|
|
348
|
+
* Whether `kwHits` is shared with a snapshot, so the next keyword hit
|
|
349
|
+
* must copy it before writing. `snapshot()` and `restore()` copied it
|
|
350
|
+
* on every call, about 9% of a streaming repaint, though most frames
|
|
351
|
+
* never hit a keyword before the next snapshot (see
|
|
352
|
+
* bench/stream-repaint.bench.ts).
|
|
353
|
+
*/
|
|
354
|
+
this.kwHitsShared = false;
|
|
239
355
|
/** @type {Frame[]} */
|
|
240
356
|
this.frames = [
|
|
241
357
|
{
|
|
@@ -249,6 +365,16 @@ class Tokenizer {
|
|
|
249
365
|
];
|
|
250
366
|
this.aborted = false;
|
|
251
367
|
this.iterations = 0;
|
|
368
|
+
/**
|
|
369
|
+
* Whether begin/end scans search a bounded window past the cursor
|
|
370
|
+
* instead of running to the end of the code (see `nextMatchWindowed`).
|
|
371
|
+
* Set for a parse resumed after an edit, which usually stops a few
|
|
372
|
+
* lines later: a rule with no match nearby otherwise scanned the whole
|
|
373
|
+
* tail of the document, so a one-character edit cost time in
|
|
374
|
+
* proportion to the document (see bench/incremental.bench.ts). Never
|
|
375
|
+
* set in detect mode, so the windowed path ignores `illegal`.
|
|
376
|
+
*/
|
|
377
|
+
this.windowed = false;
|
|
252
378
|
/**
|
|
253
379
|
* Embedded-language parse state, carried across segments by name - but
|
|
254
380
|
* only for the *same* embedding occurrence (matched by `beginPos`, the
|
|
@@ -269,6 +395,16 @@ class Tokenizer {
|
|
|
269
395
|
* @type {(MatchCache | undefined)[]}
|
|
270
396
|
*/
|
|
271
397
|
this.beginCache = [];
|
|
398
|
+
/**
|
|
399
|
+
* Memoized `illegal`-pattern scans (detect mode only), indexed by state
|
|
400
|
+
* index. Like `beginCache`, a state's next illegal match doesn't depend
|
|
401
|
+
* on the frame. Uncached, every lexeme re-ran the pattern from `pos`,
|
|
402
|
+
* and a pattern with no match left (the common case for a candidate
|
|
403
|
+
* that won't abort) scanned the whole remaining sample each time:
|
|
404
|
+
* quadratic in sample length (see bench/auto-detect.bench.ts).
|
|
405
|
+
* @type {(MatchCache | undefined)[]}
|
|
406
|
+
*/
|
|
407
|
+
this.illegalCache = [];
|
|
272
408
|
/**
|
|
273
409
|
* Tag names with no `</name` at or after the recorded position. See
|
|
274
410
|
* `hasClosingTag`.
|
|
@@ -276,6 +412,10 @@ class Tokenizer {
|
|
|
276
412
|
*/
|
|
277
413
|
this.noClosingTag = new Map();
|
|
278
414
|
this.noClosingTagCodeLen = 0;
|
|
415
|
+
/** Kind of the last `nextMatch` winner. @type {MatchKind} */
|
|
416
|
+
this.matchKind = "begin";
|
|
417
|
+
/** Data of the last `nextMatch` winner. @type {number | null} */
|
|
418
|
+
this.matchData = null;
|
|
279
419
|
}
|
|
280
420
|
|
|
281
421
|
/** @returns {Frame} */
|
|
@@ -338,6 +478,12 @@ class Tokenizer {
|
|
|
338
478
|
}
|
|
339
479
|
|
|
340
480
|
/**
|
|
481
|
+
* Splits `text` into keyword and plain-text events. Plain runs are sliced
|
|
482
|
+
* straight out of `text` rather than rebuilt by `+=` per word, which made
|
|
483
|
+
* every non-keyword word a string concatenation (see
|
|
484
|
+
* bench/tokenize.bench.ts). A keyword hit always ends the current TEXT
|
|
485
|
+
* event, even a relevance-only (`_`-prefixed) one, which stays in the
|
|
486
|
+
* next plain run.
|
|
341
487
|
* @param {string} text
|
|
342
488
|
* @param {CompiledState} state
|
|
343
489
|
*/
|
|
@@ -346,42 +492,44 @@ class Tokenizer {
|
|
|
346
492
|
this.text(text);
|
|
347
493
|
return;
|
|
348
494
|
}
|
|
495
|
+
if (text === "") return;
|
|
349
496
|
const keywords = state.keywords;
|
|
350
497
|
// keywordRe is always compiled alongside keywords (see compileProgram),
|
|
351
498
|
// so it is non-null whenever `keywords` is set.
|
|
352
499
|
const keywordRe = /** @type {RegExp} */ (state.keywordRe);
|
|
353
|
-
|
|
354
|
-
let
|
|
500
|
+
const caseInsensitive = this.program.ir.caseInsensitive;
|
|
501
|
+
let kwHits = this.kwHits;
|
|
502
|
+
let textStart = 0;
|
|
355
503
|
keywordRe.lastIndex = 0;
|
|
356
504
|
let match = keywordRe.exec(text);
|
|
357
505
|
while (match) {
|
|
358
|
-
|
|
359
|
-
const word =
|
|
360
|
-
? match[0].toLowerCase()
|
|
361
|
-
: match[0];
|
|
506
|
+
const lexeme = match[0];
|
|
507
|
+
const word = caseInsensitive ? lexeme.toLowerCase() : lexeme;
|
|
362
508
|
const data = keywords[word];
|
|
363
509
|
if (data) {
|
|
364
510
|
const [kind, keywordRelevance] = data;
|
|
365
|
-
this.text(
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
511
|
+
this.text(text.substring(textStart, match.index));
|
|
512
|
+
if (this.kwHitsShared) {
|
|
513
|
+
// Copied into a null-prototype object before any read, so a
|
|
514
|
+
// restored plain-object snapshot can't leak `constructor` etc.
|
|
515
|
+
kwHits = Object.assign(Object.create(null), kwHits);
|
|
516
|
+
this.kwHits = kwHits;
|
|
517
|
+
this.kwHitsShared = false;
|
|
370
518
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
519
|
+
const hits = (kwHits[word] || 0) + 1;
|
|
520
|
+
kwHits[word] = hits;
|
|
521
|
+
if (hits <= MAX_KEYWORD_HITS) this.relevance += keywordRelevance;
|
|
522
|
+
if (kind.charCodeAt(0) === 95) {
|
|
523
|
+
// "_": relevance-only keyword, not highlighted
|
|
524
|
+
textStart = match.index;
|
|
374
525
|
} else {
|
|
375
|
-
this.emitKeyword(
|
|
526
|
+
this.emitKeyword(lexeme, kind);
|
|
527
|
+
textStart = keywordRe.lastIndex;
|
|
376
528
|
}
|
|
377
|
-
} else {
|
|
378
|
-
buf += match[0];
|
|
379
529
|
}
|
|
380
|
-
lastIndex = keywordRe.lastIndex;
|
|
381
530
|
match = keywordRe.exec(text);
|
|
382
531
|
}
|
|
383
|
-
|
|
384
|
-
this.text(buf);
|
|
532
|
+
this.text(textStart === 0 ? text : text.substring(textStart));
|
|
385
533
|
}
|
|
386
534
|
|
|
387
535
|
/** @param {CompiledState} state */
|
|
@@ -478,8 +626,11 @@ class Tokenizer {
|
|
|
478
626
|
* boundary re-scans every rule's pattern from scratch. A cached match stays
|
|
479
627
|
* valid until `this.pos` passes it; a cached miss stays valid until the
|
|
480
628
|
* code grows (streaming append).
|
|
629
|
+
* Returns a plain boolean, not a `cache is MatchCache` predicate, so a
|
|
630
|
+
* stale cache isn't narrowed to `undefined` where `nextMatch` refreshes
|
|
631
|
+
* it in place.
|
|
481
632
|
* @param {MatchCache | undefined} cache
|
|
482
|
-
* @returns {
|
|
633
|
+
* @returns {boolean}
|
|
483
634
|
*/
|
|
484
635
|
isCacheValid(cache) {
|
|
485
636
|
if (!cache) return false;
|
|
@@ -573,103 +724,325 @@ class Tokenizer {
|
|
|
573
724
|
|
|
574
725
|
/**
|
|
575
726
|
* The next lexeme: earliest match wins; ties break by priority (begin
|
|
576
|
-
* rules in order, then ends innermost-first, then illegal).
|
|
577
|
-
*
|
|
727
|
+
* rules in order, then ends innermost-first, then illegal). Returns the
|
|
728
|
+
* winning match and leaves its kind/data in `matchKind`/`matchData`.
|
|
729
|
+
*
|
|
730
|
+
* Runs once per lexeme, so it allocates nothing on cache hits: no
|
|
731
|
+
* `consider()` closure over mutable locals, no `{ match, kind, data }`
|
|
732
|
+
* result object, and stale caches are refreshed in place (see
|
|
733
|
+
* bench/tokenize.bench.ts).
|
|
734
|
+
* @returns {RegExpExecArray | null}
|
|
578
735
|
*/
|
|
579
736
|
nextMatch() {
|
|
580
737
|
/** @type {RegExpExecArray | null} */
|
|
581
738
|
let best = null;
|
|
582
|
-
let
|
|
583
|
-
/** @type {
|
|
584
|
-
let bestKind = "";
|
|
739
|
+
let bestIndex = Number.POSITIVE_INFINITY;
|
|
740
|
+
/** @type {MatchKind} */
|
|
741
|
+
let bestKind = "begin";
|
|
585
742
|
/** @type {number | null} */
|
|
586
743
|
let bestData = null;
|
|
587
|
-
/**
|
|
588
|
-
* @param {RegExpExecArray | null} match
|
|
589
|
-
* @param {number} pri
|
|
590
|
-
* @param {"begin" | "end" | "illegal"} kind
|
|
591
|
-
* @param {number | null} data
|
|
592
|
-
*/
|
|
593
|
-
const consider = (match, pri, kind, data) => {
|
|
594
|
-
if (!match) return;
|
|
595
|
-
if (
|
|
596
|
-
best === null ||
|
|
597
|
-
match.index < best.index ||
|
|
598
|
-
(match.index === best.index && pri < bestPri)
|
|
599
|
-
) {
|
|
600
|
-
best = match;
|
|
601
|
-
bestPri = pri;
|
|
602
|
-
bestKind = kind;
|
|
603
|
-
bestData = data;
|
|
604
|
-
}
|
|
605
|
-
};
|
|
606
744
|
|
|
607
745
|
const state = this.top.state;
|
|
608
|
-
|
|
746
|
+
const rules = state.rules;
|
|
747
|
+
const states = this.program.states;
|
|
748
|
+
// Begin rules come first and in priority order, so among them a strict
|
|
749
|
+
// `<` keeps the earliest rule on ties.
|
|
750
|
+
for (let i = 0; i < rules.length; i++) {
|
|
609
751
|
// rules[i] is in-bounds by the loop condition.
|
|
610
|
-
const ruleIdx = /** @type {number} */ (
|
|
611
|
-
const child = /** @type {CompiledState} */ (this.program.states[ruleIdx]);
|
|
752
|
+
const ruleIdx = /** @type {number} */ (rules[i]);
|
|
612
753
|
let cache = this.beginCache[ruleIdx];
|
|
613
754
|
// Guard closures (this.beginGuard(child)) are only built on a cache
|
|
614
755
|
// miss - passing them in as a pre-built thunk, as this used to via a
|
|
615
756
|
// shared cachedMatch() helper, allocated one on every call regardless
|
|
616
757
|
// of hit/miss, since JS evaluates argument expressions eagerly.
|
|
617
758
|
if (!this.isCacheValid(cache)) {
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
759
|
+
const child = /** @type {CompiledState} */ (states[ruleIdx]);
|
|
760
|
+
const match = this.execValid(
|
|
761
|
+
// Every state referenced from a `rules` list has a `begin` pattern.
|
|
762
|
+
/** @type {RegExp} */ (child.beginRe),
|
|
763
|
+
this.pos,
|
|
764
|
+
this.beginGuard(child),
|
|
765
|
+
);
|
|
766
|
+
if (cache) {
|
|
767
|
+
cache.codeLen = this.code.length;
|
|
768
|
+
cache.match = match;
|
|
769
|
+
} else {
|
|
770
|
+
cache = { codeLen: this.code.length, match };
|
|
771
|
+
this.beginCache[ruleIdx] = cache;
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
const match = /** @type {MatchCache} */ (cache).match;
|
|
775
|
+
if (match !== null && match.index < bestIndex) {
|
|
776
|
+
best = match;
|
|
777
|
+
bestIndex = match.index;
|
|
778
|
+
bestData = ruleIdx;
|
|
628
779
|
}
|
|
629
|
-
consider(cache.match, i, "begin", ruleIdx);
|
|
630
780
|
}
|
|
631
|
-
// End candidates walk outward while `endsWithParent` chains allow.
|
|
632
|
-
|
|
781
|
+
// End candidates walk outward while `endsWithParent` chains allow. They
|
|
782
|
+
// rank below every begin rule and innermost-first, so they also only win
|
|
783
|
+
// with a strictly earlier index.
|
|
633
784
|
for (let d = this.frames.length - 1; d >= 1; d--) {
|
|
634
785
|
// d ranges over [1, frames.length - 1], always in bounds.
|
|
635
786
|
const frame = /** @type {Frame} */ (this.frames[d]);
|
|
636
|
-
|
|
787
|
+
const endRe = frame.state.endRe;
|
|
788
|
+
if (endRe) {
|
|
637
789
|
let cache = frame.endCache;
|
|
638
790
|
if (!this.isCacheValid(cache)) {
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
}
|
|
650
|
-
|
|
791
|
+
const match = this.execValid(
|
|
792
|
+
endRe,
|
|
793
|
+
this.pos,
|
|
794
|
+
frame.state.endSameAsBegin
|
|
795
|
+
? (/** @type {RegExpExecArray} */ m) => m[1] === frame.beginMatch
|
|
796
|
+
: null,
|
|
797
|
+
);
|
|
798
|
+
if (cache) {
|
|
799
|
+
cache.codeLen = this.code.length;
|
|
800
|
+
cache.match = match;
|
|
801
|
+
} else {
|
|
802
|
+
cache = { codeLen: this.code.length, match };
|
|
803
|
+
frame.endCache = cache;
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
const match = /** @type {MatchCache} */ (cache).match;
|
|
807
|
+
if (match !== null && match.index < bestIndex) {
|
|
808
|
+
best = match;
|
|
809
|
+
bestIndex = match.index;
|
|
810
|
+
bestKind = "end";
|
|
811
|
+
bestData = d;
|
|
651
812
|
}
|
|
652
|
-
consider(cache.match, pri++, "end", d);
|
|
653
813
|
}
|
|
654
814
|
if (!frame.state.endsWithParent) break;
|
|
655
815
|
}
|
|
656
816
|
if (this.detect && state.illegalRe) {
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
817
|
+
const stateIdx = this.top.idx;
|
|
818
|
+
let cache = this.illegalCache[stateIdx];
|
|
819
|
+
if (!this.isCacheValid(cache)) {
|
|
820
|
+
const match = this.execValid(state.illegalRe, this.pos, null);
|
|
821
|
+
if (cache) {
|
|
822
|
+
cache.codeLen = this.code.length;
|
|
823
|
+
cache.match = match;
|
|
824
|
+
} else {
|
|
825
|
+
cache = { codeLen: this.code.length, match };
|
|
826
|
+
this.illegalCache[stateIdx] = cache;
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
const match = /** @type {MatchCache} */ (cache).match;
|
|
830
|
+
if (match !== null && match.index < bestIndex) {
|
|
831
|
+
best = match;
|
|
832
|
+
bestKind = "illegal";
|
|
833
|
+
bestData = null;
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
this.matchKind = bestKind;
|
|
837
|
+
this.matchData = bestData;
|
|
838
|
+
return best;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
/**
|
|
842
|
+
* The end-pattern guard for `frame` (see `nextMatch`).
|
|
843
|
+
* @param {Frame} frame
|
|
844
|
+
* @returns {((m: RegExpExecArray) => boolean) | null}
|
|
845
|
+
*/
|
|
846
|
+
endGuard(frame) {
|
|
847
|
+
return frame.state.endSameAsBegin ? (m) => m[1] === frame.beginMatch : null;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Whether a windowed cache still answers "first match at or after
|
|
852
|
+
* `pos`": a match `pos` hasn't passed, or a miss whose window reaches
|
|
853
|
+
* past `pos` in unchanged code.
|
|
854
|
+
* @param {WindowCache} cache
|
|
855
|
+
* @returns {boolean}
|
|
856
|
+
*/
|
|
857
|
+
isWindowValid(cache) {
|
|
858
|
+
if (cache.match !== null) return cache.match.index >= this.pos;
|
|
859
|
+
return cache.codeLen === this.code.length && cache.until > this.pos;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
/**
|
|
863
|
+
* `execValid` over a window of `WINDOW_BASE << cache.level` start
|
|
864
|
+
* positions from `from`, written into `cache`: the first guard-passing
|
|
865
|
+
* match in the window, or a miss with `until` at the window's end. A
|
|
866
|
+
* window that reaches the end of the code, or a level past the last,
|
|
867
|
+
* gives the same answer as `execValid` (`until` is `Infinity`). Matches
|
|
868
|
+
* come from `re.exec` itself, so they're the same objects an unwindowed
|
|
869
|
+
* scan returns (see `windowRegExp`).
|
|
870
|
+
* @param {WindowCache} cache
|
|
871
|
+
* @param {RegExp} re
|
|
872
|
+
* @param {number} from
|
|
873
|
+
* @param {((m: RegExpExecArray) => boolean) | null} guard
|
|
874
|
+
*/
|
|
875
|
+
scanWindowed(cache, re, from, guard) {
|
|
876
|
+
const code = this.code;
|
|
877
|
+
cache.codeLen = code.length;
|
|
878
|
+
cache.until = Number.POSITIVE_INFINITY;
|
|
879
|
+
const size = WINDOW_BASE << cache.level;
|
|
880
|
+
// A window reaching the end of the code saves nothing, and a plain
|
|
881
|
+
// `exec` scans faster: near the end of a document (typing at the end,
|
|
882
|
+
// say) every scan is plain. Under the `u` flag a window could start
|
|
883
|
+
// inside a surrogate pair, where the window regex and `re` might step
|
|
884
|
+
// differently; scan plainly there too.
|
|
885
|
+
if (
|
|
886
|
+
cache.level >= WINDOW_LEVELS ||
|
|
887
|
+
from + size > code.length ||
|
|
888
|
+
re.unicode
|
|
889
|
+
) {
|
|
890
|
+
cache.match = this.execValid(re, from, guard);
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
893
|
+
const windowRe = windowRegExp(re, cache.level);
|
|
894
|
+
let start = from;
|
|
895
|
+
for (;;) {
|
|
896
|
+
windowRe.lastIndex = start;
|
|
897
|
+
const found = windowRe.exec(code);
|
|
898
|
+
if (found === null) {
|
|
899
|
+
cache.match = null;
|
|
900
|
+
// The window covered every start up to `code.length` inclusive.
|
|
901
|
+
if (start + size <= code.length) cache.until = start + size;
|
|
902
|
+
return;
|
|
903
|
+
}
|
|
904
|
+
const at = start + found[0].length;
|
|
905
|
+
re.lastIndex = at;
|
|
906
|
+
const match = re.exec(code);
|
|
907
|
+
if (match === null || match.index !== at) {
|
|
908
|
+
// Unreachable: the lookahead just matched `re` at `at`. Kept so a
|
|
909
|
+
// regex engine quirk can't change output, only cost.
|
|
910
|
+
cache.match = this.execValid(re, from, guard);
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
if (!guard || guard(match)) {
|
|
914
|
+
cache.match = match;
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
if (at >= code.length) {
|
|
918
|
+
cache.match = null;
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
start = at + 1;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
/**
|
|
926
|
+
* `nextMatch` for a windowed tokenizer (see `windowed`). A cached miss
|
|
927
|
+
* only rules out matches before its `until`, so its window is widened
|
|
928
|
+
* (twice the size, from `until`) while the rule could still beat the
|
|
929
|
+
* best match so far, then the candidates are compared again. Windows
|
|
930
|
+
* never widen past `stopAt`: `run` stops before any lexeme at or past
|
|
931
|
+
* it, so when nothing can start earlier this returns a match at or past
|
|
932
|
+
* `stopAt` (not necessarily the earliest) or null. Detect mode never
|
|
933
|
+
* runs windowed, so `illegal` is not scanned.
|
|
934
|
+
* @param {number} stopAt
|
|
935
|
+
* @returns {RegExpExecArray | null}
|
|
936
|
+
*/
|
|
937
|
+
nextMatchWindowed(stopAt) {
|
|
938
|
+
const rules = this.top.state.rules;
|
|
939
|
+
const states = this.program.states;
|
|
940
|
+
const frames = this.frames;
|
|
941
|
+
const beginCache = /** @type {(WindowCache | undefined)[]} */ (
|
|
942
|
+
this.beginCache
|
|
943
|
+
);
|
|
944
|
+
for (;;) {
|
|
945
|
+
/** @type {RegExpExecArray | null} */
|
|
946
|
+
let best = null;
|
|
947
|
+
let bestIndex = Number.POSITIVE_INFINITY;
|
|
948
|
+
/** @type {MatchKind} */
|
|
949
|
+
let bestKind = "begin";
|
|
950
|
+
/** @type {number | null} */
|
|
951
|
+
let bestData = null;
|
|
952
|
+
let minUntil = Number.POSITIVE_INFINITY;
|
|
953
|
+
for (let i = 0; i < rules.length; i++) {
|
|
954
|
+
const ruleIdx = /** @type {number} */ (rules[i]);
|
|
955
|
+
let cache = beginCache[ruleIdx];
|
|
956
|
+
if (cache === undefined) {
|
|
957
|
+
cache = { codeLen: -1, match: null, until: 0, level: 0 };
|
|
958
|
+
beginCache[ruleIdx] = cache;
|
|
959
|
+
}
|
|
960
|
+
if (!this.isWindowValid(cache)) {
|
|
961
|
+
// A miss whose window `pos` has passed: widen the next one.
|
|
962
|
+
if (cache.match === null && cache.codeLen === this.code.length) {
|
|
963
|
+
cache.level++;
|
|
964
|
+
}
|
|
965
|
+
const child = /** @type {CompiledState} */ (states[ruleIdx]);
|
|
966
|
+
this.scanWindowed(
|
|
967
|
+
cache,
|
|
968
|
+
/** @type {RegExp} */ (child.beginRe),
|
|
969
|
+
this.pos,
|
|
970
|
+
this.beginGuard(child),
|
|
971
|
+
);
|
|
972
|
+
}
|
|
973
|
+
const match = cache.match;
|
|
974
|
+
if (match === null) {
|
|
975
|
+
if (cache.until < minUntil) minUntil = cache.until;
|
|
976
|
+
} else if (match.index < bestIndex) {
|
|
977
|
+
best = match;
|
|
978
|
+
bestIndex = match.index;
|
|
979
|
+
bestData = ruleIdx;
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
for (let d = frames.length - 1; d >= 1; d--) {
|
|
983
|
+
const frame = /** @type {Frame} */ (frames[d]);
|
|
984
|
+
const endRe = frame.state.endRe;
|
|
985
|
+
if (endRe) {
|
|
986
|
+
let cache = /** @type {WindowCache | undefined} */ (frame.endCache);
|
|
987
|
+
if (cache === undefined) {
|
|
988
|
+
cache = { codeLen: -1, match: null, until: 0, level: 0 };
|
|
989
|
+
frame.endCache = cache;
|
|
990
|
+
}
|
|
991
|
+
if (!this.isWindowValid(cache)) {
|
|
992
|
+
if (cache.match === null && cache.codeLen === this.code.length) {
|
|
993
|
+
cache.level++;
|
|
994
|
+
}
|
|
995
|
+
this.scanWindowed(cache, endRe, this.pos, this.endGuard(frame));
|
|
996
|
+
}
|
|
997
|
+
const match = cache.match;
|
|
998
|
+
if (match === null) {
|
|
999
|
+
if (cache.until < minUntil) minUntil = cache.until;
|
|
1000
|
+
} else if (match.index < bestIndex) {
|
|
1001
|
+
best = match;
|
|
1002
|
+
bestIndex = match.index;
|
|
1003
|
+
bestKind = "end";
|
|
1004
|
+
bestData = d;
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
if (!frame.state.endsWithParent) break;
|
|
1008
|
+
}
|
|
1009
|
+
// Every unresolved rule starts strictly after `limit`, so it can
|
|
1010
|
+
// neither beat nor tie the best match, nor start before `stopAt`.
|
|
1011
|
+
// (`minUntil` stays Infinity when every rule is resolved.)
|
|
1012
|
+
const limit = Math.min(bestIndex, stopAt - 1);
|
|
1013
|
+
if (minUntil > limit || minUntil === Number.POSITIVE_INFINITY) {
|
|
1014
|
+
this.matchKind = bestKind;
|
|
1015
|
+
this.matchData = bestData;
|
|
1016
|
+
return best;
|
|
1017
|
+
}
|
|
1018
|
+
for (let i = 0; i < rules.length; i++) {
|
|
1019
|
+
const ruleIdx = /** @type {number} */ (rules[i]);
|
|
1020
|
+
const cache = /** @type {WindowCache} */ (beginCache[ruleIdx]);
|
|
1021
|
+
if (cache.match !== null || cache.until > limit) continue;
|
|
1022
|
+
const child = /** @type {CompiledState} */ (states[ruleIdx]);
|
|
1023
|
+
cache.level++;
|
|
1024
|
+
this.scanWindowed(
|
|
1025
|
+
cache,
|
|
1026
|
+
/** @type {RegExp} */ (child.beginRe),
|
|
1027
|
+
cache.until,
|
|
1028
|
+
this.beginGuard(child),
|
|
1029
|
+
);
|
|
1030
|
+
}
|
|
1031
|
+
for (let d = frames.length - 1; d >= 1; d--) {
|
|
1032
|
+
const frame = /** @type {Frame} */ (frames[d]);
|
|
1033
|
+
const cache = /** @type {WindowCache | undefined} */ (frame.endCache);
|
|
1034
|
+
if (cache && cache.match === null && cache.until <= limit) {
|
|
1035
|
+
cache.level++;
|
|
1036
|
+
this.scanWindowed(
|
|
1037
|
+
cache,
|
|
1038
|
+
/** @type {RegExp} */ (frame.state.endRe),
|
|
1039
|
+
cache.until,
|
|
1040
|
+
this.endGuard(frame),
|
|
1041
|
+
);
|
|
1042
|
+
}
|
|
1043
|
+
if (!frame.state.endsWithParent) break;
|
|
1044
|
+
}
|
|
663
1045
|
}
|
|
664
|
-
return best === null
|
|
665
|
-
? null
|
|
666
|
-
: {
|
|
667
|
-
match: best,
|
|
668
|
-
// consider() only assigns bestKind alongside best, so bestKind is
|
|
669
|
-
// never "" once best is non-null.
|
|
670
|
-
kind: /** @type {"begin" | "end" | "illegal"} */ (bestKind),
|
|
671
|
-
data: bestData,
|
|
672
|
-
};
|
|
673
1046
|
}
|
|
674
1047
|
|
|
675
1048
|
// --- state transitions ---
|
|
@@ -703,7 +1076,7 @@ class Tokenizer {
|
|
|
703
1076
|
* @returns {number}
|
|
704
1077
|
*/
|
|
705
1078
|
doBegin(match, idx) {
|
|
706
|
-
// idx comes from nextMatch's "begin"
|
|
1079
|
+
// idx comes from nextMatch's "begin" matchData, which is always a valid
|
|
707
1080
|
// program.states index (see the ruleIdx comment above).
|
|
708
1081
|
const state = /** @type {CompiledState} */ (this.program.states[idx]);
|
|
709
1082
|
const lexeme = match[0];
|
|
@@ -786,34 +1159,34 @@ class Tokenizer {
|
|
|
786
1159
|
if (this.aborted) return;
|
|
787
1160
|
for (;;) {
|
|
788
1161
|
this.iterations++;
|
|
789
|
-
if (this.iterations > 500000) {
|
|
1162
|
+
if (this.iterations > 500000 && this.iterations > this.pos * 3) {
|
|
790
1163
|
throw new TokenizerLoopError(this.program.ir.name, this.iterations);
|
|
791
1164
|
}
|
|
792
|
-
const
|
|
793
|
-
|
|
794
|
-
|
|
1165
|
+
const match = this.windowed
|
|
1166
|
+
? this.nextMatchWindowed(stopAt)
|
|
1167
|
+
: this.nextMatch();
|
|
1168
|
+
if (!match) break;
|
|
1169
|
+
if (match.index >= stopAt) return;
|
|
795
1170
|
|
|
796
1171
|
const framesBefore = this.frames.length;
|
|
797
1172
|
const topBefore = this.top;
|
|
798
|
-
this.buffer += this.code.slice(this.pos,
|
|
1173
|
+
this.buffer += this.code.slice(this.pos, match.index);
|
|
799
1174
|
|
|
800
1175
|
let consumed;
|
|
801
|
-
|
|
1176
|
+
const kind = this.matchKind;
|
|
1177
|
+
if (kind === "illegal") {
|
|
802
1178
|
this.aborted = true;
|
|
803
1179
|
return;
|
|
804
|
-
} else if (
|
|
805
|
-
//
|
|
1180
|
+
} else if (kind === "begin") {
|
|
1181
|
+
// nextMatch() only leaves null data for "illegal"; "begin" always
|
|
806
1182
|
// carries the numeric rule index.
|
|
807
|
-
consumed = this.doBegin(
|
|
808
|
-
found.match,
|
|
809
|
-
/** @type {number} */ (found.data),
|
|
810
|
-
);
|
|
1183
|
+
consumed = this.doBegin(match, /** @type {number} */ (this.matchData));
|
|
811
1184
|
} else {
|
|
812
1185
|
// Same invariant as above: "end" always carries the numeric depth.
|
|
813
|
-
consumed = this.doEnd(
|
|
1186
|
+
consumed = this.doEnd(match, /** @type {number} */ (this.matchData));
|
|
814
1187
|
}
|
|
815
1188
|
|
|
816
|
-
let next =
|
|
1189
|
+
let next = match.index + consumed;
|
|
817
1190
|
if (
|
|
818
1191
|
next === this.pos &&
|
|
819
1192
|
this.frames.length === framesBefore &&
|
|
@@ -852,11 +1225,14 @@ class Tokenizer {
|
|
|
852
1225
|
* @returns {Snapshot}
|
|
853
1226
|
*/
|
|
854
1227
|
snapshot() {
|
|
1228
|
+
// Shared, not copied: the next keyword hit copies it (see
|
|
1229
|
+
// `kwHitsShared`). Callers treat snapshots as read-only.
|
|
1230
|
+
this.kwHitsShared = true;
|
|
855
1231
|
return {
|
|
856
1232
|
pos: this.pos,
|
|
857
1233
|
buffer: this.buffer,
|
|
858
1234
|
relevance: this.relevance,
|
|
859
|
-
kwHits:
|
|
1235
|
+
kwHits: this.kwHits,
|
|
860
1236
|
frames: this.frames.map((f) => ({
|
|
861
1237
|
idx: f.idx,
|
|
862
1238
|
beginMatch: f.beginMatch,
|
|
@@ -885,7 +1261,9 @@ class Tokenizer {
|
|
|
885
1261
|
this.pos = snap.pos;
|
|
886
1262
|
this.buffer = snap.buffer;
|
|
887
1263
|
this.relevance = snap.relevance;
|
|
888
|
-
|
|
1264
|
+
// Shared until the next keyword hit, like `snapshot()`.
|
|
1265
|
+
this.kwHits = snap.kwHits;
|
|
1266
|
+
this.kwHitsShared = true;
|
|
889
1267
|
this.frames = snap.frames.map((f) => ({
|
|
890
1268
|
idx: f.idx,
|
|
891
1269
|
// f.idx was captured from this same program's frames by snapshot(),
|
|
@@ -924,18 +1302,66 @@ export function scopeToCssClass(name, prefix) {
|
|
|
924
1302
|
return `${prefix}${name}`;
|
|
925
1303
|
}
|
|
926
1304
|
|
|
1305
|
+
/**
|
|
1306
|
+
* Built `<span class="...">` open tags, by class prefix and then scope.
|
|
1307
|
+
* Building one per OPEN event (`scopeToCssClass` plus a template string)
|
|
1308
|
+
* cost about a quarter of renderHtml's time, though a document only has a
|
|
1309
|
+
* few dozen distinct scopes (see bench/render.bench.ts). The vocabulary is
|
|
1310
|
+
* bounded by the registered grammars, but `OPEN_TAG_LIMIT` caps it anyway
|
|
1311
|
+
* so arbitrary class prefixes can't grow it without bound.
|
|
1312
|
+
* @type {Map<string, Map<string, string>>}
|
|
1313
|
+
*/
|
|
1314
|
+
const openTagCache = new Map();
|
|
1315
|
+
const OPEN_TAG_LIMIT = 2048;
|
|
1316
|
+
let openTagCount = 0;
|
|
1317
|
+
|
|
1318
|
+
/**
|
|
1319
|
+
* The open-tag cache for `prefix`, to pass to `openTag`.
|
|
1320
|
+
* @param {string} prefix
|
|
1321
|
+
* @returns {Map<string, string>}
|
|
1322
|
+
*/
|
|
1323
|
+
function openTagsFor(prefix) {
|
|
1324
|
+
let tags = openTagCache.get(prefix);
|
|
1325
|
+
if (tags === undefined) {
|
|
1326
|
+
tags = new Map();
|
|
1327
|
+
openTagCache.set(prefix, tags);
|
|
1328
|
+
}
|
|
1329
|
+
return tags;
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
/**
|
|
1333
|
+
* `<span class="...">` for `scope`, memoized in `tags` (from `openTagsFor`).
|
|
1334
|
+
* @param {Map<string, string>} tags
|
|
1335
|
+
* @param {string} scope
|
|
1336
|
+
* @param {string} prefix
|
|
1337
|
+
* @returns {string}
|
|
1338
|
+
*/
|
|
1339
|
+
function openTag(tags, scope, prefix) {
|
|
1340
|
+
let tag = tags.get(scope);
|
|
1341
|
+
if (tag === undefined) {
|
|
1342
|
+
if (openTagCount >= OPEN_TAG_LIMIT) {
|
|
1343
|
+
for (const cached of openTagCache.values()) cached.clear();
|
|
1344
|
+
openTagCount = 0;
|
|
1345
|
+
}
|
|
1346
|
+
tag = `<span class="${scopeToCssClass(scope, prefix)}">`;
|
|
1347
|
+
tags.set(scope, tag);
|
|
1348
|
+
openTagCount++;
|
|
1349
|
+
}
|
|
1350
|
+
return tag;
|
|
1351
|
+
}
|
|
1352
|
+
|
|
927
1353
|
/**
|
|
928
1354
|
* @param {ScopeEvent[]} events
|
|
929
1355
|
* @param {{ classPrefix?: string }} [options]
|
|
930
1356
|
* @returns {string} hljs-compatible HTML
|
|
931
1357
|
*/
|
|
932
1358
|
export function renderHtml(events, { classPrefix = "hljs-" } = {}) {
|
|
1359
|
+
const tags = openTagsFor(classPrefix);
|
|
933
1360
|
let out = "";
|
|
934
1361
|
for (const event of events) {
|
|
935
1362
|
if (event.t === TEXT) out += escapeHtml(event.v);
|
|
936
|
-
else if (event.t === OPEN)
|
|
937
|
-
|
|
938
|
-
} else out += "</span>";
|
|
1363
|
+
else if (event.t === OPEN) out += openTag(tags, event.s, classPrefix);
|
|
1364
|
+
else out += "</span>";
|
|
939
1365
|
}
|
|
940
1366
|
return out;
|
|
941
1367
|
}
|
|
@@ -960,20 +1386,19 @@ export function extendLines(
|
|
|
960
1386
|
const stack = [...openScopes];
|
|
961
1387
|
/** @type {string[]} */
|
|
962
1388
|
const completedLines = [];
|
|
1389
|
+
const openTags = openTagsFor(classPrefix);
|
|
963
1390
|
// Cache the concatenated open/close tag strings for the current stack so a
|
|
964
1391
|
// run of newlines between OPEN/CLOSE events reuses them instead of
|
|
965
1392
|
// re-joining the whole stack per line break.
|
|
966
1393
|
/** @type {string[]} */
|
|
967
|
-
const tags = stack.map(
|
|
968
|
-
(scope) => `<span class="${scopeToCssClass(scope, classPrefix)}">`,
|
|
969
|
-
);
|
|
1394
|
+
const tags = stack.map((scope) => openTag(openTags, scope, classPrefix));
|
|
970
1395
|
let reopen = tags.join("");
|
|
971
1396
|
let closeAll = "</span>".repeat(stack.length);
|
|
972
1397
|
let current = pendingHtml;
|
|
973
1398
|
for (const event of newEvents) {
|
|
974
1399
|
if (event.t === OPEN) {
|
|
975
1400
|
stack.push(event.s);
|
|
976
|
-
const tag =
|
|
1401
|
+
const tag = openTag(openTags, event.s, classPrefix);
|
|
977
1402
|
tags.push(tag);
|
|
978
1403
|
reopen += tag;
|
|
979
1404
|
closeAll += "</span>";
|
|
@@ -1345,8 +1770,11 @@ export function createRegistry() {
|
|
|
1345
1770
|
* resumes at `snapshot` when given (otherwise from the start). Used by
|
|
1346
1771
|
* incremental-tokenize.js, which knows the whole document and steps
|
|
1347
1772
|
* through it with `advance()`; further `append()` calls still extend it.
|
|
1773
|
+
* `from.windowed` bounds each pattern scan to a window that grows past
|
|
1774
|
+
* the cursor (see `Tokenizer#windowed`), for callers that expect to
|
|
1775
|
+
* stop soon, like a reparse after an edit. Output is unchanged.
|
|
1348
1776
|
* @param {string} language
|
|
1349
|
-
* @param {{ from?: { code: string, snapshot?: Snapshot } }} [options]
|
|
1777
|
+
* @param {{ from?: { code: string, snapshot?: Snapshot, windowed?: boolean } }} [options]
|
|
1350
1778
|
* @returns {StreamSession}
|
|
1351
1779
|
*/
|
|
1352
1780
|
createSession(language, { from } = {}) {
|
|
@@ -1355,10 +1783,18 @@ export function createRegistry() {
|
|
|
1355
1783
|
let tokenizer = new Tokenizer(this, program);
|
|
1356
1784
|
const registry = this;
|
|
1357
1785
|
let staged = "";
|
|
1786
|
+
/**
|
|
1787
|
+
* Index of the last `\n` in `staged`, or -1. Kept up to date so
|
|
1788
|
+
* `append()` only searches the new text: a `staged.lastIndexOf()`
|
|
1789
|
+
* rescanned the whole open line on every append, O(n^2) over a long
|
|
1790
|
+
* single-line stream (see bench/stream-long-line.bench.ts).
|
|
1791
|
+
*/
|
|
1792
|
+
let stagedNewline = -1;
|
|
1358
1793
|
let fed = from ? from.code : "";
|
|
1359
1794
|
if (from) {
|
|
1360
1795
|
tokenizer.code = from.code;
|
|
1361
1796
|
if (from.snapshot) tokenizer.restore(from.snapshot);
|
|
1797
|
+
if (from.windowed) tokenizer.windowed = true;
|
|
1362
1798
|
}
|
|
1363
1799
|
/**
|
|
1364
1800
|
* Lazily built on the first `replace()` call; stays null for sessions
|
|
@@ -1370,11 +1806,13 @@ export function createRegistry() {
|
|
|
1370
1806
|
/** @param {string} text */
|
|
1371
1807
|
append(text) {
|
|
1372
1808
|
fed += text;
|
|
1809
|
+
const textNewline = text.lastIndexOf("\n");
|
|
1810
|
+
if (textNewline >= 0) stagedNewline = staged.length + textNewline;
|
|
1373
1811
|
staged += text;
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1812
|
+
if (stagedNewline >= 0) {
|
|
1813
|
+
tokenizer.code += staged.slice(0, stagedNewline + 1);
|
|
1814
|
+
staged = staged.slice(stagedNewline + 1);
|
|
1815
|
+
stagedNewline = -1;
|
|
1378
1816
|
tokenizer.run();
|
|
1379
1817
|
}
|
|
1380
1818
|
},
|
|
@@ -1388,6 +1826,25 @@ export function createRegistry() {
|
|
|
1388
1826
|
advance(stopAt) {
|
|
1389
1827
|
tokenizer.run(stopAt);
|
|
1390
1828
|
},
|
|
1829
|
+
/**
|
|
1830
|
+
* `append(text.slice(fedLength))`, given `text` (everything fed so
|
|
1831
|
+
* far plus more) as a slice of one larger string. `append()` builds
|
|
1832
|
+
* the grown text by concatenation, which the engine re-flattens
|
|
1833
|
+
* before the next regex scan: feeding a long document in batches
|
|
1834
|
+
* that way cost O(batches x length). Slicing shares the source
|
|
1835
|
+
* string's storage instead (see bench/tokenized-document.bench.ts).
|
|
1836
|
+
* @param {string} text
|
|
1837
|
+
*/
|
|
1838
|
+
feed(text) {
|
|
1839
|
+
fed = text;
|
|
1840
|
+
const lineEnd = text.lastIndexOf("\n") + 1;
|
|
1841
|
+
staged = text.slice(lineEnd);
|
|
1842
|
+
stagedNewline = -1;
|
|
1843
|
+
if (lineEnd > tokenizer.code.length) {
|
|
1844
|
+
tokenizer.code = text.slice(0, lineEnd);
|
|
1845
|
+
tokenizer.run();
|
|
1846
|
+
}
|
|
1847
|
+
},
|
|
1391
1848
|
/**
|
|
1392
1849
|
* Replaces the fed text's `[from, to)` range with `text`, as if the
|
|
1393
1850
|
* session had been fed the resulting string from the start.
|
|
@@ -1397,11 +1854,16 @@ export function createRegistry() {
|
|
|
1397
1854
|
* reusing the unaffected suffix when tokenizer state reconverges.
|
|
1398
1855
|
* Does not support a `replace()` across a language change; create a
|
|
1399
1856
|
* new session for that instead.
|
|
1857
|
+
*
|
|
1858
|
+
* Returns how many leading `events()` entries are the same objects
|
|
1859
|
+
* as before the call. The first call re-parses, so it returns 0.
|
|
1400
1860
|
* @param {number} from
|
|
1401
1861
|
* @param {number} to
|
|
1402
1862
|
* @param {string} text
|
|
1863
|
+
* @returns {number}
|
|
1403
1864
|
*/
|
|
1404
1865
|
replace(from, to, text) {
|
|
1866
|
+
const first = !incremental;
|
|
1405
1867
|
if (!incremental) {
|
|
1406
1868
|
incremental = parseIncremental(
|
|
1407
1869
|
/** @type {Registry} */ (registry),
|
|
@@ -1409,11 +1871,12 @@ export function createRegistry() {
|
|
|
1409
1871
|
fed,
|
|
1410
1872
|
);
|
|
1411
1873
|
}
|
|
1874
|
+
const previous = incremental;
|
|
1412
1875
|
const newCode = fed.slice(0, from) + text + fed.slice(to);
|
|
1413
1876
|
incremental = reparseIncremental(
|
|
1414
1877
|
/** @type {Registry} */ (registry),
|
|
1415
1878
|
language,
|
|
1416
|
-
|
|
1879
|
+
previous,
|
|
1417
1880
|
newCode,
|
|
1418
1881
|
);
|
|
1419
1882
|
fed = newCode;
|
|
@@ -1432,12 +1895,80 @@ export function createRegistry() {
|
|
|
1432
1895
|
// the events up to the checkpoint's own count, then run the live
|
|
1433
1896
|
// tokenizer once - exactly what `append()` does for newly fed
|
|
1434
1897
|
// complete lines - to resolve it the same way.
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
)
|
|
1898
|
+
//
|
|
1899
|
+
// The events are truncated in place rather than copied, which
|
|
1900
|
+
// copied every event before the change a second time (see
|
|
1901
|
+
// bench/stream-repaint.bench.ts). When the code changed,
|
|
1902
|
+
// `reparseIncremental` returns a fresh array, so the events
|
|
1903
|
+
// `events()` returned before this call stay as they were. When it
|
|
1904
|
+
// didn't, it returns `previous` itself, whose events are the array
|
|
1905
|
+
// `events()` and any `finish()` result handed out, so that one is
|
|
1906
|
+
// copied instead. Either way `incremental.events` may hold events
|
|
1907
|
+
// past the last checkpoint, which the next re-parse doesn't keep:
|
|
1908
|
+
// it resumes at or before that checkpoint, and whatever it carries
|
|
1909
|
+
// over past it is cut off here again.
|
|
1910
|
+
tokenizer.events =
|
|
1911
|
+
incremental === previous
|
|
1912
|
+
? incremental.events.slice(0, lastCheckpoint.eventCount)
|
|
1913
|
+
: incremental.events;
|
|
1914
|
+
tokenizer.events.length = lastCheckpoint.eventCount;
|
|
1439
1915
|
tokenizer.run();
|
|
1440
1916
|
staged = fed.slice(tokenizer.pos);
|
|
1917
|
+
stagedNewline = staged.lastIndexOf("\n");
|
|
1918
|
+
if (first) return 0;
|
|
1919
|
+
if (incremental === previous) return lastCheckpoint.eventCount;
|
|
1920
|
+
return incremental.reuse?.head ?? 0;
|
|
1921
|
+
},
|
|
1922
|
+
/**
|
|
1923
|
+
* The last of the last `replace()`'s checkpoints (none before the
|
|
1924
|
+
* first) within both limits: a point where rendering can resume,
|
|
1925
|
+
* since `events()[0, eventCount)` covers the text before `textPos`
|
|
1926
|
+
* and leaves `scopes` open. Lets stream-regenerate.js skip its walk
|
|
1927
|
+
* over the events before a change.
|
|
1928
|
+
*
|
|
1929
|
+
* At a checkpoint the open scopes are the scoped frames' (begin
|
|
1930
|
+
* and end matches open and close them, and keyword and
|
|
1931
|
+
* sub-language runs are balanced), and the text not yet emitted is
|
|
1932
|
+
* `buffer`, which ends at `pos`. Checkpoints only move forward, so
|
|
1933
|
+
* this is a binary search.
|
|
1934
|
+
* @param {{ eventCount?: number, textPos?: number }} limits
|
|
1935
|
+
* @returns {{ textPos: number, eventCount: number, scopes: string[] } | undefined}
|
|
1936
|
+
*/
|
|
1937
|
+
checkpointBefore({
|
|
1938
|
+
eventCount = Number.POSITIVE_INFINITY,
|
|
1939
|
+
textPos = Number.POSITIVE_INFINITY,
|
|
1940
|
+
}) {
|
|
1941
|
+
const list = incremental ? incremental.checkpoints : [];
|
|
1942
|
+
let lo = 0;
|
|
1943
|
+
let hi = list.length - 1;
|
|
1944
|
+
/** @type {Snapshot | undefined} */
|
|
1945
|
+
let found;
|
|
1946
|
+
while (lo <= hi) {
|
|
1947
|
+
const mid = (lo + hi) >> 1;
|
|
1948
|
+
const checkpoint = /** @type {Snapshot} */ (list[mid]);
|
|
1949
|
+
if (
|
|
1950
|
+
checkpoint.eventCount <= eventCount &&
|
|
1951
|
+
checkpoint.pos - checkpoint.buffer.length <= textPos
|
|
1952
|
+
) {
|
|
1953
|
+
found = checkpoint;
|
|
1954
|
+
lo = mid + 1;
|
|
1955
|
+
} else {
|
|
1956
|
+
hi = mid - 1;
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
if (!found) return undefined;
|
|
1960
|
+
/** @type {string[]} */
|
|
1961
|
+
const scopes = [];
|
|
1962
|
+
for (const frame of found.frames) {
|
|
1963
|
+
const scope =
|
|
1964
|
+
frame.idx === 0 ? undefined : program.states[frame.idx]?.scope;
|
|
1965
|
+
if (scope) scopes.push(scope);
|
|
1966
|
+
}
|
|
1967
|
+
return {
|
|
1968
|
+
textPos: found.pos - found.buffer.length,
|
|
1969
|
+
eventCount: found.eventCount,
|
|
1970
|
+
scopes,
|
|
1971
|
+
};
|
|
1441
1972
|
},
|
|
1442
1973
|
/**
|
|
1443
1974
|
* Multi-line lookahead (e.g. ruby heredocs) may need the full text
|
|
@@ -1449,17 +1980,49 @@ export function createRegistry() {
|
|
|
1449
1980
|
if (canonicalize) return registry.highlight(fed, { language });
|
|
1450
1981
|
tokenizer.code += staged;
|
|
1451
1982
|
staged = "";
|
|
1983
|
+
stagedNewline = -1;
|
|
1452
1984
|
tokenizer.run();
|
|
1453
1985
|
const result = tokenizer.finish();
|
|
1986
|
+
const events = result.events;
|
|
1987
|
+
// `value` renders on first read: the live preview
|
|
1988
|
+
// (stream-preview.js) and incremental-tokenize.js only read
|
|
1989
|
+
// `events`, and rendering the whole document's HTML they then
|
|
1990
|
+
// dropped was wasted work (see bench/stream-repaint.bench.ts,
|
|
1991
|
+
// bench/incremental.bench.ts). A later append() pushes onto
|
|
1992
|
+
// this same array, so render only what finish() returned.
|
|
1993
|
+
const eventCount = events.length;
|
|
1994
|
+
/** @type {string | undefined} */
|
|
1995
|
+
let value;
|
|
1454
1996
|
return {
|
|
1455
1997
|
language,
|
|
1456
1998
|
relevance: result.relevance,
|
|
1457
|
-
events
|
|
1458
|
-
value
|
|
1999
|
+
events,
|
|
2000
|
+
get value() {
|
|
2001
|
+
value ??= renderHtml(
|
|
2002
|
+
events.length === eventCount
|
|
2003
|
+
? events
|
|
2004
|
+
: events.slice(0, eventCount),
|
|
2005
|
+
);
|
|
2006
|
+
return value;
|
|
2007
|
+
},
|
|
1459
2008
|
};
|
|
1460
2009
|
},
|
|
1461
2010
|
snapshot: () => tokenizer.snapshot(),
|
|
1462
2011
|
events: () => tokenizer.events,
|
|
2012
|
+
/**
|
|
2013
|
+
* Returns the events produced since the last call and lets the
|
|
2014
|
+
* session forget them, so a long session doesn't hold every event
|
|
2015
|
+
* it ever made (~140 MB per 100k lines of TypeScript). For callers
|
|
2016
|
+
* that consume events once, like TokenizedDocument. Afterward,
|
|
2017
|
+
* `events()`, `finish()`, and snapshot `eventCount`s only cover
|
|
2018
|
+
* later events, so don't mix it with `replace()`.
|
|
2019
|
+
* @returns {ScopeEvent[]}
|
|
2020
|
+
*/
|
|
2021
|
+
takeEvents() {
|
|
2022
|
+
const taken = tokenizer.events;
|
|
2023
|
+
tokenizer.events = [];
|
|
2024
|
+
return taken;
|
|
2025
|
+
},
|
|
1463
2026
|
};
|
|
1464
2027
|
},
|
|
1465
2028
|
|