svelte-highlight 7.22.0 → 7.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/engine.js CHANGED
@@ -69,7 +69,23 @@ import {
69
69
  * earliest guard-passing match at or after the position `this.pos` had when
70
70
  * it was computed (or `null` if none exists in `this.code` as of `codeLen`
71
71
  * characters). See `Tokenizer#nextMatch`'s cache-hit checks.
72
- * @typedef {{ codeLen: number, match: RegExpExecArray | null }} MatchCache
72
+ *
73
+ * A windowed tokenizer (see `Tokenizer#windowed`) also sets `until` and
74
+ * `level`. With `match: null`, `until` is the end of the window it scanned:
75
+ * no guard-passing match starts before it. `until` is `Infinity` once a scan
76
+ * reached the end of the code. `level` picks the window size for the next
77
+ * scan (see `scanWindowed`).
78
+ * @typedef {{
79
+ * codeLen: number,
80
+ * match: RegExpExecArray | null,
81
+ * until?: number,
82
+ * level?: number,
83
+ * }} MatchCache
84
+ */
85
+
86
+ /**
87
+ * A windowed tokenizer's `MatchCache`, which always has `until`/`level`.
88
+ * @typedef {Required<MatchCache>} WindowCache
73
89
  */
74
90
 
75
91
  /**
@@ -109,7 +125,10 @@ import {
109
125
  */
110
126
 
111
127
  /**
112
- * @typedef {{ match: RegExpExecArray, kind: "begin" | "end" | "illegal", data: number | null }} MatchResult
128
+ * Which kind of pattern produced `nextMatch`'s winner. Its `matchData` is
129
+ * the rule's state index for "begin", the frame depth for "end", and null
130
+ * for "illegal".
131
+ * @typedef {"begin" | "end" | "illegal"} MatchKind
113
132
  */
114
133
 
115
134
  const MAX_KEYWORD_HITS = 7;
@@ -122,6 +141,54 @@ const MAX_KEYWORD_HITS = 7;
122
141
  */
123
142
  const DETECT_SAMPLE_LIMIT = 8000;
124
143
 
144
+ /**
145
+ * Window sizes for `Tokenizer#scanWindowed`: `WINDOW_BASE << level` match
146
+ * start positions, for `level` in `[0, WINDOW_LEVELS)`. Past the last level
147
+ * the scan runs to the end of the code, as an unwindowed scan does. A
148
+ * window scan reads more slowly per character than a plain `exec` (which
149
+ * can skip ahead for a literal first character), so the windows stay small
150
+ * and an edit that never re-converges soon falls back to plain scans (see
151
+ * bench/incremental.bench.ts).
152
+ */
153
+ const WINDOW_BASE = 256;
154
+ const WINDOW_LEVELS = 5;
155
+
156
+ /**
157
+ * Window regexes by source regex and level (see `windowRegExp`).
158
+ * @type {WeakMap<RegExp, RegExp[]>}
159
+ */
160
+ const windowRegExps = new WeakMap();
161
+
162
+ /**
163
+ * A sticky regex that, run at `lastIndex = from`, matches the shortest run
164
+ * of up to `(WINDOW_BASE << level) - 1` characters after which `re` matches.
165
+ * So `from + match[0].length` is the first position in the window where an
166
+ * attempt of `re` succeeds, with the same result an unbounded `re.exec`
167
+ * from `from` would find there: the lookahead sees the whole code, so
168
+ * anchors, `\b`, and lookarounds behave exactly as in `re`. The prefix is
169
+ * non-capturing, so `re`'s group numbers and backreferences don't shift.
170
+ * @param {RegExp} re
171
+ * @param {number} level
172
+ * @returns {RegExp}
173
+ */
174
+ function windowRegExp(re, level) {
175
+ let byLevel = windowRegExps.get(re);
176
+ if (byLevel === undefined) {
177
+ byLevel = [];
178
+ windowRegExps.set(re, byLevel);
179
+ }
180
+ let windowRe = byLevel[level];
181
+ if (windowRe === undefined) {
182
+ const span = (WINDOW_BASE << level) - 1;
183
+ windowRe = new RegExp(
184
+ `[\\s\\S]{0,${span}}?(?=(?:${re.source}))`,
185
+ `${re.flags.replace("g", "")}y`,
186
+ );
187
+ byLevel[level] = windowRe;
188
+ }
189
+ return windowRe;
190
+ }
191
+
125
192
  // Tokenizer#isTrulyOpeningTag lookaheads.
126
193
  const XML_TAG_DEFAULT_PARAM_RE = /^\s*=/;
127
194
  const XML_TAG_EXTENDS_CONSTRAINT_RE = /^\s+extends\s+/;
@@ -150,8 +217,11 @@ export class UnknownLanguageError extends Error {
150
217
  }
151
218
 
152
219
  /**
153
- * Thrown by `Tokenizer#run` once a parse exceeds 500,000 iterations,
154
- * guarding against a grammar bug that never advances `pos`.
220
+ * Thrown by `Tokenizer#run` once a parse exceeds both 500,000 iterations
221
+ * and three iterations per character consumed, guarding against a grammar
222
+ * bug that never advances `pos`. The per-character term keeps large inputs
223
+ * legal: real grammars take well under one iteration per character, and a
224
+ * flat cap rejected plain JS past ~80k lines.
155
225
  */
156
226
  export class TokenizerLoopError extends Error {
157
227
  /**
@@ -169,26 +239,64 @@ export class TokenizerLoopError extends Error {
169
239
  /** Scope name prefix marking a sub-language boundary (`language:css`). */
170
240
  const LANGUAGE_SCOPE_PREFIX = "language:";
171
241
 
172
- const HTML_ESCAPE_RE = /[&<>"']/g;
173
- /** @type {Record<string, string>} */
174
- const HTML_ESCAPE_MAP = {
175
- "&": "&amp;",
176
- "<": "&lt;",
177
- ">": "&gt;",
178
- '"': "&quot;",
179
- "'": "&#x27;",
180
- };
181
-
182
242
  /**
183
243
  * Five sequential `.replace()` passes (one per character) previously cost
184
244
  * ~87% of renderHtml's time, since every TEXT event runs through this. A
185
- * single regex pass, skipped entirely when there's nothing to escape, cuts
186
- * that to a fraction (see bench/render.bench.ts).
245
+ * single pass that returns `value` untouched when there's nothing to escape
246
+ * cuts that to a fraction. It scans char codes rather than calling a regex:
247
+ * most TEXT values are a few characters long, where a regex call's fixed
248
+ * cost dominated (see bench/render.bench.ts).
187
249
  * @param {string} value
188
250
  */
189
251
  export function escapeHtml(value) {
190
- if (!HTML_ESCAPE_RE.test(value)) return value;
191
- return value.replace(HTML_ESCAPE_RE, (char) => HTML_ESCAPE_MAP[char] ?? char);
252
+ for (let i = 0; i < value.length; i++) {
253
+ const code = value.charCodeAt(i);
254
+ // `>` (62) is the highest code point that needs escaping.
255
+ if (code <= 62 && htmlEscape(code) !== null)
256
+ return escapeHtmlFrom(value, i);
257
+ }
258
+ return value;
259
+ }
260
+
261
+ /**
262
+ * The entity for an escaped char code, or null if it needs no escaping.
263
+ * @param {number} code
264
+ * @returns {string | null}
265
+ */
266
+ function htmlEscape(code) {
267
+ switch (code) {
268
+ case 38:
269
+ return "&amp;";
270
+ case 60:
271
+ return "&lt;";
272
+ case 62:
273
+ return "&gt;";
274
+ case 34:
275
+ return "&quot;";
276
+ case 39:
277
+ return "&#x27;";
278
+ default:
279
+ return null;
280
+ }
281
+ }
282
+
283
+ /**
284
+ * `escapeHtml`'s slow path, from the first index `start` that needs escaping.
285
+ * @param {string} value
286
+ * @param {number} start
287
+ */
288
+ function escapeHtmlFrom(value, start) {
289
+ let out = value.slice(0, start);
290
+ let last = start;
291
+ for (let i = start; i < value.length; i++) {
292
+ const code = value.charCodeAt(i);
293
+ if (code > 62) continue;
294
+ const entity = htmlEscape(code);
295
+ if (entity === null) continue;
296
+ out += value.slice(last, i) + entity;
297
+ last = i + 1;
298
+ }
299
+ return out + value.slice(last);
192
300
  }
193
301
 
194
302
  /**
@@ -236,6 +344,14 @@ class Tokenizer {
236
344
  this.openScopes = 0;
237
345
  /** @type {Record<string, number>} */
238
346
  this.kwHits = Object.create(null);
347
+ /**
348
+ * Whether `kwHits` is shared with a snapshot, so the next keyword hit
349
+ * must copy it before writing. `snapshot()` and `restore()` copied it
350
+ * on every call, about 9% of a streaming repaint, though most frames
351
+ * never hit a keyword before the next snapshot (see
352
+ * bench/stream-repaint.bench.ts).
353
+ */
354
+ this.kwHitsShared = false;
239
355
  /** @type {Frame[]} */
240
356
  this.frames = [
241
357
  {
@@ -249,6 +365,16 @@ class Tokenizer {
249
365
  ];
250
366
  this.aborted = false;
251
367
  this.iterations = 0;
368
+ /**
369
+ * Whether begin/end scans search a bounded window past the cursor
370
+ * instead of running to the end of the code (see `nextMatchWindowed`).
371
+ * Set for a parse resumed after an edit, which usually stops a few
372
+ * lines later: a rule with no match nearby otherwise scanned the whole
373
+ * tail of the document, so a one-character edit cost time in
374
+ * proportion to the document (see bench/incremental.bench.ts). Never
375
+ * set in detect mode, so the windowed path ignores `illegal`.
376
+ */
377
+ this.windowed = false;
252
378
  /**
253
379
  * Embedded-language parse state, carried across segments by name - but
254
380
  * only for the *same* embedding occurrence (matched by `beginPos`, the
@@ -269,6 +395,16 @@ class Tokenizer {
269
395
  * @type {(MatchCache | undefined)[]}
270
396
  */
271
397
  this.beginCache = [];
398
+ /**
399
+ * Memoized `illegal`-pattern scans (detect mode only), indexed by state
400
+ * index. Like `beginCache`, a state's next illegal match doesn't depend
401
+ * on the frame. Uncached, every lexeme re-ran the pattern from `pos`,
402
+ * and a pattern with no match left (the common case for a candidate
403
+ * that won't abort) scanned the whole remaining sample each time:
404
+ * quadratic in sample length (see bench/auto-detect.bench.ts).
405
+ * @type {(MatchCache | undefined)[]}
406
+ */
407
+ this.illegalCache = [];
272
408
  /**
273
409
  * Tag names with no `</name` at or after the recorded position. See
274
410
  * `hasClosingTag`.
@@ -276,6 +412,10 @@ class Tokenizer {
276
412
  */
277
413
  this.noClosingTag = new Map();
278
414
  this.noClosingTagCodeLen = 0;
415
+ /** Kind of the last `nextMatch` winner. @type {MatchKind} */
416
+ this.matchKind = "begin";
417
+ /** Data of the last `nextMatch` winner. @type {number | null} */
418
+ this.matchData = null;
279
419
  }
280
420
 
281
421
  /** @returns {Frame} */
@@ -338,6 +478,12 @@ class Tokenizer {
338
478
  }
339
479
 
340
480
  /**
481
+ * Splits `text` into keyword and plain-text events. Plain runs are sliced
482
+ * straight out of `text` rather than rebuilt by `+=` per word, which made
483
+ * every non-keyword word a string concatenation (see
484
+ * bench/tokenize.bench.ts). A keyword hit always ends the current TEXT
485
+ * event, even a relevance-only (`_`-prefixed) one, which stays in the
486
+ * next plain run.
341
487
  * @param {string} text
342
488
  * @param {CompiledState} state
343
489
  */
@@ -346,42 +492,44 @@ class Tokenizer {
346
492
  this.text(text);
347
493
  return;
348
494
  }
495
+ if (text === "") return;
349
496
  const keywords = state.keywords;
350
497
  // keywordRe is always compiled alongside keywords (see compileProgram),
351
498
  // so it is non-null whenever `keywords` is set.
352
499
  const keywordRe = /** @type {RegExp} */ (state.keywordRe);
353
- let lastIndex = 0;
354
- let buf = "";
500
+ const caseInsensitive = this.program.ir.caseInsensitive;
501
+ let kwHits = this.kwHits;
502
+ let textStart = 0;
355
503
  keywordRe.lastIndex = 0;
356
504
  let match = keywordRe.exec(text);
357
505
  while (match) {
358
- buf += text.substring(lastIndex, match.index);
359
- const word = this.program.ir.caseInsensitive
360
- ? match[0].toLowerCase()
361
- : match[0];
506
+ const lexeme = match[0];
507
+ const word = caseInsensitive ? lexeme.toLowerCase() : lexeme;
362
508
  const data = keywords[word];
363
509
  if (data) {
364
510
  const [kind, keywordRelevance] = data;
365
- this.text(buf);
366
- buf = "";
367
- this.kwHits[word] = (this.kwHits[word] || 0) + 1;
368
- if (this.kwHits[word] <= MAX_KEYWORD_HITS) {
369
- this.relevance += keywordRelevance;
511
+ this.text(text.substring(textStart, match.index));
512
+ if (this.kwHitsShared) {
513
+ // Copied into a null-prototype object before any read, so a
514
+ // restored plain-object snapshot can't leak `constructor` etc.
515
+ kwHits = Object.assign(Object.create(null), kwHits);
516
+ this.kwHits = kwHits;
517
+ this.kwHitsShared = false;
370
518
  }
371
- if (kind.startsWith("_")) {
372
- // relevance-only keyword, not highlighted
373
- buf += match[0];
519
+ const hits = (kwHits[word] || 0) + 1;
520
+ kwHits[word] = hits;
521
+ if (hits <= MAX_KEYWORD_HITS) this.relevance += keywordRelevance;
522
+ if (kind.charCodeAt(0) === 95) {
523
+ // "_": relevance-only keyword, not highlighted
524
+ textStart = match.index;
374
525
  } else {
375
- this.emitKeyword(match[0], kind);
526
+ this.emitKeyword(lexeme, kind);
527
+ textStart = keywordRe.lastIndex;
376
528
  }
377
- } else {
378
- buf += match[0];
379
529
  }
380
- lastIndex = keywordRe.lastIndex;
381
530
  match = keywordRe.exec(text);
382
531
  }
383
- buf += text.substring(lastIndex);
384
- this.text(buf);
532
+ this.text(textStart === 0 ? text : text.substring(textStart));
385
533
  }
386
534
 
387
535
  /** @param {CompiledState} state */
@@ -478,8 +626,11 @@ class Tokenizer {
478
626
  * boundary re-scans every rule's pattern from scratch. A cached match stays
479
627
  * valid until `this.pos` passes it; a cached miss stays valid until the
480
628
  * code grows (streaming append).
629
+ * Returns a plain boolean, not a `cache is MatchCache` predicate, so a
630
+ * stale cache isn't narrowed to `undefined` where `nextMatch` refreshes
631
+ * it in place.
481
632
  * @param {MatchCache | undefined} cache
482
- * @returns {cache is MatchCache}
633
+ * @returns {boolean}
483
634
  */
484
635
  isCacheValid(cache) {
485
636
  if (!cache) return false;
@@ -573,103 +724,325 @@ class Tokenizer {
573
724
 
574
725
  /**
575
726
  * The next lexeme: earliest match wins; ties break by priority (begin
576
- * rules in order, then ends innermost-first, then illegal).
577
- * @returns {MatchResult | null}
727
+ * rules in order, then ends innermost-first, then illegal). Returns the
728
+ * winning match and leaves its kind/data in `matchKind`/`matchData`.
729
+ *
730
+ * Runs once per lexeme, so it allocates nothing on cache hits: no
731
+ * `consider()` closure over mutable locals, no `{ match, kind, data }`
732
+ * result object, and stale caches are refreshed in place (see
733
+ * bench/tokenize.bench.ts).
734
+ * @returns {RegExpExecArray | null}
578
735
  */
579
736
  nextMatch() {
580
737
  /** @type {RegExpExecArray | null} */
581
738
  let best = null;
582
- let bestPri = Number.POSITIVE_INFINITY;
583
- /** @type {"begin" | "end" | "illegal" | ""} */
584
- let bestKind = "";
739
+ let bestIndex = Number.POSITIVE_INFINITY;
740
+ /** @type {MatchKind} */
741
+ let bestKind = "begin";
585
742
  /** @type {number | null} */
586
743
  let bestData = null;
587
- /**
588
- * @param {RegExpExecArray | null} match
589
- * @param {number} pri
590
- * @param {"begin" | "end" | "illegal"} kind
591
- * @param {number | null} data
592
- */
593
- const consider = (match, pri, kind, data) => {
594
- if (!match) return;
595
- if (
596
- best === null ||
597
- match.index < best.index ||
598
- (match.index === best.index && pri < bestPri)
599
- ) {
600
- best = match;
601
- bestPri = pri;
602
- bestKind = kind;
603
- bestData = data;
604
- }
605
- };
606
744
 
607
745
  const state = this.top.state;
608
- for (let i = 0; i < state.rules.length; i++) {
746
+ const rules = state.rules;
747
+ const states = this.program.states;
748
+ // Begin rules come first and in priority order, so among them a strict
749
+ // `<` keeps the earliest rule on ties.
750
+ for (let i = 0; i < rules.length; i++) {
609
751
  // rules[i] is in-bounds by the loop condition.
610
- const ruleIdx = /** @type {number} */ (state.rules[i]);
611
- const child = /** @type {CompiledState} */ (this.program.states[ruleIdx]);
752
+ const ruleIdx = /** @type {number} */ (rules[i]);
612
753
  let cache = this.beginCache[ruleIdx];
613
754
  // Guard closures (this.beginGuard(child)) are only built on a cache
614
755
  // miss - passing them in as a pre-built thunk, as this used to via a
615
756
  // shared cachedMatch() helper, allocated one on every call regardless
616
757
  // of hit/miss, since JS evaluates argument expressions eagerly.
617
758
  if (!this.isCacheValid(cache)) {
618
- cache = {
619
- codeLen: this.code.length,
620
- match: this.execValid(
621
- // Every state referenced from a `rules` list has a `begin` pattern.
622
- /** @type {RegExp} */ (child.beginRe),
623
- this.pos,
624
- this.beginGuard(child),
625
- ),
626
- };
627
- this.beginCache[ruleIdx] = cache;
759
+ const child = /** @type {CompiledState} */ (states[ruleIdx]);
760
+ const match = this.execValid(
761
+ // Every state referenced from a `rules` list has a `begin` pattern.
762
+ /** @type {RegExp} */ (child.beginRe),
763
+ this.pos,
764
+ this.beginGuard(child),
765
+ );
766
+ if (cache) {
767
+ cache.codeLen = this.code.length;
768
+ cache.match = match;
769
+ } else {
770
+ cache = { codeLen: this.code.length, match };
771
+ this.beginCache[ruleIdx] = cache;
772
+ }
773
+ }
774
+ const match = /** @type {MatchCache} */ (cache).match;
775
+ if (match !== null && match.index < bestIndex) {
776
+ best = match;
777
+ bestIndex = match.index;
778
+ bestData = ruleIdx;
628
779
  }
629
- consider(cache.match, i, "begin", ruleIdx);
630
780
  }
631
- // End candidates walk outward while `endsWithParent` chains allow.
632
- let pri = state.rules.length;
781
+ // End candidates walk outward while `endsWithParent` chains allow. They
782
+ // rank below every begin rule and innermost-first, so they also only win
783
+ // with a strictly earlier index.
633
784
  for (let d = this.frames.length - 1; d >= 1; d--) {
634
785
  // d ranges over [1, frames.length - 1], always in bounds.
635
786
  const frame = /** @type {Frame} */ (this.frames[d]);
636
- if (frame.state.endRe) {
787
+ const endRe = frame.state.endRe;
788
+ if (endRe) {
637
789
  let cache = frame.endCache;
638
790
  if (!this.isCacheValid(cache)) {
639
- cache = {
640
- codeLen: this.code.length,
641
- match: this.execValid(
642
- /** @type {RegExp} */ (frame.state.endRe),
643
- this.pos,
644
- frame.state.endSameAsBegin
645
- ? (/** @type {RegExpExecArray} */ m) =>
646
- m[1] === frame.beginMatch
647
- : null,
648
- ),
649
- };
650
- frame.endCache = cache;
791
+ const match = this.execValid(
792
+ endRe,
793
+ this.pos,
794
+ frame.state.endSameAsBegin
795
+ ? (/** @type {RegExpExecArray} */ m) => m[1] === frame.beginMatch
796
+ : null,
797
+ );
798
+ if (cache) {
799
+ cache.codeLen = this.code.length;
800
+ cache.match = match;
801
+ } else {
802
+ cache = { codeLen: this.code.length, match };
803
+ frame.endCache = cache;
804
+ }
805
+ }
806
+ const match = /** @type {MatchCache} */ (cache).match;
807
+ if (match !== null && match.index < bestIndex) {
808
+ best = match;
809
+ bestIndex = match.index;
810
+ bestKind = "end";
811
+ bestData = d;
651
812
  }
652
- consider(cache.match, pri++, "end", d);
653
813
  }
654
814
  if (!frame.state.endsWithParent) break;
655
815
  }
656
816
  if (this.detect && state.illegalRe) {
657
- consider(
658
- this.execValid(state.illegalRe, this.pos, null),
659
- pri + 1,
660
- "illegal",
661
- null,
662
- );
817
+ const stateIdx = this.top.idx;
818
+ let cache = this.illegalCache[stateIdx];
819
+ if (!this.isCacheValid(cache)) {
820
+ const match = this.execValid(state.illegalRe, this.pos, null);
821
+ if (cache) {
822
+ cache.codeLen = this.code.length;
823
+ cache.match = match;
824
+ } else {
825
+ cache = { codeLen: this.code.length, match };
826
+ this.illegalCache[stateIdx] = cache;
827
+ }
828
+ }
829
+ const match = /** @type {MatchCache} */ (cache).match;
830
+ if (match !== null && match.index < bestIndex) {
831
+ best = match;
832
+ bestKind = "illegal";
833
+ bestData = null;
834
+ }
835
+ }
836
+ this.matchKind = bestKind;
837
+ this.matchData = bestData;
838
+ return best;
839
+ }
840
+
841
+ /**
842
+ * The end-pattern guard for `frame` (see `nextMatch`).
843
+ * @param {Frame} frame
844
+ * @returns {((m: RegExpExecArray) => boolean) | null}
845
+ */
846
+ endGuard(frame) {
847
+ return frame.state.endSameAsBegin ? (m) => m[1] === frame.beginMatch : null;
848
+ }
849
+
850
+ /**
851
+ * Whether a windowed cache still answers "first match at or after
852
+ * `pos`": a match `pos` hasn't passed, or a miss whose window reaches
853
+ * past `pos` in unchanged code.
854
+ * @param {WindowCache} cache
855
+ * @returns {boolean}
856
+ */
857
+ isWindowValid(cache) {
858
+ if (cache.match !== null) return cache.match.index >= this.pos;
859
+ return cache.codeLen === this.code.length && cache.until > this.pos;
860
+ }
861
+
862
+ /**
863
+ * `execValid` over a window of `WINDOW_BASE << cache.level` start
864
+ * positions from `from`, written into `cache`: the first guard-passing
865
+ * match in the window, or a miss with `until` at the window's end. A
866
+ * window that reaches the end of the code, or a level past the last,
867
+ * gives the same answer as `execValid` (`until` is `Infinity`). Matches
868
+ * come from `re.exec` itself, so they're the same objects an unwindowed
869
+ * scan returns (see `windowRegExp`).
870
+ * @param {WindowCache} cache
871
+ * @param {RegExp} re
872
+ * @param {number} from
873
+ * @param {((m: RegExpExecArray) => boolean) | null} guard
874
+ */
875
+ scanWindowed(cache, re, from, guard) {
876
+ const code = this.code;
877
+ cache.codeLen = code.length;
878
+ cache.until = Number.POSITIVE_INFINITY;
879
+ const size = WINDOW_BASE << cache.level;
880
+ // A window reaching the end of the code saves nothing, and a plain
881
+ // `exec` scans faster: near the end of a document (typing at the end,
882
+ // say) every scan is plain. Under the `u` flag a window could start
883
+ // inside a surrogate pair, where the window regex and `re` might step
884
+ // differently; scan plainly there too.
885
+ if (
886
+ cache.level >= WINDOW_LEVELS ||
887
+ from + size > code.length ||
888
+ re.unicode
889
+ ) {
890
+ cache.match = this.execValid(re, from, guard);
891
+ return;
892
+ }
893
+ const windowRe = windowRegExp(re, cache.level);
894
+ let start = from;
895
+ for (;;) {
896
+ windowRe.lastIndex = start;
897
+ const found = windowRe.exec(code);
898
+ if (found === null) {
899
+ cache.match = null;
900
+ // The window covered every start up to `code.length` inclusive.
901
+ if (start + size <= code.length) cache.until = start + size;
902
+ return;
903
+ }
904
+ const at = start + found[0].length;
905
+ re.lastIndex = at;
906
+ const match = re.exec(code);
907
+ if (match === null || match.index !== at) {
908
+ // Unreachable: the lookahead just matched `re` at `at`. Kept so a
909
+ // regex engine quirk can't change output, only cost.
910
+ cache.match = this.execValid(re, from, guard);
911
+ return;
912
+ }
913
+ if (!guard || guard(match)) {
914
+ cache.match = match;
915
+ return;
916
+ }
917
+ if (at >= code.length) {
918
+ cache.match = null;
919
+ return;
920
+ }
921
+ start = at + 1;
922
+ }
923
+ }
924
+
925
+ /**
926
+ * `nextMatch` for a windowed tokenizer (see `windowed`). A cached miss
927
+ * only rules out matches before its `until`, so its window is widened
928
+ * (twice the size, from `until`) while the rule could still beat the
929
+ * best match so far, then the candidates are compared again. Windows
930
+ * never widen past `stopAt`: `run` stops before any lexeme at or past
931
+ * it, so when nothing can start earlier this returns a match at or past
932
+ * `stopAt` (not necessarily the earliest) or null. Detect mode never
933
+ * runs windowed, so `illegal` is not scanned.
934
+ * @param {number} stopAt
935
+ * @returns {RegExpExecArray | null}
936
+ */
937
+ nextMatchWindowed(stopAt) {
938
+ const rules = this.top.state.rules;
939
+ const states = this.program.states;
940
+ const frames = this.frames;
941
+ const beginCache = /** @type {(WindowCache | undefined)[]} */ (
942
+ this.beginCache
943
+ );
944
+ for (;;) {
945
+ /** @type {RegExpExecArray | null} */
946
+ let best = null;
947
+ let bestIndex = Number.POSITIVE_INFINITY;
948
+ /** @type {MatchKind} */
949
+ let bestKind = "begin";
950
+ /** @type {number | null} */
951
+ let bestData = null;
952
+ let minUntil = Number.POSITIVE_INFINITY;
953
+ for (let i = 0; i < rules.length; i++) {
954
+ const ruleIdx = /** @type {number} */ (rules[i]);
955
+ let cache = beginCache[ruleIdx];
956
+ if (cache === undefined) {
957
+ cache = { codeLen: -1, match: null, until: 0, level: 0 };
958
+ beginCache[ruleIdx] = cache;
959
+ }
960
+ if (!this.isWindowValid(cache)) {
961
+ // A miss whose window `pos` has passed: widen the next one.
962
+ if (cache.match === null && cache.codeLen === this.code.length) {
963
+ cache.level++;
964
+ }
965
+ const child = /** @type {CompiledState} */ (states[ruleIdx]);
966
+ this.scanWindowed(
967
+ cache,
968
+ /** @type {RegExp} */ (child.beginRe),
969
+ this.pos,
970
+ this.beginGuard(child),
971
+ );
972
+ }
973
+ const match = cache.match;
974
+ if (match === null) {
975
+ if (cache.until < minUntil) minUntil = cache.until;
976
+ } else if (match.index < bestIndex) {
977
+ best = match;
978
+ bestIndex = match.index;
979
+ bestData = ruleIdx;
980
+ }
981
+ }
982
+ for (let d = frames.length - 1; d >= 1; d--) {
983
+ const frame = /** @type {Frame} */ (frames[d]);
984
+ const endRe = frame.state.endRe;
985
+ if (endRe) {
986
+ let cache = /** @type {WindowCache | undefined} */ (frame.endCache);
987
+ if (cache === undefined) {
988
+ cache = { codeLen: -1, match: null, until: 0, level: 0 };
989
+ frame.endCache = cache;
990
+ }
991
+ if (!this.isWindowValid(cache)) {
992
+ if (cache.match === null && cache.codeLen === this.code.length) {
993
+ cache.level++;
994
+ }
995
+ this.scanWindowed(cache, endRe, this.pos, this.endGuard(frame));
996
+ }
997
+ const match = cache.match;
998
+ if (match === null) {
999
+ if (cache.until < minUntil) minUntil = cache.until;
1000
+ } else if (match.index < bestIndex) {
1001
+ best = match;
1002
+ bestIndex = match.index;
1003
+ bestKind = "end";
1004
+ bestData = d;
1005
+ }
1006
+ }
1007
+ if (!frame.state.endsWithParent) break;
1008
+ }
1009
+ // Every unresolved rule starts strictly after `limit`, so it can
1010
+ // neither beat nor tie the best match, nor start before `stopAt`.
1011
+ // (`minUntil` stays Infinity when every rule is resolved.)
1012
+ const limit = Math.min(bestIndex, stopAt - 1);
1013
+ if (minUntil > limit || minUntil === Number.POSITIVE_INFINITY) {
1014
+ this.matchKind = bestKind;
1015
+ this.matchData = bestData;
1016
+ return best;
1017
+ }
1018
+ for (let i = 0; i < rules.length; i++) {
1019
+ const ruleIdx = /** @type {number} */ (rules[i]);
1020
+ const cache = /** @type {WindowCache} */ (beginCache[ruleIdx]);
1021
+ if (cache.match !== null || cache.until > limit) continue;
1022
+ const child = /** @type {CompiledState} */ (states[ruleIdx]);
1023
+ cache.level++;
1024
+ this.scanWindowed(
1025
+ cache,
1026
+ /** @type {RegExp} */ (child.beginRe),
1027
+ cache.until,
1028
+ this.beginGuard(child),
1029
+ );
1030
+ }
1031
+ for (let d = frames.length - 1; d >= 1; d--) {
1032
+ const frame = /** @type {Frame} */ (frames[d]);
1033
+ const cache = /** @type {WindowCache | undefined} */ (frame.endCache);
1034
+ if (cache && cache.match === null && cache.until <= limit) {
1035
+ cache.level++;
1036
+ this.scanWindowed(
1037
+ cache,
1038
+ /** @type {RegExp} */ (frame.state.endRe),
1039
+ cache.until,
1040
+ this.endGuard(frame),
1041
+ );
1042
+ }
1043
+ if (!frame.state.endsWithParent) break;
1044
+ }
663
1045
  }
664
- return best === null
665
- ? null
666
- : {
667
- match: best,
668
- // consider() only assigns bestKind alongside best, so bestKind is
669
- // never "" once best is non-null.
670
- kind: /** @type {"begin" | "end" | "illegal"} */ (bestKind),
671
- data: bestData,
672
- };
673
1046
  }
674
1047
 
675
1048
  // --- state transitions ---
@@ -703,7 +1076,7 @@ class Tokenizer {
703
1076
  * @returns {number}
704
1077
  */
705
1078
  doBegin(match, idx) {
706
- // idx comes from nextMatch's "begin" data, which is always a valid
1079
+ // idx comes from nextMatch's "begin" matchData, which is always a valid
707
1080
  // program.states index (see the ruleIdx comment above).
708
1081
  const state = /** @type {CompiledState} */ (this.program.states[idx]);
709
1082
  const lexeme = match[0];
@@ -786,34 +1159,34 @@ class Tokenizer {
786
1159
  if (this.aborted) return;
787
1160
  for (;;) {
788
1161
  this.iterations++;
789
- if (this.iterations > 500000) {
1162
+ if (this.iterations > 500000 && this.iterations > this.pos * 3) {
790
1163
  throw new TokenizerLoopError(this.program.ir.name, this.iterations);
791
1164
  }
792
- const found = this.nextMatch();
793
- if (!found) break;
794
- if (found.match.index >= stopAt) return;
1165
+ const match = this.windowed
1166
+ ? this.nextMatchWindowed(stopAt)
1167
+ : this.nextMatch();
1168
+ if (!match) break;
1169
+ if (match.index >= stopAt) return;
795
1170
 
796
1171
  const framesBefore = this.frames.length;
797
1172
  const topBefore = this.top;
798
- this.buffer += this.code.slice(this.pos, found.match.index);
1173
+ this.buffer += this.code.slice(this.pos, match.index);
799
1174
 
800
1175
  let consumed;
801
- if (found.kind === "illegal") {
1176
+ const kind = this.matchKind;
1177
+ if (kind === "illegal") {
802
1178
  this.aborted = true;
803
1179
  return;
804
- } else if (found.kind === "begin") {
805
- // consider() only passes null data for "illegal"; "begin" always
1180
+ } else if (kind === "begin") {
1181
+ // nextMatch() only leaves null data for "illegal"; "begin" always
806
1182
  // carries the numeric rule index.
807
- consumed = this.doBegin(
808
- found.match,
809
- /** @type {number} */ (found.data),
810
- );
1183
+ consumed = this.doBegin(match, /** @type {number} */ (this.matchData));
811
1184
  } else {
812
1185
  // Same invariant as above: "end" always carries the numeric depth.
813
- consumed = this.doEnd(found.match, /** @type {number} */ (found.data));
1186
+ consumed = this.doEnd(match, /** @type {number} */ (this.matchData));
814
1187
  }
815
1188
 
816
- let next = found.match.index + consumed;
1189
+ let next = match.index + consumed;
817
1190
  if (
818
1191
  next === this.pos &&
819
1192
  this.frames.length === framesBefore &&
@@ -852,11 +1225,14 @@ class Tokenizer {
852
1225
  * @returns {Snapshot}
853
1226
  */
854
1227
  snapshot() {
1228
+ // Shared, not copied: the next keyword hit copies it (see
1229
+ // `kwHitsShared`). Callers treat snapshots as read-only.
1230
+ this.kwHitsShared = true;
855
1231
  return {
856
1232
  pos: this.pos,
857
1233
  buffer: this.buffer,
858
1234
  relevance: this.relevance,
859
- kwHits: { ...this.kwHits },
1235
+ kwHits: this.kwHits,
860
1236
  frames: this.frames.map((f) => ({
861
1237
  idx: f.idx,
862
1238
  beginMatch: f.beginMatch,
@@ -885,7 +1261,9 @@ class Tokenizer {
885
1261
  this.pos = snap.pos;
886
1262
  this.buffer = snap.buffer;
887
1263
  this.relevance = snap.relevance;
888
- this.kwHits = Object.assign(Object.create(null), snap.kwHits);
1264
+ // Shared until the next keyword hit, like `snapshot()`.
1265
+ this.kwHits = snap.kwHits;
1266
+ this.kwHitsShared = true;
889
1267
  this.frames = snap.frames.map((f) => ({
890
1268
  idx: f.idx,
891
1269
  // f.idx was captured from this same program's frames by snapshot(),
@@ -924,18 +1302,66 @@ export function scopeToCssClass(name, prefix) {
924
1302
  return `${prefix}${name}`;
925
1303
  }
926
1304
 
1305
+ /**
1306
+ * Built `<span class="...">` open tags, by class prefix and then scope.
1307
+ * Building one per OPEN event (`scopeToCssClass` plus a template string)
1308
+ * cost about a quarter of renderHtml's time, though a document only has a
1309
+ * few dozen distinct scopes (see bench/render.bench.ts). The vocabulary is
1310
+ * bounded by the registered grammars, but `OPEN_TAG_LIMIT` caps it anyway
1311
+ * so arbitrary class prefixes can't grow it without bound.
1312
+ * @type {Map<string, Map<string, string>>}
1313
+ */
1314
+ const openTagCache = new Map();
1315
+ const OPEN_TAG_LIMIT = 2048;
1316
+ let openTagCount = 0;
1317
+
1318
+ /**
1319
+ * The open-tag cache for `prefix`, to pass to `openTag`.
1320
+ * @param {string} prefix
1321
+ * @returns {Map<string, string>}
1322
+ */
1323
+ function openTagsFor(prefix) {
1324
+ let tags = openTagCache.get(prefix);
1325
+ if (tags === undefined) {
1326
+ tags = new Map();
1327
+ openTagCache.set(prefix, tags);
1328
+ }
1329
+ return tags;
1330
+ }
1331
+
1332
+ /**
1333
+ * `<span class="...">` for `scope`, memoized in `tags` (from `openTagsFor`).
1334
+ * @param {Map<string, string>} tags
1335
+ * @param {string} scope
1336
+ * @param {string} prefix
1337
+ * @returns {string}
1338
+ */
1339
+ function openTag(tags, scope, prefix) {
1340
+ let tag = tags.get(scope);
1341
+ if (tag === undefined) {
1342
+ if (openTagCount >= OPEN_TAG_LIMIT) {
1343
+ for (const cached of openTagCache.values()) cached.clear();
1344
+ openTagCount = 0;
1345
+ }
1346
+ tag = `<span class="${scopeToCssClass(scope, prefix)}">`;
1347
+ tags.set(scope, tag);
1348
+ openTagCount++;
1349
+ }
1350
+ return tag;
1351
+ }
1352
+
927
1353
  /**
928
1354
  * @param {ScopeEvent[]} events
929
1355
  * @param {{ classPrefix?: string }} [options]
930
1356
  * @returns {string} hljs-compatible HTML
931
1357
  */
932
1358
  export function renderHtml(events, { classPrefix = "hljs-" } = {}) {
1359
+ const tags = openTagsFor(classPrefix);
933
1360
  let out = "";
934
1361
  for (const event of events) {
935
1362
  if (event.t === TEXT) out += escapeHtml(event.v);
936
- else if (event.t === OPEN) {
937
- out += `<span class="${scopeToCssClass(event.s, classPrefix)}">`;
938
- } else out += "</span>";
1363
+ else if (event.t === OPEN) out += openTag(tags, event.s, classPrefix);
1364
+ else out += "</span>";
939
1365
  }
940
1366
  return out;
941
1367
  }
@@ -960,20 +1386,19 @@ export function extendLines(
960
1386
  const stack = [...openScopes];
961
1387
  /** @type {string[]} */
962
1388
  const completedLines = [];
1389
+ const openTags = openTagsFor(classPrefix);
963
1390
  // Cache the concatenated open/close tag strings for the current stack so a
964
1391
  // run of newlines between OPEN/CLOSE events reuses them instead of
965
1392
  // re-joining the whole stack per line break.
966
1393
  /** @type {string[]} */
967
- const tags = stack.map(
968
- (scope) => `<span class="${scopeToCssClass(scope, classPrefix)}">`,
969
- );
1394
+ const tags = stack.map((scope) => openTag(openTags, scope, classPrefix));
970
1395
  let reopen = tags.join("");
971
1396
  let closeAll = "</span>".repeat(stack.length);
972
1397
  let current = pendingHtml;
973
1398
  for (const event of newEvents) {
974
1399
  if (event.t === OPEN) {
975
1400
  stack.push(event.s);
976
- const tag = `<span class="${scopeToCssClass(event.s, classPrefix)}">`;
1401
+ const tag = openTag(openTags, event.s, classPrefix);
977
1402
  tags.push(tag);
978
1403
  reopen += tag;
979
1404
  closeAll += "</span>";
@@ -1345,8 +1770,11 @@ export function createRegistry() {
1345
1770
  * resumes at `snapshot` when given (otherwise from the start). Used by
1346
1771
  * incremental-tokenize.js, which knows the whole document and steps
1347
1772
  * through it with `advance()`; further `append()` calls still extend it.
1773
+ * `from.windowed` bounds each pattern scan to a window that grows past
1774
+ * the cursor (see `Tokenizer#windowed`), for callers that expect to
1775
+ * stop soon, like a reparse after an edit. Output is unchanged.
1348
1776
  * @param {string} language
1349
- * @param {{ from?: { code: string, snapshot?: Snapshot } }} [options]
1777
+ * @param {{ from?: { code: string, snapshot?: Snapshot, windowed?: boolean } }} [options]
1350
1778
  * @returns {StreamSession}
1351
1779
  */
1352
1780
  createSession(language, { from } = {}) {
@@ -1355,10 +1783,18 @@ export function createRegistry() {
1355
1783
  let tokenizer = new Tokenizer(this, program);
1356
1784
  const registry = this;
1357
1785
  let staged = "";
1786
+ /**
1787
+ * Index of the last `\n` in `staged`, or -1. Kept up to date so
1788
+ * `append()` only searches the new text: a `staged.lastIndexOf()`
1789
+ * rescanned the whole open line on every append, O(n^2) over a long
1790
+ * single-line stream (see bench/stream-long-line.bench.ts).
1791
+ */
1792
+ let stagedNewline = -1;
1358
1793
  let fed = from ? from.code : "";
1359
1794
  if (from) {
1360
1795
  tokenizer.code = from.code;
1361
1796
  if (from.snapshot) tokenizer.restore(from.snapshot);
1797
+ if (from.windowed) tokenizer.windowed = true;
1362
1798
  }
1363
1799
  /**
1364
1800
  * Lazily built on the first `replace()` call; stays null for sessions
@@ -1370,11 +1806,13 @@ export function createRegistry() {
1370
1806
  /** @param {string} text */
1371
1807
  append(text) {
1372
1808
  fed += text;
1809
+ const textNewline = text.lastIndexOf("\n");
1810
+ if (textNewline >= 0) stagedNewline = staged.length + textNewline;
1373
1811
  staged += text;
1374
- const newline = staged.lastIndexOf("\n");
1375
- if (newline >= 0) {
1376
- tokenizer.code += staged.slice(0, newline + 1);
1377
- staged = staged.slice(newline + 1);
1812
+ if (stagedNewline >= 0) {
1813
+ tokenizer.code += staged.slice(0, stagedNewline + 1);
1814
+ staged = staged.slice(stagedNewline + 1);
1815
+ stagedNewline = -1;
1378
1816
  tokenizer.run();
1379
1817
  }
1380
1818
  },
@@ -1388,6 +1826,25 @@ export function createRegistry() {
1388
1826
  advance(stopAt) {
1389
1827
  tokenizer.run(stopAt);
1390
1828
  },
1829
+ /**
1830
+ * `append(text.slice(fedLength))`, given `text` (everything fed so
1831
+ * far plus more) as a slice of one larger string. `append()` builds
1832
+ * the grown text by concatenation, which the engine re-flattens
1833
+ * before the next regex scan: feeding a long document in batches
1834
+ * that way cost O(batches x length). Slicing shares the source
1835
+ * string's storage instead (see bench/tokenized-document.bench.ts).
1836
+ * @param {string} text
1837
+ */
1838
+ feed(text) {
1839
+ fed = text;
1840
+ const lineEnd = text.lastIndexOf("\n") + 1;
1841
+ staged = text.slice(lineEnd);
1842
+ stagedNewline = -1;
1843
+ if (lineEnd > tokenizer.code.length) {
1844
+ tokenizer.code = text.slice(0, lineEnd);
1845
+ tokenizer.run();
1846
+ }
1847
+ },
1391
1848
  /**
1392
1849
  * Replaces the fed text's `[from, to)` range with `text`, as if the
1393
1850
  * session had been fed the resulting string from the start.
@@ -1397,11 +1854,16 @@ export function createRegistry() {
1397
1854
  * reusing the unaffected suffix when tokenizer state reconverges.
1398
1855
  * Does not support a `replace()` across a language change; create a
1399
1856
  * new session for that instead.
1857
+ *
1858
+ * Returns how many leading `events()` entries are the same objects
1859
+ * as before the call. The first call re-parses, so it returns 0.
1400
1860
  * @param {number} from
1401
1861
  * @param {number} to
1402
1862
  * @param {string} text
1863
+ * @returns {number}
1403
1864
  */
1404
1865
  replace(from, to, text) {
1866
+ const first = !incremental;
1405
1867
  if (!incremental) {
1406
1868
  incremental = parseIncremental(
1407
1869
  /** @type {Registry} */ (registry),
@@ -1409,11 +1871,12 @@ export function createRegistry() {
1409
1871
  fed,
1410
1872
  );
1411
1873
  }
1874
+ const previous = incremental;
1412
1875
  const newCode = fed.slice(0, from) + text + fed.slice(to);
1413
1876
  incremental = reparseIncremental(
1414
1877
  /** @type {Registry} */ (registry),
1415
1878
  language,
1416
- incremental,
1879
+ previous,
1417
1880
  newCode,
1418
1881
  );
1419
1882
  fed = newCode;
@@ -1432,12 +1895,80 @@ export function createRegistry() {
1432
1895
  // the events up to the checkpoint's own count, then run the live
1433
1896
  // tokenizer once - exactly what `append()` does for newly fed
1434
1897
  // complete lines - to resolve it the same way.
1435
- tokenizer.events = incremental.events.slice(
1436
- 0,
1437
- lastCheckpoint.eventCount,
1438
- );
1898
+ //
1899
+ // The events are truncated in place rather than copied, which
1900
+ // copied every event before the change a second time (see
1901
+ // bench/stream-repaint.bench.ts). When the code changed,
1902
+ // `reparseIncremental` returns a fresh array, so the events
1903
+ // `events()` returned before this call stay as they were. When it
1904
+ // didn't, it returns `previous` itself, whose events are the array
1905
+ // `events()` and any `finish()` result handed out, so that one is
1906
+ // copied instead. Either way `incremental.events` may hold events
1907
+ // past the last checkpoint, which the next re-parse doesn't keep:
1908
+ // it resumes at or before that checkpoint, and whatever it carries
1909
+ // over past it is cut off here again.
1910
+ tokenizer.events =
1911
+ incremental === previous
1912
+ ? incremental.events.slice(0, lastCheckpoint.eventCount)
1913
+ : incremental.events;
1914
+ tokenizer.events.length = lastCheckpoint.eventCount;
1439
1915
  tokenizer.run();
1440
1916
  staged = fed.slice(tokenizer.pos);
1917
+ stagedNewline = staged.lastIndexOf("\n");
1918
+ if (first) return 0;
1919
+ if (incremental === previous) return lastCheckpoint.eventCount;
1920
+ return incremental.reuse?.head ?? 0;
1921
+ },
1922
+ /**
1923
+ * The last of the last `replace()`'s checkpoints (none before the
1924
+ * first) within both limits: a point where rendering can resume,
1925
+ * since `events()[0, eventCount)` covers the text before `textPos`
1926
+ * and leaves `scopes` open. Lets stream-regenerate.js skip its walk
1927
+ * over the events before a change.
1928
+ *
1929
+ * At a checkpoint the open scopes are the scoped frames' (begin
1930
+ * and end matches open and close them, and keyword and
1931
+ * sub-language runs are balanced), and the text not yet emitted is
1932
+ * `buffer`, which ends at `pos`. Checkpoints only move forward, so
1933
+ * this is a binary search.
1934
+ * @param {{ eventCount?: number, textPos?: number }} limits
1935
+ * @returns {{ textPos: number, eventCount: number, scopes: string[] } | undefined}
1936
+ */
1937
+ checkpointBefore({
1938
+ eventCount = Number.POSITIVE_INFINITY,
1939
+ textPos = Number.POSITIVE_INFINITY,
1940
+ }) {
1941
+ const list = incremental ? incremental.checkpoints : [];
1942
+ let lo = 0;
1943
+ let hi = list.length - 1;
1944
+ /** @type {Snapshot | undefined} */
1945
+ let found;
1946
+ while (lo <= hi) {
1947
+ const mid = (lo + hi) >> 1;
1948
+ const checkpoint = /** @type {Snapshot} */ (list[mid]);
1949
+ if (
1950
+ checkpoint.eventCount <= eventCount &&
1951
+ checkpoint.pos - checkpoint.buffer.length <= textPos
1952
+ ) {
1953
+ found = checkpoint;
1954
+ lo = mid + 1;
1955
+ } else {
1956
+ hi = mid - 1;
1957
+ }
1958
+ }
1959
+ if (!found) return undefined;
1960
+ /** @type {string[]} */
1961
+ const scopes = [];
1962
+ for (const frame of found.frames) {
1963
+ const scope =
1964
+ frame.idx === 0 ? undefined : program.states[frame.idx]?.scope;
1965
+ if (scope) scopes.push(scope);
1966
+ }
1967
+ return {
1968
+ textPos: found.pos - found.buffer.length,
1969
+ eventCount: found.eventCount,
1970
+ scopes,
1971
+ };
1441
1972
  },
1442
1973
  /**
1443
1974
  * Multi-line lookahead (e.g. ruby heredocs) may need the full text
@@ -1449,17 +1980,49 @@ export function createRegistry() {
1449
1980
  if (canonicalize) return registry.highlight(fed, { language });
1450
1981
  tokenizer.code += staged;
1451
1982
  staged = "";
1983
+ stagedNewline = -1;
1452
1984
  tokenizer.run();
1453
1985
  const result = tokenizer.finish();
1986
+ const events = result.events;
1987
+ // `value` renders on first read: the live preview
1988
+ // (stream-preview.js) and incremental-tokenize.js only read
1989
+ // `events`, and rendering the whole document's HTML they then
1990
+ // dropped was wasted work (see bench/stream-repaint.bench.ts,
1991
+ // bench/incremental.bench.ts). A later append() pushes onto
1992
+ // this same array, so render only what finish() returned.
1993
+ const eventCount = events.length;
1994
+ /** @type {string | undefined} */
1995
+ let value;
1454
1996
  return {
1455
1997
  language,
1456
1998
  relevance: result.relevance,
1457
- events: result.events,
1458
- value: renderHtml(result.events),
1999
+ events,
2000
+ get value() {
2001
+ value ??= renderHtml(
2002
+ events.length === eventCount
2003
+ ? events
2004
+ : events.slice(0, eventCount),
2005
+ );
2006
+ return value;
2007
+ },
1459
2008
  };
1460
2009
  },
1461
2010
  snapshot: () => tokenizer.snapshot(),
1462
2011
  events: () => tokenizer.events,
2012
+ /**
2013
+ * Returns the events produced since the last call and lets the
2014
+ * session forget them, so a long session doesn't hold every event
2015
+ * it ever made (~140 MB per 100k lines of TypeScript). For callers
2016
+ * that consume events once, like TokenizedDocument. Afterward,
2017
+ * `events()`, `finish()`, and snapshot `eventCount`s only cover
2018
+ * later events, so don't mix it with `replace()`.
2019
+ * @returns {ScopeEvent[]}
2020
+ */
2021
+ takeEvents() {
2022
+ const taken = tokenizer.events;
2023
+ tokenizer.events = [];
2024
+ return taken;
2025
+ },
1463
2026
  };
1464
2027
  },
1465
2028