svelte-highlight 7.22.0 → 7.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,7 @@
11
11
  * @returns {{
12
12
  * appendLines: (lines: string[]) => void,
13
13
  * reset: () => void,
14
+ * truncate: (lines: number) => void,
14
15
  * toString: () => string,
15
16
  * lineCount: number,
16
17
  * }}
@@ -31,6 +32,22 @@ export function createCompletedHtmlBuffer() {
31
32
  html = "";
32
33
  lineCount = 0;
33
34
  },
35
+ /**
36
+ * Keeps the first `lines` lines. Walks back from the end, so the cost
37
+ * scales with the lines dropped, not the lines kept.
38
+ * @param {number} lines
39
+ */
40
+ truncate(lines) {
41
+ if (lines >= lineCount) return;
42
+ let end = html.length;
43
+ for (let n = lineCount; n > lines; n--) {
44
+ end = html.lastIndexOf("\n", end - 1);
45
+ }
46
+ // With `lines === 0`, the walk passes the first line, which has no
47
+ // "\n" before it, and `end` is -1.
48
+ html = lines === 0 ? "" : html.slice(0, end);
49
+ lineCount = lines;
50
+ },
34
51
  toString() {
35
52
  return html;
36
53
  },
package/stream-preview.js CHANGED
@@ -75,11 +75,16 @@ export function computeStagedTailPreview({
75
75
  // Valid only for a pure append since the checkpoint: the committed session
76
76
  // must not have advanced (no newline completed, so openScopes/pendingHtml
77
77
  // - the checkpoint's base state - are still current) and `fedCode` must
78
- // still start with the code the checkpoint was taken against.
78
+ // still start with the code the checkpoint was taken against. Only the
79
+ // part past the committed position needs comparing: `session` already
80
+ // covers everything before it and must match `fedCode` there anyway. A
81
+ // whole-buffer compare made every call O(stream), not O(line) -
82
+ // bench/stream-repaint.bench.ts.
79
83
  const canResume =
80
84
  cache !== undefined &&
81
85
  cache.committedPos === snapshot.pos &&
82
- fedCode.startsWith(cache.fedCode);
86
+ fedCode.length >= cache.fedCode.length &&
87
+ fedCode.startsWith(cache.fedCode.slice(snapshot.pos), snapshot.pos);
83
88
 
84
89
  const baseSnapshot = canResume ? cache.snapshot : snapshot;
85
90
  const baseOpenScopes = canResume ? cache.openScopes : openScopes;
@@ -0,0 +1,262 @@
1
+ /**
2
+ * HighlightStream's regenerate path: the stream's code changed in a way
3
+ * that isn't a pure append (an LLM "regenerate the last paragraph", say).
4
+ * Patches the session in place, then re-renders only from the sealed chunk
5
+ * the change lands in.
6
+ *
7
+ * A line's HTML depends only on its events and the scopes open at its
8
+ * start. So a sealed chunk that ends before the change, and whose events
9
+ * the patched parse left alone, stays as it is. Only the lines from the
10
+ * chunk the change lands in get rendered again; the lines before it are
11
+ * just walked to compare events and track scopes. Before, every line and
12
+ * every chunk was rebuilt - bench/stream-repaint.bench.ts.
13
+ */
14
+
15
+ import { CLOSE, extendLines, OPEN, TEXT } from "./engine.js";
16
+ import {
17
+ buildSealedChunkHtml,
18
+ pushSealedChunk,
19
+ } from "./stream-sealed-chunks.js";
20
+ import { diffText } from "./text-diff.js";
21
+
22
+ /**
23
+ * @typedef {import("./engine.d.ts").ScopeEvent} ScopeEvent
24
+ * @typedef {import("./engine.d.ts").StreamSession} StreamSession
25
+ * @typedef {ReturnType<typeof import("./stream-highlighted.js").createCompletedHtmlBuffer>} CompletedHtmlBuffer
26
+ * @typedef {NonNullable<ReturnType<StreamSession["checkpointBefore"]>>} Checkpoint
27
+ */
28
+
29
+ /**
30
+ * Where `extendLinesAfterPatch` can start its walk after a `replace()`
31
+ * that kept the first `keptEvents` events (see `StreamSession#replace`):
32
+ * the last checkpoint before the break of the last line start that
33
+ * certainly survives. Those events match the previous ones by identity,
34
+ * so the walk from the top would only have confirmed them. Undefined when
35
+ * no line start is that early.
36
+ *
37
+ * Before, every regenerate walked every event before the change, about a
38
+ * fifth of a regenerate at 8,000 lines (bench/stream-repaint.bench.ts).
39
+ * A session's first `replace()` re-parses and keeps no events, so the
40
+ * first regenerate still walks from the top.
41
+ * @param {StreamSession} session
42
+ * @param {number} keptEvents
43
+ * @param {number[]} lineStarts Ascending.
44
+ * @returns {Checkpoint | undefined}
45
+ */
46
+ export function walkStart(session, keptEvents, lineStarts) {
47
+ // The text the kept events cover ends at the last checkpoint they reach.
48
+ const kept = session.checkpointBefore({ eventCount: keptEvents });
49
+ if (!kept) return undefined;
50
+ // The last line start whose break lies in that text.
51
+ let last = -1;
52
+ while (
53
+ last + 1 < lineStarts.length &&
54
+ /** @type {number} */ (lineStarts[last + 1]) - 1 < kept.textPos
55
+ ) {
56
+ last++;
57
+ }
58
+ if (last < 0) return undefined;
59
+ return session.checkpointBefore({
60
+ textPos: /** @type {number} */ (lineStarts[last]) - 1,
61
+ });
62
+ }
63
+
64
+ /**
65
+ * Offsets of the line starts every `chunkLines` lines in `text`, at most
66
+ * `maxChunks` of them, up to `end`.
67
+ * @param {string} text
68
+ * @param {number} end
69
+ * @param {number} chunkLines
70
+ * @param {number} maxChunks
71
+ * @returns {number[]}
72
+ */
73
+ function chunkStarts(text, end, chunkLines, maxChunks) {
74
+ /** @type {number[]} */
75
+ const starts = [];
76
+ let lines = 0;
77
+ for (
78
+ let i = text.indexOf("\n");
79
+ i !== -1 && i < end && starts.length < maxChunks;
80
+ i = text.indexOf("\n", i + 1)
81
+ ) {
82
+ if (++lines === chunkLines) {
83
+ starts.push(i + 1);
84
+ lines = 0;
85
+ }
86
+ }
87
+ return starts;
88
+ }
89
+
90
+ /**
91
+ * @param {ScopeEvent} a
92
+ * @param {ScopeEvent | undefined} b
93
+ */
94
+ function sameEvent(a, b) {
95
+ if (a === b) return true;
96
+ if (b === undefined || a.t !== b.t) return false;
97
+ if (a.t === TEXT) return a.v === /** @type {{ v: string }} */ (b).v;
98
+ if (a.t === OPEN) return a.s === /** @type {{ s: string }} */ (b).s;
99
+ return true;
100
+ }
101
+
102
+ /**
103
+ * Renders a patched session's `events` from the last of `lineStarts` (text
104
+ * offsets of line starts, ascending) whose earlier events all match
105
+ * `previousEvents`, the events before the patch. Lines before that point
106
+ * render exactly as they did, so only the lines from it on are rendered.
107
+ *
108
+ * Checking the events, not just the text, matters: with multi-line
109
+ * lookahead (a Ruby heredoc, say), the patched parse can tokenize an
110
+ * unchanged line differently from the stream that first rendered it.
111
+ *
112
+ * Returns how many of `lineStarts` are kept (rendering resumes at
113
+ * `lineStarts[kept - 1]`, or at the top if 0) and `extendLines`' result
114
+ * from there.
115
+ *
116
+ * `from`, when given, is a point known to follow only events that match
117
+ * `previousEvents` (see `walkStart`), so the walk starts there instead of
118
+ * at the top, with the same result.
119
+ * @param {ScopeEvent[]} events
120
+ * @param {ScopeEvent[]} previousEvents
121
+ * @param {number[]} lineStarts
122
+ * @param {Checkpoint} [from]
123
+ * @returns {{ kept: number, result: ReturnType<typeof extendLines> }}
124
+ */
125
+ export function extendLinesAfterPatch(
126
+ events,
127
+ previousEvents,
128
+ lineStarts,
129
+ from,
130
+ ) {
131
+ let kept = 0;
132
+ // Where line `lineStarts[kept - 1]` starts: its line break's event, the
133
+ // break's index in it, and the scopes open there.
134
+ let resumeEvent = 0;
135
+ let resumeAt = 0;
136
+ /** @type {string[]} */
137
+ let resumeStack = [];
138
+ /** @type {string[]} */
139
+ const stack = from ? [...from.scopes] : [];
140
+ let pos = from ? from.textPos : 0;
141
+ let recorded = false;
142
+ // Line breaks before `from` are passed; `walkStart` picks `from` before
143
+ // the break of the last line start kept, so the walk still records it.
144
+ while (
145
+ kept < lineStarts.length &&
146
+ /** @type {number} */ (lineStarts[kept]) - 1 < pos
147
+ ) {
148
+ kept++;
149
+ }
150
+ for (
151
+ let i = from ? from.eventCount : 0;
152
+ i < events.length && kept < lineStarts.length;
153
+ i++
154
+ ) {
155
+ const event = /** @type {ScopeEvent} */ (events[i]);
156
+ if (!sameEvent(event, previousEvents[i])) break;
157
+ if (event.t === OPEN) {
158
+ stack.push(event.s);
159
+ } else if (event.t === CLOSE) {
160
+ stack.pop();
161
+ } else {
162
+ const end = pos + event.v.length;
163
+ while (
164
+ kept < lineStarts.length &&
165
+ /** @type {number} */ (lineStarts[kept]) - 1 < end
166
+ ) {
167
+ resumeEvent = i;
168
+ resumeAt = /** @type {number} */ (lineStarts[kept]) - 1 - pos;
169
+ resumeStack = [...stack];
170
+ kept++;
171
+ recorded = true;
172
+ }
173
+ pos = end;
174
+ }
175
+ }
176
+ // Unreachable with `walkStart`'s `from`; kept so a bad one costs a full
177
+ // walk instead of wrong output.
178
+ if (from && kept > 0 && !recorded) {
179
+ return extendLinesAfterPatch(events, previousEvents, lineStarts);
180
+ }
181
+ if (kept === 0) return { kept, result: extendLines(events, [], "") };
182
+
183
+ // Resume from the line break, so extendLines opens the line with the
184
+ // scopes reopened, as it does after every break. The line it completes
185
+ // first is the end of the line before: drop it.
186
+ const text = /** @type {{ v: string }} */ (events[resumeEvent]).v;
187
+ /** @type {ScopeEvent[]} */
188
+ const rest = [{ t: TEXT, v: text.slice(resumeAt) }];
189
+ for (let j = resumeEvent + 1; j < events.length; j++) {
190
+ rest.push(/** @type {ScopeEvent} */ (events[j]));
191
+ }
192
+ const result = extendLines(rest, resumeStack, "");
193
+ result.completedLines = result.completedLines.slice(1);
194
+ return { kept, result };
195
+ }
196
+
197
+ /**
198
+ * @param {{
199
+ * session: StreamSession,
200
+ * fedCode: string,
201
+ * code: string,
202
+ * sealedChunks: string[],
203
+ * completedHtml: CompletedHtmlBuffer,
204
+ * chunkLines: number,
205
+ * }} options `session` was fed exactly `fedCode`; `completedHtml` holds
206
+ * its completed lines, and `sealedChunks` the first of them, in chunks of
207
+ * `chunkLines`. `completedHtml` is updated in place; `sealedChunks` is
208
+ * left as is.
209
+ * @returns {{
210
+ * sealedChunks: string[],
211
+ * sealedLineCount: number,
212
+ * unsealedLines: string[],
213
+ * pendingHtml: string,
214
+ * openScopes: string[],
215
+ * committedCount: number,
216
+ * }}
217
+ */
218
+ export function regenerate({
219
+ session,
220
+ fedCode,
221
+ code,
222
+ sealedChunks,
223
+ completedHtml,
224
+ chunkLines,
225
+ }) {
226
+ const previousEvents = session.events();
227
+ const { start, removed, inserted } = diffText(fedCode, code);
228
+ const keptEvents = session.replace(start, start + removed.length, inserted);
229
+ const events = session.events();
230
+
231
+ // Keep the sealed chunks that end before the changed line, as far as
232
+ // the patched events still match.
233
+ const lineStarts = chunkStarts(code, start, chunkLines, sealedChunks.length);
234
+ const { kept: keptChunks, result } = extendLinesAfterPatch(
235
+ events,
236
+ previousEvents,
237
+ lineStarts,
238
+ keptEvents > 0 ? walkStart(session, keptEvents, lineStarts) : undefined,
239
+ );
240
+
241
+ let sealedLineCount = keptChunks * chunkLines;
242
+ completedHtml.truncate(sealedLineCount);
243
+ completedHtml.appendLines(result.completedLines);
244
+ let chunks = sealedChunks.slice(0, keptChunks);
245
+ let unsealedLines = result.completedLines;
246
+ while (unsealedLines.length >= chunkLines) {
247
+ chunks = pushSealedChunk(
248
+ chunks,
249
+ buildSealedChunkHtml(unsealedLines.slice(0, chunkLines), sealedLineCount),
250
+ );
251
+ sealedLineCount += chunkLines;
252
+ unsealedLines = unsealedLines.slice(chunkLines);
253
+ }
254
+ return {
255
+ sealedChunks: chunks,
256
+ sealedLineCount,
257
+ unsealedLines,
258
+ pendingHtml: result.pendingHtml,
259
+ openScopes: result.openScopes,
260
+ committedCount: events.length,
261
+ };
262
+ }
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Ownership registry for HighlightStyle's injected `<style>` tags, kept out
3
+ * of the component so it can be unit-tested (see
4
+ * tests/style-registry.test.ts).
5
+ *
6
+ * Instances sharing a key queue up in registration order; only the oldest
7
+ * still-registered instance renders the `<style>`. A store (rather than a
8
+ * plain ref count) means that when that instance leaves, the remaining
9
+ * subscribers reactively re-derive a new owner instead of the tag being
10
+ * dropped while siblings still need it.
11
+ */
12
+ import { writable } from "svelte/store";
13
+
14
+ /** @typedef {import("svelte/store").Writable<symbol[]>} Owners */
15
+
16
+ export function createStyleRegistry() {
17
+ /** @type {Map<string, Owners>} */
18
+ const registries = new Map();
19
+
20
+ return {
21
+ /**
22
+ * Adds `token` as the newest owner of `key`.
23
+ * @param {string} key
24
+ * @param {symbol} token
25
+ * @returns {Owners} The key's owner list, to subscribe to.
26
+ */
27
+ register(key, token) {
28
+ let owners = registries.get(key);
29
+ if (!owners) {
30
+ owners = writable([]);
31
+ registries.set(key, owners);
32
+ }
33
+ owners.update((list) => [...list, token]);
34
+ return owners;
35
+ },
36
+
37
+ /**
38
+ * Removes `token` from `owners`, and drops `key` once its last owner
39
+ * leaves. Keys embed the full theme CSS, so a long-lived app that
40
+ * cycles themes would otherwise keep every theme it ever showed.
41
+ * @param {string} key
42
+ * @param {Owners} owners The store `register` returned for `key`.
43
+ * @param {symbol} token
44
+ */
45
+ unregister(key, owners, token) {
46
+ let empty = false;
47
+ owners.update((list) => {
48
+ const remaining = list.filter((owner) => owner !== token);
49
+ empty = remaining.length === 0;
50
+ return remaining;
51
+ });
52
+ if (empty && registries.get(key) === owners) registries.delete(key);
53
+ },
54
+
55
+ /** Number of keys with at least one owner. */
56
+ get size() {
57
+ return registries.size;
58
+ },
59
+ };
60
+ }
package/text-diff.js CHANGED
@@ -2,6 +2,9 @@ const HIGH_SURROGATE_MIN = 0xd800;
2
2
  const HIGH_SURROGATE_MAX = 0xdbff;
3
3
  const LOW_SURROGATE_MIN = 0xdc00;
4
4
  const LOW_SURROGATE_MAX = 0xdfff;
5
+ // Chunk sizes for galloping over a shared prefix/suffix (see commonPrefix).
6
+ const GALLOP_MAX = 4096;
7
+ const GALLOP_MIN = 16;
5
8
 
6
9
  /** @param {number} code */
7
10
  function isHighSurrogate(code) {
@@ -13,6 +16,67 @@ function isLowSurrogate(code) {
13
16
  return code >= LOW_SURROGATE_MIN && code <= LOW_SURROGATE_MAX;
14
17
  }
15
18
 
19
+ /**
20
+ * Length of the shared prefix of `a` and `b`, capped at `max`.
21
+ *
22
+ * The first GALLOP_MIN characters are compared one at a time, so inputs
23
+ * that differ early never pay for a slice. Past that, it gallops in native
24
+ * string compares, from GALLOP_MAX-character chunks down to GALLOP_MIN,
25
+ * instead of one JS iteration per character: the prefix is the whole
26
+ * document on a pure append, the common case while typing. A larger step
27
+ * fails at most once before shrinking, so each level after the first costs
28
+ * O(step). text-diff.bench.ts: ~25x faster on a 50k-character append,
29
+ * ~5x on a 1k one.
30
+ * @param {string} a
31
+ * @param {string} b
32
+ * @param {number} max
33
+ */
34
+ function commonPrefix(a, b, max) {
35
+ // charCodeAt (not bracket indexing), so no 1-character string is
36
+ // allocated per position compared.
37
+ let n = 0;
38
+ while (n < GALLOP_MIN && n < max && a.charCodeAt(n) === b.charCodeAt(n)) n++;
39
+ if (n < GALLOP_MIN) return n;
40
+ for (let step = GALLOP_MAX; step >= GALLOP_MIN; step >>= 2) {
41
+ while (n + step <= max && b.startsWith(a.slice(n, n + step), n)) {
42
+ n += step;
43
+ }
44
+ }
45
+ while (n < max && a.charCodeAt(n) === b.charCodeAt(n)) n++;
46
+ return n;
47
+ }
48
+
49
+ /**
50
+ * Length of the shared suffix of `a` and `b`, capped at `max`. Mirrors
51
+ * `commonPrefix`.
52
+ * @param {string} a
53
+ * @param {string} b
54
+ * @param {number} max
55
+ */
56
+ function commonSuffix(a, b, max) {
57
+ const aEnd = a.length - 1;
58
+ const bEnd = b.length - 1;
59
+ let n = 0;
60
+ while (
61
+ n < GALLOP_MIN &&
62
+ n < max &&
63
+ a.charCodeAt(aEnd - n) === b.charCodeAt(bEnd - n)
64
+ ) {
65
+ n++;
66
+ }
67
+ if (n < GALLOP_MIN) return n;
68
+ for (let step = GALLOP_MAX; step >= GALLOP_MIN; step >>= 2) {
69
+ while (
70
+ n + step <= max &&
71
+ b.endsWith(a.slice(a.length - n - step, a.length - n), b.length - n)
72
+ ) {
73
+ n += step;
74
+ }
75
+ }
76
+ while (n < max && a.charCodeAt(aEnd - n) === b.charCodeAt(bEnd - n)) n++;
77
+ return n;
78
+ }
79
+
16
80
  /**
17
81
  * Diffs two strings down to a common-prefix/suffix trim. Trims on Unicode
18
82
  * code points (never splitting a surrogate pair between the shared prefix
@@ -24,26 +88,10 @@ function isLowSurrogate(code) {
24
88
  export function diffText(before, after) {
25
89
  const minLength = Math.min(before.length, after.length);
26
90
 
27
- // charCodeAt (not bracket indexing) so this loop - which walks the entire
28
- // unchanged prefix on a pure-append edit, the common case while typing -
29
- // doesn't allocate a 1-character string per position compared.
30
- let prefix = 0;
31
- while (
32
- prefix < minLength &&
33
- before.charCodeAt(prefix) === after.charCodeAt(prefix)
34
- )
35
- prefix++;
91
+ let prefix = commonPrefix(before, after, minLength);
36
92
  if (isHighSurrogate(before.charCodeAt(prefix - 1))) prefix--;
37
93
 
38
- let suffix = 0;
39
- const maxSuffix = minLength - prefix;
40
- while (
41
- suffix < maxSuffix &&
42
- before.charCodeAt(before.length - 1 - suffix) ===
43
- after.charCodeAt(after.length - 1 - suffix)
44
- ) {
45
- suffix++;
46
- }
94
+ let suffix = commonSuffix(before, after, minLength - prefix);
47
95
  if (isLowSurrogate(before.charCodeAt(before.length - suffix))) suffix--;
48
96
 
49
97
  return {
@@ -16,8 +16,16 @@ export interface TokenizedDocument {
16
16
  lineCount(): number;
17
17
  /** Highlighted HTML per line for [start, end) — same line HTML extendLines produces. Tokenizes lazily. */
18
18
  lineRange(start: number, end: number): string[];
19
+ /** Plain source text per line for [start, end), without the `"\n"`. Never triggers tokenization. */
20
+ textRange(start: number, end: number): string[];
19
21
  /** Lines tokenized so far (monotonically grows; for tests/introspection). */
20
22
  tokenizedThrough(): number;
23
+ /**
24
+ * Tokenize and checkpoint through `line` without rendering, so a later
25
+ * `lineRange` there is cheap (e.g. tokenizing ahead in idle time).
26
+ * Returns true once the whole document has been fed.
27
+ */
28
+ tokenizeThrough(line: number): boolean;
21
29
  /**
22
30
  * Number of retained engine checkpoints; one is created every
23
31
  * `checkpointInterval` lines of tokenized content and never evicted (see
@@ -97,7 +97,6 @@ export function createTokenizedDocument({
97
97
  /** @type {string[]} */
98
98
  let openScopesStack = [];
99
99
  let pendingHtmlStr = "";
100
- let renderedEventCount = 0;
101
100
 
102
101
  /** @type {Checkpoint[]} */
103
102
  let checkpoints = [];
@@ -107,8 +106,14 @@ export function createTokenizedDocument({
107
106
 
108
107
  /** @param {number} scanFrom Character offset to resume scanning from. */
109
108
  function extendLineOffsets(scanFrom) {
110
- for (let i = scanFrom; i < code.length; i++) {
111
- if (code.charCodeAt(i) === 10) lineStartOffsets.push(i + 1);
109
+ // indexOf jumps between line breaks natively instead of testing every
110
+ // character in JS.
111
+ for (
112
+ let i = code.indexOf("\n", scanFrom);
113
+ i !== -1;
114
+ i = code.indexOf("\n", i + 1)
115
+ ) {
116
+ lineStartOffsets.push(i + 1);
112
117
  }
113
118
  }
114
119
 
@@ -134,7 +139,6 @@ export function createTokenizedDocument({
134
139
  committedLineCount = 0;
135
140
  openScopesStack = [];
136
141
  pendingHtmlStr = "";
137
- renderedEventCount = 0;
138
142
  checkpoints = [
139
143
  {
140
144
  line: 0,
@@ -164,23 +168,27 @@ export function createTokenizedDocument({
164
168
  while (committedLineCount < clampedTarget && fedLineCount < total) {
165
169
  const batchEndLine = Math.min(fedLineCount + checkpointInterval, total);
166
170
  const batchEndOffset = lineStartOffset(batchEndLine);
167
- const chunk = code.slice(fedOffset, batchEndOffset);
168
- if (chunk.length > 0) session.append(chunk);
171
+ // A slice of `code`, not the batch alone: append()-ing batches made
172
+ // the engine re-flatten the grown text on each one, so a jump to the
173
+ // end of a 500k-line file took 13.7 s instead of 3.3 s.
174
+ if (batchEndOffset > fedOffset) {
175
+ session.feed(code.slice(0, batchEndOffset));
176
+ }
169
177
  fedOffset = batchEndOffset;
170
178
  fedLineCount = batchEndLine;
171
179
 
172
- const committedEvents = session.events();
173
- if (committedEvents.length > renderedEventCount) {
174
- const result = extendLines(
175
- committedEvents.slice(renderedEventCount),
176
- openScopesStack,
177
- pendingHtmlStr,
178
- { classPrefix },
179
- );
180
+ // Each event is only needed once, to extend the line state; taking
181
+ // them keeps the session from retaining the whole document's events
182
+ // (~140 MB per 100k lines of TypeScript). Windows resume from
183
+ // checkpoint snapshots with fresh events (see lineRange).
184
+ const newEvents = session.takeEvents();
185
+ if (newEvents.length > 0) {
186
+ const result = extendLines(newEvents, openScopesStack, pendingHtmlStr, {
187
+ classPrefix,
188
+ });
180
189
  committedLineCount += result.completedLines.length;
181
190
  openScopesStack = result.openScopes;
182
191
  pendingHtmlStr = result.pendingHtml;
183
- renderedEventCount = committedEvents.length;
184
192
  }
185
193
 
186
194
  checkpoints.push({
@@ -260,10 +268,47 @@ export function createTokenizedDocument({
260
268
  return combined.slice(s - checkpoint.line, e - checkpoint.line);
261
269
  },
262
270
 
271
+ /**
272
+ * Plain source text per line for [start, end), sliced straight from
273
+ * `code` via `lineStartOffsets`. Never tokenizes - for consumers like
274
+ * `createSearch` that only need the text `lineRange` would decode back to.
275
+ * @param {number} start
276
+ * @param {number} end
277
+ * @returns {string[]}
278
+ */
279
+ textRange(start, end) {
280
+ const total = lineStartOffsets.length;
281
+ const s = Math.max(0, Math.min(start, total));
282
+ const e = Math.max(s, Math.min(end, total));
283
+ /** @type {string[]} */
284
+ const lines = [];
285
+ for (let i = s; i < e; i++) {
286
+ // Every line but the last ends one character before the next
287
+ // line's start (its "\n"); the last runs to the end of `code`.
288
+ const lineEnd =
289
+ i + 1 < total ? lineStartOffset(i + 1) - 1 : code.length;
290
+ lines.push(code.slice(lineStartOffset(i), lineEnd));
291
+ }
292
+ return lines;
293
+ },
294
+
263
295
  tokenizedThrough() {
264
296
  return committedLineCount;
265
297
  },
266
298
 
299
+ /**
300
+ * Tokenizes and checkpoints through `line` without rendering anything,
301
+ * so a later `lineRange` there is cheap. For tokenizing ahead in idle
302
+ * time (see HighlightVirtual's `tokenizeAhead`). Returns true once the
303
+ * whole document has been fed.
304
+ * @param {number} line
305
+ * @returns {boolean}
306
+ */
307
+ tokenizeThrough(line) {
308
+ ensureTokenizedThrough(line);
309
+ return fedLineCount >= lineStartOffsets.length;
310
+ },
311
+
267
312
  /**
268
313
  * Number of retained engine checkpoints; one is created every
269
314
  * `checkpointInterval` lines of tokenized content and never evicted (see
package/worker.js CHANGED
@@ -132,6 +132,12 @@ export function serveHighlighter(
132
132
  }
133
133
  case "session.snapshot": {
134
134
  const session = getSession(msg.sessionId);
135
+ // `want` names the one field the client reads, so `snapshot()`
136
+ // doesn't structured-clone every event so far just to drop them -
137
+ // bench/worker.bench.ts. Without it (an older client), reply with
138
+ // both.
139
+ if (msg.want === "snapshot") return { snapshot: session.snapshot() };
140
+ if (msg.want === "events") return { events: session.events() };
135
141
  return { snapshot: session.snapshot(), events: session.events() };
136
142
  }
137
143
  default:
@@ -258,13 +264,19 @@ function createRemoteHighlighter(worker) {
258
264
  async snapshot() {
259
265
  if (pendingAppend) await pendingAppend.promise;
260
266
  await created;
261
- const { snapshot } = await send("session.snapshot", { sessionId });
267
+ const { snapshot } = await send("session.snapshot", {
268
+ sessionId,
269
+ want: "snapshot",
270
+ });
262
271
  return snapshot;
263
272
  },
264
273
  async events() {
265
274
  if (pendingAppend) await pendingAppend.promise;
266
275
  await created;
267
- const { events } = await send("session.snapshot", { sessionId });
276
+ const { events } = await send("session.snapshot", {
277
+ sessionId,
278
+ want: "events",
279
+ });
268
280
  return events;
269
281
  },
270
282
  };