svelte-highlight 7.22.0 → 7.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AnsiOutput.svelte +6 -30
- package/FileTabs.svelte +3 -1
- package/HighlightEditable.svelte +115 -143
- package/HighlightStream.svelte +44 -48
- package/HighlightStyle.svelte +9 -21
- package/HighlightVirtual.svelte +82 -4
- package/HighlightVirtual.svelte.d.ts +19 -2
- package/MarkdownStream.svelte +16 -9
- package/README.md +27 -7
- package/ansi-color.js +109 -22
- package/ansi-output.js +93 -0
- package/ansi.d.ts +18 -0
- package/ansi.js +85 -24
- package/css-walk.js +9 -12
- package/editable-css-paint.js +118 -0
- package/editable-dom-paint.js +217 -146
- package/editable-line-dom.js +241 -0
- package/engine.d.ts +28 -6
- package/engine.js +707 -144
- package/fence.d.ts +5 -5
- package/fence.js +196 -56
- package/highlight-theme.js +10 -4
- package/incremental-tokenize.js +95 -17
- package/package.json +1 -1
- package/preprocessor.js +16 -2
- package/scoped.js +15 -13
- package/search.d.ts +6 -4
- package/search.js +427 -85
- package/stream-final-highlight.js +52 -0
- package/stream-highlighted.js +17 -0
- package/stream-preview.js +7 -2
- package/stream-regenerate.js +262 -0
- package/style-registry.js +60 -0
- package/text-diff.js +66 -18
- package/tokenized-document.d.ts +8 -0
- package/tokenized-document.js +60 -15
- package/worker.js +14 -2
package/stream-highlighted.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* @returns {{
|
|
12
12
|
* appendLines: (lines: string[]) => void,
|
|
13
13
|
* reset: () => void,
|
|
14
|
+
* truncate: (lines: number) => void,
|
|
14
15
|
* toString: () => string,
|
|
15
16
|
* lineCount: number,
|
|
16
17
|
* }}
|
|
@@ -31,6 +32,22 @@ export function createCompletedHtmlBuffer() {
|
|
|
31
32
|
html = "";
|
|
32
33
|
lineCount = 0;
|
|
33
34
|
},
|
|
35
|
+
/**
|
|
36
|
+
* Keeps the first `lines` lines. Walks back from the end, so the cost
|
|
37
|
+
* scales with the lines dropped, not the lines kept.
|
|
38
|
+
* @param {number} lines
|
|
39
|
+
*/
|
|
40
|
+
truncate(lines) {
|
|
41
|
+
if (lines >= lineCount) return;
|
|
42
|
+
let end = html.length;
|
|
43
|
+
for (let n = lineCount; n > lines; n--) {
|
|
44
|
+
end = html.lastIndexOf("\n", end - 1);
|
|
45
|
+
}
|
|
46
|
+
// With `lines === 0`, the walk passes the first line, which has no
|
|
47
|
+
// "\n" before it, and `end` is -1.
|
|
48
|
+
html = lines === 0 ? "" : html.slice(0, end);
|
|
49
|
+
lineCount = lines;
|
|
50
|
+
},
|
|
34
51
|
toString() {
|
|
35
52
|
return html;
|
|
36
53
|
},
|
package/stream-preview.js
CHANGED
|
@@ -75,11 +75,16 @@ export function computeStagedTailPreview({
|
|
|
75
75
|
// Valid only for a pure append since the checkpoint: the committed session
|
|
76
76
|
// must not have advanced (no newline completed, so openScopes/pendingHtml
|
|
77
77
|
// - the checkpoint's base state - are still current) and `fedCode` must
|
|
78
|
-
// still start with the code the checkpoint was taken against.
|
|
78
|
+
// still start with the code the checkpoint was taken against. Only the
|
|
79
|
+
// part past the committed position needs comparing: `session` already
|
|
80
|
+
// covers everything before it and must match `fedCode` there anyway. A
|
|
81
|
+
// whole-buffer compare made every call O(stream), not O(line) -
|
|
82
|
+
// bench/stream-repaint.bench.ts.
|
|
79
83
|
const canResume =
|
|
80
84
|
cache !== undefined &&
|
|
81
85
|
cache.committedPos === snapshot.pos &&
|
|
82
|
-
fedCode.
|
|
86
|
+
fedCode.length >= cache.fedCode.length &&
|
|
87
|
+
fedCode.startsWith(cache.fedCode.slice(snapshot.pos), snapshot.pos);
|
|
83
88
|
|
|
84
89
|
const baseSnapshot = canResume ? cache.snapshot : snapshot;
|
|
85
90
|
const baseOpenScopes = canResume ? cache.openScopes : openScopes;
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HighlightStream's regenerate path: the stream's code changed in a way
|
|
3
|
+
* that isn't a pure append (an LLM "regenerate the last paragraph", say).
|
|
4
|
+
* Patches the session in place, then re-renders only from the sealed chunk
|
|
5
|
+
* the change lands in.
|
|
6
|
+
*
|
|
7
|
+
* A line's HTML depends only on its events and the scopes open at its
|
|
8
|
+
* start. So a sealed chunk that ends before the change, and whose events
|
|
9
|
+
* the patched parse left alone, stays as it is. Only the lines from the
|
|
10
|
+
* chunk the change lands in get rendered again; the lines before it are
|
|
11
|
+
* just walked to compare events and track scopes. Before, every line and
|
|
12
|
+
* every chunk was rebuilt - bench/stream-repaint.bench.ts.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { CLOSE, extendLines, OPEN, TEXT } from "./engine.js";
|
|
16
|
+
import {
|
|
17
|
+
buildSealedChunkHtml,
|
|
18
|
+
pushSealedChunk,
|
|
19
|
+
} from "./stream-sealed-chunks.js";
|
|
20
|
+
import { diffText } from "./text-diff.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @typedef {import("./engine.d.ts").ScopeEvent} ScopeEvent
|
|
24
|
+
* @typedef {import("./engine.d.ts").StreamSession} StreamSession
|
|
25
|
+
* @typedef {ReturnType<typeof import("./stream-highlighted.js").createCompletedHtmlBuffer>} CompletedHtmlBuffer
|
|
26
|
+
* @typedef {NonNullable<ReturnType<StreamSession["checkpointBefore"]>>} Checkpoint
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Where `extendLinesAfterPatch` can start its walk after a `replace()`
|
|
31
|
+
* that kept the first `keptEvents` events (see `StreamSession#replace`):
|
|
32
|
+
* the last checkpoint before the break of the last line start that
|
|
33
|
+
* certainly survives. Those events match the previous ones by identity,
|
|
34
|
+
* so the walk from the top would only have confirmed them. Undefined when
|
|
35
|
+
* no line start is that early.
|
|
36
|
+
*
|
|
37
|
+
* Before, every regenerate walked every event before the change, about a
|
|
38
|
+
* fifth of a regenerate at 8,000 lines (bench/stream-repaint.bench.ts).
|
|
39
|
+
* A session's first `replace()` re-parses and keeps no events, so the
|
|
40
|
+
* first regenerate still walks from the top.
|
|
41
|
+
* @param {StreamSession} session
|
|
42
|
+
* @param {number} keptEvents
|
|
43
|
+
* @param {number[]} lineStarts Ascending.
|
|
44
|
+
* @returns {Checkpoint | undefined}
|
|
45
|
+
*/
|
|
46
|
+
export function walkStart(session, keptEvents, lineStarts) {
|
|
47
|
+
// The text the kept events cover ends at the last checkpoint they reach.
|
|
48
|
+
const kept = session.checkpointBefore({ eventCount: keptEvents });
|
|
49
|
+
if (!kept) return undefined;
|
|
50
|
+
// The last line start whose break lies in that text.
|
|
51
|
+
let last = -1;
|
|
52
|
+
while (
|
|
53
|
+
last + 1 < lineStarts.length &&
|
|
54
|
+
/** @type {number} */ (lineStarts[last + 1]) - 1 < kept.textPos
|
|
55
|
+
) {
|
|
56
|
+
last++;
|
|
57
|
+
}
|
|
58
|
+
if (last < 0) return undefined;
|
|
59
|
+
return session.checkpointBefore({
|
|
60
|
+
textPos: /** @type {number} */ (lineStarts[last]) - 1,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Offsets of the line starts every `chunkLines` lines in `text`, at most
|
|
66
|
+
* `maxChunks` of them, up to `end`.
|
|
67
|
+
* @param {string} text
|
|
68
|
+
* @param {number} end
|
|
69
|
+
* @param {number} chunkLines
|
|
70
|
+
* @param {number} maxChunks
|
|
71
|
+
* @returns {number[]}
|
|
72
|
+
*/
|
|
73
|
+
function chunkStarts(text, end, chunkLines, maxChunks) {
|
|
74
|
+
/** @type {number[]} */
|
|
75
|
+
const starts = [];
|
|
76
|
+
let lines = 0;
|
|
77
|
+
for (
|
|
78
|
+
let i = text.indexOf("\n");
|
|
79
|
+
i !== -1 && i < end && starts.length < maxChunks;
|
|
80
|
+
i = text.indexOf("\n", i + 1)
|
|
81
|
+
) {
|
|
82
|
+
if (++lines === chunkLines) {
|
|
83
|
+
starts.push(i + 1);
|
|
84
|
+
lines = 0;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return starts;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* @param {ScopeEvent} a
|
|
92
|
+
* @param {ScopeEvent | undefined} b
|
|
93
|
+
*/
|
|
94
|
+
function sameEvent(a, b) {
|
|
95
|
+
if (a === b) return true;
|
|
96
|
+
if (b === undefined || a.t !== b.t) return false;
|
|
97
|
+
if (a.t === TEXT) return a.v === /** @type {{ v: string }} */ (b).v;
|
|
98
|
+
if (a.t === OPEN) return a.s === /** @type {{ s: string }} */ (b).s;
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Renders a patched session's `events` from the last of `lineStarts` (text
|
|
104
|
+
* offsets of line starts, ascending) whose earlier events all match
|
|
105
|
+
* `previousEvents`, the events before the patch. Lines before that point
|
|
106
|
+
* render exactly as they did, so only the lines from it on are rendered.
|
|
107
|
+
*
|
|
108
|
+
* Checking the events, not just the text, matters: with multi-line
|
|
109
|
+
* lookahead (a Ruby heredoc, say), the patched parse can tokenize an
|
|
110
|
+
* unchanged line differently from the stream that first rendered it.
|
|
111
|
+
*
|
|
112
|
+
* Returns how many of `lineStarts` are kept (rendering resumes at
|
|
113
|
+
* `lineStarts[kept - 1]`, or at the top if 0) and `extendLines`' result
|
|
114
|
+
* from there.
|
|
115
|
+
*
|
|
116
|
+
* `from`, when given, is a point known to follow only events that match
|
|
117
|
+
* `previousEvents` (see `walkStart`), so the walk starts there instead of
|
|
118
|
+
* at the top, with the same result.
|
|
119
|
+
* @param {ScopeEvent[]} events
|
|
120
|
+
* @param {ScopeEvent[]} previousEvents
|
|
121
|
+
* @param {number[]} lineStarts
|
|
122
|
+
* @param {Checkpoint} [from]
|
|
123
|
+
* @returns {{ kept: number, result: ReturnType<typeof extendLines> }}
|
|
124
|
+
*/
|
|
125
|
+
export function extendLinesAfterPatch(
|
|
126
|
+
events,
|
|
127
|
+
previousEvents,
|
|
128
|
+
lineStarts,
|
|
129
|
+
from,
|
|
130
|
+
) {
|
|
131
|
+
let kept = 0;
|
|
132
|
+
// Where line `lineStarts[kept - 1]` starts: its line break's event, the
|
|
133
|
+
// break's index in it, and the scopes open there.
|
|
134
|
+
let resumeEvent = 0;
|
|
135
|
+
let resumeAt = 0;
|
|
136
|
+
/** @type {string[]} */
|
|
137
|
+
let resumeStack = [];
|
|
138
|
+
/** @type {string[]} */
|
|
139
|
+
const stack = from ? [...from.scopes] : [];
|
|
140
|
+
let pos = from ? from.textPos : 0;
|
|
141
|
+
let recorded = false;
|
|
142
|
+
// Line breaks before `from` are passed; `walkStart` picks `from` before
|
|
143
|
+
// the break of the last line start kept, so the walk still records it.
|
|
144
|
+
while (
|
|
145
|
+
kept < lineStarts.length &&
|
|
146
|
+
/** @type {number} */ (lineStarts[kept]) - 1 < pos
|
|
147
|
+
) {
|
|
148
|
+
kept++;
|
|
149
|
+
}
|
|
150
|
+
for (
|
|
151
|
+
let i = from ? from.eventCount : 0;
|
|
152
|
+
i < events.length && kept < lineStarts.length;
|
|
153
|
+
i++
|
|
154
|
+
) {
|
|
155
|
+
const event = /** @type {ScopeEvent} */ (events[i]);
|
|
156
|
+
if (!sameEvent(event, previousEvents[i])) break;
|
|
157
|
+
if (event.t === OPEN) {
|
|
158
|
+
stack.push(event.s);
|
|
159
|
+
} else if (event.t === CLOSE) {
|
|
160
|
+
stack.pop();
|
|
161
|
+
} else {
|
|
162
|
+
const end = pos + event.v.length;
|
|
163
|
+
while (
|
|
164
|
+
kept < lineStarts.length &&
|
|
165
|
+
/** @type {number} */ (lineStarts[kept]) - 1 < end
|
|
166
|
+
) {
|
|
167
|
+
resumeEvent = i;
|
|
168
|
+
resumeAt = /** @type {number} */ (lineStarts[kept]) - 1 - pos;
|
|
169
|
+
resumeStack = [...stack];
|
|
170
|
+
kept++;
|
|
171
|
+
recorded = true;
|
|
172
|
+
}
|
|
173
|
+
pos = end;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
// Unreachable with `walkStart`'s `from`; kept so a bad one costs a full
|
|
177
|
+
// walk instead of wrong output.
|
|
178
|
+
if (from && kept > 0 && !recorded) {
|
|
179
|
+
return extendLinesAfterPatch(events, previousEvents, lineStarts);
|
|
180
|
+
}
|
|
181
|
+
if (kept === 0) return { kept, result: extendLines(events, [], "") };
|
|
182
|
+
|
|
183
|
+
// Resume from the line break, so extendLines opens the line with the
|
|
184
|
+
// scopes reopened, as it does after every break. The line it completes
|
|
185
|
+
// first is the end of the line before: drop it.
|
|
186
|
+
const text = /** @type {{ v: string }} */ (events[resumeEvent]).v;
|
|
187
|
+
/** @type {ScopeEvent[]} */
|
|
188
|
+
const rest = [{ t: TEXT, v: text.slice(resumeAt) }];
|
|
189
|
+
for (let j = resumeEvent + 1; j < events.length; j++) {
|
|
190
|
+
rest.push(/** @type {ScopeEvent} */ (events[j]));
|
|
191
|
+
}
|
|
192
|
+
const result = extendLines(rest, resumeStack, "");
|
|
193
|
+
result.completedLines = result.completedLines.slice(1);
|
|
194
|
+
return { kept, result };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* @param {{
|
|
199
|
+
* session: StreamSession,
|
|
200
|
+
* fedCode: string,
|
|
201
|
+
* code: string,
|
|
202
|
+
* sealedChunks: string[],
|
|
203
|
+
* completedHtml: CompletedHtmlBuffer,
|
|
204
|
+
* chunkLines: number,
|
|
205
|
+
* }} options `session` was fed exactly `fedCode`; `completedHtml` holds
|
|
206
|
+
* its completed lines, and `sealedChunks` the first of them, in chunks of
|
|
207
|
+
* `chunkLines`. `completedHtml` is updated in place; `sealedChunks` is
|
|
208
|
+
* left as is.
|
|
209
|
+
* @returns {{
|
|
210
|
+
* sealedChunks: string[],
|
|
211
|
+
* sealedLineCount: number,
|
|
212
|
+
* unsealedLines: string[],
|
|
213
|
+
* pendingHtml: string,
|
|
214
|
+
* openScopes: string[],
|
|
215
|
+
* committedCount: number,
|
|
216
|
+
* }}
|
|
217
|
+
*/
|
|
218
|
+
export function regenerate({
|
|
219
|
+
session,
|
|
220
|
+
fedCode,
|
|
221
|
+
code,
|
|
222
|
+
sealedChunks,
|
|
223
|
+
completedHtml,
|
|
224
|
+
chunkLines,
|
|
225
|
+
}) {
|
|
226
|
+
const previousEvents = session.events();
|
|
227
|
+
const { start, removed, inserted } = diffText(fedCode, code);
|
|
228
|
+
const keptEvents = session.replace(start, start + removed.length, inserted);
|
|
229
|
+
const events = session.events();
|
|
230
|
+
|
|
231
|
+
// Keep the sealed chunks that end before the changed line, as far as
|
|
232
|
+
// the patched events still match.
|
|
233
|
+
const lineStarts = chunkStarts(code, start, chunkLines, sealedChunks.length);
|
|
234
|
+
const { kept: keptChunks, result } = extendLinesAfterPatch(
|
|
235
|
+
events,
|
|
236
|
+
previousEvents,
|
|
237
|
+
lineStarts,
|
|
238
|
+
keptEvents > 0 ? walkStart(session, keptEvents, lineStarts) : undefined,
|
|
239
|
+
);
|
|
240
|
+
|
|
241
|
+
let sealedLineCount = keptChunks * chunkLines;
|
|
242
|
+
completedHtml.truncate(sealedLineCount);
|
|
243
|
+
completedHtml.appendLines(result.completedLines);
|
|
244
|
+
let chunks = sealedChunks.slice(0, keptChunks);
|
|
245
|
+
let unsealedLines = result.completedLines;
|
|
246
|
+
while (unsealedLines.length >= chunkLines) {
|
|
247
|
+
chunks = pushSealedChunk(
|
|
248
|
+
chunks,
|
|
249
|
+
buildSealedChunkHtml(unsealedLines.slice(0, chunkLines), sealedLineCount),
|
|
250
|
+
);
|
|
251
|
+
sealedLineCount += chunkLines;
|
|
252
|
+
unsealedLines = unsealedLines.slice(chunkLines);
|
|
253
|
+
}
|
|
254
|
+
return {
|
|
255
|
+
sealedChunks: chunks,
|
|
256
|
+
sealedLineCount,
|
|
257
|
+
unsealedLines,
|
|
258
|
+
pendingHtml: result.pendingHtml,
|
|
259
|
+
openScopes: result.openScopes,
|
|
260
|
+
committedCount: events.length,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ownership registry for HighlightStyle's injected `<style>` tags, kept out
|
|
3
|
+
* of the component so it can be unit-tested (see
|
|
4
|
+
* tests/style-registry.test.ts).
|
|
5
|
+
*
|
|
6
|
+
* Instances sharing a key queue up in registration order; only the oldest
|
|
7
|
+
* still-registered instance renders the `<style>`. A store (rather than a
|
|
8
|
+
* plain ref count) means that when that instance leaves, the remaining
|
|
9
|
+
* subscribers reactively re-derive a new owner instead of the tag being
|
|
10
|
+
* dropped while siblings still need it.
|
|
11
|
+
*/
|
|
12
|
+
import { writable } from "svelte/store";
|
|
13
|
+
|
|
14
|
+
/** @typedef {import("svelte/store").Writable<symbol[]>} Owners */
|
|
15
|
+
|
|
16
|
+
export function createStyleRegistry() {
|
|
17
|
+
/** @type {Map<string, Owners>} */
|
|
18
|
+
const registries = new Map();
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
/**
|
|
22
|
+
* Adds `token` as the newest owner of `key`.
|
|
23
|
+
* @param {string} key
|
|
24
|
+
* @param {symbol} token
|
|
25
|
+
* @returns {Owners} The key's owner list, to subscribe to.
|
|
26
|
+
*/
|
|
27
|
+
register(key, token) {
|
|
28
|
+
let owners = registries.get(key);
|
|
29
|
+
if (!owners) {
|
|
30
|
+
owners = writable([]);
|
|
31
|
+
registries.set(key, owners);
|
|
32
|
+
}
|
|
33
|
+
owners.update((list) => [...list, token]);
|
|
34
|
+
return owners;
|
|
35
|
+
},
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Removes `token` from `owners`, and drops `key` once its last owner
|
|
39
|
+
* leaves. Keys embed the full theme CSS, so a long-lived app that
|
|
40
|
+
* cycles themes would otherwise keep every theme it ever showed.
|
|
41
|
+
* @param {string} key
|
|
42
|
+
* @param {Owners} owners The store `register` returned for `key`.
|
|
43
|
+
* @param {symbol} token
|
|
44
|
+
*/
|
|
45
|
+
unregister(key, owners, token) {
|
|
46
|
+
let empty = false;
|
|
47
|
+
owners.update((list) => {
|
|
48
|
+
const remaining = list.filter((owner) => owner !== token);
|
|
49
|
+
empty = remaining.length === 0;
|
|
50
|
+
return remaining;
|
|
51
|
+
});
|
|
52
|
+
if (empty && registries.get(key) === owners) registries.delete(key);
|
|
53
|
+
},
|
|
54
|
+
|
|
55
|
+
/** Number of keys with at least one owner. */
|
|
56
|
+
get size() {
|
|
57
|
+
return registries.size;
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
package/text-diff.js
CHANGED
|
@@ -2,6 +2,9 @@ const HIGH_SURROGATE_MIN = 0xd800;
|
|
|
2
2
|
const HIGH_SURROGATE_MAX = 0xdbff;
|
|
3
3
|
const LOW_SURROGATE_MIN = 0xdc00;
|
|
4
4
|
const LOW_SURROGATE_MAX = 0xdfff;
|
|
5
|
+
// Chunk sizes for galloping over a shared prefix/suffix (see commonPrefix).
|
|
6
|
+
const GALLOP_MAX = 4096;
|
|
7
|
+
const GALLOP_MIN = 16;
|
|
5
8
|
|
|
6
9
|
/** @param {number} code */
|
|
7
10
|
function isHighSurrogate(code) {
|
|
@@ -13,6 +16,67 @@ function isLowSurrogate(code) {
|
|
|
13
16
|
return code >= LOW_SURROGATE_MIN && code <= LOW_SURROGATE_MAX;
|
|
14
17
|
}
|
|
15
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Length of the shared prefix of `a` and `b`, capped at `max`.
|
|
21
|
+
*
|
|
22
|
+
* The first GALLOP_MIN characters are compared one at a time, so inputs
|
|
23
|
+
* that differ early never pay for a slice. Past that, it gallops in native
|
|
24
|
+
* string compares, from GALLOP_MAX-character chunks down to GALLOP_MIN,
|
|
25
|
+
* instead of one JS iteration per character: the prefix is the whole
|
|
26
|
+
* document on a pure append, the common case while typing. A larger step
|
|
27
|
+
* fails at most once before shrinking, so each level after the first costs
|
|
28
|
+
* O(step). text-diff.bench.ts: ~25x faster on a 50k-character append,
|
|
29
|
+
* ~5x on a 1k one.
|
|
30
|
+
* @param {string} a
|
|
31
|
+
* @param {string} b
|
|
32
|
+
* @param {number} max
|
|
33
|
+
*/
|
|
34
|
+
function commonPrefix(a, b, max) {
|
|
35
|
+
// charCodeAt (not bracket indexing), so no 1-character string is
|
|
36
|
+
// allocated per position compared.
|
|
37
|
+
let n = 0;
|
|
38
|
+
while (n < GALLOP_MIN && n < max && a.charCodeAt(n) === b.charCodeAt(n)) n++;
|
|
39
|
+
if (n < GALLOP_MIN) return n;
|
|
40
|
+
for (let step = GALLOP_MAX; step >= GALLOP_MIN; step >>= 2) {
|
|
41
|
+
while (n + step <= max && b.startsWith(a.slice(n, n + step), n)) {
|
|
42
|
+
n += step;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
while (n < max && a.charCodeAt(n) === b.charCodeAt(n)) n++;
|
|
46
|
+
return n;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Length of the shared suffix of `a` and `b`, capped at `max`. Mirrors
|
|
51
|
+
* `commonPrefix`.
|
|
52
|
+
* @param {string} a
|
|
53
|
+
* @param {string} b
|
|
54
|
+
* @param {number} max
|
|
55
|
+
*/
|
|
56
|
+
function commonSuffix(a, b, max) {
|
|
57
|
+
const aEnd = a.length - 1;
|
|
58
|
+
const bEnd = b.length - 1;
|
|
59
|
+
let n = 0;
|
|
60
|
+
while (
|
|
61
|
+
n < GALLOP_MIN &&
|
|
62
|
+
n < max &&
|
|
63
|
+
a.charCodeAt(aEnd - n) === b.charCodeAt(bEnd - n)
|
|
64
|
+
) {
|
|
65
|
+
n++;
|
|
66
|
+
}
|
|
67
|
+
if (n < GALLOP_MIN) return n;
|
|
68
|
+
for (let step = GALLOP_MAX; step >= GALLOP_MIN; step >>= 2) {
|
|
69
|
+
while (
|
|
70
|
+
n + step <= max &&
|
|
71
|
+
b.endsWith(a.slice(a.length - n - step, a.length - n), b.length - n)
|
|
72
|
+
) {
|
|
73
|
+
n += step;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
while (n < max && a.charCodeAt(aEnd - n) === b.charCodeAt(bEnd - n)) n++;
|
|
77
|
+
return n;
|
|
78
|
+
}
|
|
79
|
+
|
|
16
80
|
/**
|
|
17
81
|
* Diffs two strings down to a common-prefix/suffix trim. Trims on Unicode
|
|
18
82
|
* code points (never splitting a surrogate pair between the shared prefix
|
|
@@ -24,26 +88,10 @@ function isLowSurrogate(code) {
|
|
|
24
88
|
export function diffText(before, after) {
|
|
25
89
|
const minLength = Math.min(before.length, after.length);
|
|
26
90
|
|
|
27
|
-
|
|
28
|
-
// unchanged prefix on a pure-append edit, the common case while typing -
|
|
29
|
-
// doesn't allocate a 1-character string per position compared.
|
|
30
|
-
let prefix = 0;
|
|
31
|
-
while (
|
|
32
|
-
prefix < minLength &&
|
|
33
|
-
before.charCodeAt(prefix) === after.charCodeAt(prefix)
|
|
34
|
-
)
|
|
35
|
-
prefix++;
|
|
91
|
+
let prefix = commonPrefix(before, after, minLength);
|
|
36
92
|
if (isHighSurrogate(before.charCodeAt(prefix - 1))) prefix--;
|
|
37
93
|
|
|
38
|
-
let suffix =
|
|
39
|
-
const maxSuffix = minLength - prefix;
|
|
40
|
-
while (
|
|
41
|
-
suffix < maxSuffix &&
|
|
42
|
-
before.charCodeAt(before.length - 1 - suffix) ===
|
|
43
|
-
after.charCodeAt(after.length - 1 - suffix)
|
|
44
|
-
) {
|
|
45
|
-
suffix++;
|
|
46
|
-
}
|
|
94
|
+
let suffix = commonSuffix(before, after, minLength - prefix);
|
|
47
95
|
if (isLowSurrogate(before.charCodeAt(before.length - suffix))) suffix--;
|
|
48
96
|
|
|
49
97
|
return {
|
package/tokenized-document.d.ts
CHANGED
|
@@ -16,8 +16,16 @@ export interface TokenizedDocument {
|
|
|
16
16
|
lineCount(): number;
|
|
17
17
|
/** Highlighted HTML per line for [start, end) — same line HTML extendLines produces. Tokenizes lazily. */
|
|
18
18
|
lineRange(start: number, end: number): string[];
|
|
19
|
+
/** Plain source text per line for [start, end), without the `"\n"`. Never triggers tokenization. */
|
|
20
|
+
textRange(start: number, end: number): string[];
|
|
19
21
|
/** Lines tokenized so far (monotonically grows; for tests/introspection). */
|
|
20
22
|
tokenizedThrough(): number;
|
|
23
|
+
/**
|
|
24
|
+
* Tokenize and checkpoint through `line` without rendering, so a later
|
|
25
|
+
* `lineRange` there is cheap (e.g. tokenizing ahead in idle time).
|
|
26
|
+
* Returns true once the whole document has been fed.
|
|
27
|
+
*/
|
|
28
|
+
tokenizeThrough(line: number): boolean;
|
|
21
29
|
/**
|
|
22
30
|
* Number of retained engine checkpoints; one is created every
|
|
23
31
|
* `checkpointInterval` lines of tokenized content and never evicted (see
|
package/tokenized-document.js
CHANGED
|
@@ -97,7 +97,6 @@ export function createTokenizedDocument({
|
|
|
97
97
|
/** @type {string[]} */
|
|
98
98
|
let openScopesStack = [];
|
|
99
99
|
let pendingHtmlStr = "";
|
|
100
|
-
let renderedEventCount = 0;
|
|
101
100
|
|
|
102
101
|
/** @type {Checkpoint[]} */
|
|
103
102
|
let checkpoints = [];
|
|
@@ -107,8 +106,14 @@ export function createTokenizedDocument({
|
|
|
107
106
|
|
|
108
107
|
/** @param {number} scanFrom Character offset to resume scanning from. */
|
|
109
108
|
function extendLineOffsets(scanFrom) {
|
|
110
|
-
|
|
111
|
-
|
|
109
|
+
// indexOf jumps between line breaks natively instead of testing every
|
|
110
|
+
// character in JS.
|
|
111
|
+
for (
|
|
112
|
+
let i = code.indexOf("\n", scanFrom);
|
|
113
|
+
i !== -1;
|
|
114
|
+
i = code.indexOf("\n", i + 1)
|
|
115
|
+
) {
|
|
116
|
+
lineStartOffsets.push(i + 1);
|
|
112
117
|
}
|
|
113
118
|
}
|
|
114
119
|
|
|
@@ -134,7 +139,6 @@ export function createTokenizedDocument({
|
|
|
134
139
|
committedLineCount = 0;
|
|
135
140
|
openScopesStack = [];
|
|
136
141
|
pendingHtmlStr = "";
|
|
137
|
-
renderedEventCount = 0;
|
|
138
142
|
checkpoints = [
|
|
139
143
|
{
|
|
140
144
|
line: 0,
|
|
@@ -164,23 +168,27 @@ export function createTokenizedDocument({
|
|
|
164
168
|
while (committedLineCount < clampedTarget && fedLineCount < total) {
|
|
165
169
|
const batchEndLine = Math.min(fedLineCount + checkpointInterval, total);
|
|
166
170
|
const batchEndOffset = lineStartOffset(batchEndLine);
|
|
167
|
-
|
|
168
|
-
|
|
171
|
+
// A slice of `code`, not the batch alone: append()-ing batches made
|
|
172
|
+
// the engine re-flatten the grown text on each one, so a jump to the
|
|
173
|
+
// end of a 500k-line file took 13.7 s instead of 3.3 s.
|
|
174
|
+
if (batchEndOffset > fedOffset) {
|
|
175
|
+
session.feed(code.slice(0, batchEndOffset));
|
|
176
|
+
}
|
|
169
177
|
fedOffset = batchEndOffset;
|
|
170
178
|
fedLineCount = batchEndLine;
|
|
171
179
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
+
// Each event is only needed once, to extend the line state; taking
|
|
181
|
+
// them keeps the session from retaining the whole document's events
|
|
182
|
+
// (~140 MB per 100k lines of TypeScript). Windows resume from
|
|
183
|
+
// checkpoint snapshots with fresh events (see lineRange).
|
|
184
|
+
const newEvents = session.takeEvents();
|
|
185
|
+
if (newEvents.length > 0) {
|
|
186
|
+
const result = extendLines(newEvents, openScopesStack, pendingHtmlStr, {
|
|
187
|
+
classPrefix,
|
|
188
|
+
});
|
|
180
189
|
committedLineCount += result.completedLines.length;
|
|
181
190
|
openScopesStack = result.openScopes;
|
|
182
191
|
pendingHtmlStr = result.pendingHtml;
|
|
183
|
-
renderedEventCount = committedEvents.length;
|
|
184
192
|
}
|
|
185
193
|
|
|
186
194
|
checkpoints.push({
|
|
@@ -260,10 +268,47 @@ export function createTokenizedDocument({
|
|
|
260
268
|
return combined.slice(s - checkpoint.line, e - checkpoint.line);
|
|
261
269
|
},
|
|
262
270
|
|
|
271
|
+
/**
|
|
272
|
+
* Plain source text per line for [start, end), sliced straight from
|
|
273
|
+
* `code` via `lineStartOffsets`. Never tokenizes - for consumers like
|
|
274
|
+
* `createSearch` that only need the text `lineRange` would decode back to.
|
|
275
|
+
* @param {number} start
|
|
276
|
+
* @param {number} end
|
|
277
|
+
* @returns {string[]}
|
|
278
|
+
*/
|
|
279
|
+
textRange(start, end) {
|
|
280
|
+
const total = lineStartOffsets.length;
|
|
281
|
+
const s = Math.max(0, Math.min(start, total));
|
|
282
|
+
const e = Math.max(s, Math.min(end, total));
|
|
283
|
+
/** @type {string[]} */
|
|
284
|
+
const lines = [];
|
|
285
|
+
for (let i = s; i < e; i++) {
|
|
286
|
+
// Every line but the last ends one character before the next
|
|
287
|
+
// line's start (its "\n"); the last runs to the end of `code`.
|
|
288
|
+
const lineEnd =
|
|
289
|
+
i + 1 < total ? lineStartOffset(i + 1) - 1 : code.length;
|
|
290
|
+
lines.push(code.slice(lineStartOffset(i), lineEnd));
|
|
291
|
+
}
|
|
292
|
+
return lines;
|
|
293
|
+
},
|
|
294
|
+
|
|
263
295
|
tokenizedThrough() {
|
|
264
296
|
return committedLineCount;
|
|
265
297
|
},
|
|
266
298
|
|
|
299
|
+
/**
|
|
300
|
+
* Tokenizes and checkpoints through `line` without rendering anything,
|
|
301
|
+
* so a later `lineRange` there is cheap. For tokenizing ahead in idle
|
|
302
|
+
* time (see HighlightVirtual's `tokenizeAhead`). Returns true once the
|
|
303
|
+
* whole document has been fed.
|
|
304
|
+
* @param {number} line
|
|
305
|
+
* @returns {boolean}
|
|
306
|
+
*/
|
|
307
|
+
tokenizeThrough(line) {
|
|
308
|
+
ensureTokenizedThrough(line);
|
|
309
|
+
return fedLineCount >= lineStartOffsets.length;
|
|
310
|
+
},
|
|
311
|
+
|
|
267
312
|
/**
|
|
268
313
|
* Number of retained engine checkpoints; one is created every
|
|
269
314
|
* `checkpointInterval` lines of tokenized content and never evicted (see
|
package/worker.js
CHANGED
|
@@ -132,6 +132,12 @@ export function serveHighlighter(
|
|
|
132
132
|
}
|
|
133
133
|
case "session.snapshot": {
|
|
134
134
|
const session = getSession(msg.sessionId);
|
|
135
|
+
// `want` names the one field the client reads, so `snapshot()`
|
|
136
|
+
// doesn't structured-clone every event so far just to drop them -
|
|
137
|
+
// bench/worker.bench.ts. Without it (an older client), reply with
|
|
138
|
+
// both.
|
|
139
|
+
if (msg.want === "snapshot") return { snapshot: session.snapshot() };
|
|
140
|
+
if (msg.want === "events") return { events: session.events() };
|
|
135
141
|
return { snapshot: session.snapshot(), events: session.events() };
|
|
136
142
|
}
|
|
137
143
|
default:
|
|
@@ -258,13 +264,19 @@ function createRemoteHighlighter(worker) {
|
|
|
258
264
|
async snapshot() {
|
|
259
265
|
if (pendingAppend) await pendingAppend.promise;
|
|
260
266
|
await created;
|
|
261
|
-
const { snapshot } = await send("session.snapshot", {
|
|
267
|
+
const { snapshot } = await send("session.snapshot", {
|
|
268
|
+
sessionId,
|
|
269
|
+
want: "snapshot",
|
|
270
|
+
});
|
|
262
271
|
return snapshot;
|
|
263
272
|
},
|
|
264
273
|
async events() {
|
|
265
274
|
if (pendingAppend) await pendingAppend.promise;
|
|
266
275
|
await created;
|
|
267
|
-
const { events } = await send("session.snapshot", {
|
|
276
|
+
const { events } = await send("session.snapshot", {
|
|
277
|
+
sessionId,
|
|
278
|
+
want: "events",
|
|
279
|
+
});
|
|
268
280
|
return events;
|
|
269
281
|
},
|
|
270
282
|
};
|