svelte-highlight 7.22.0 → 7.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AnsiOutput.svelte +6 -30
- package/FileTabs.svelte +3 -1
- package/HighlightEditable.svelte +115 -143
- package/HighlightStream.svelte +44 -48
- package/HighlightStyle.svelte +9 -21
- package/HighlightVirtual.svelte +82 -4
- package/HighlightVirtual.svelte.d.ts +19 -2
- package/MarkdownStream.svelte +16 -9
- package/README.md +27 -7
- package/ansi-color.js +109 -22
- package/ansi-output.js +93 -0
- package/ansi.d.ts +18 -0
- package/ansi.js +85 -24
- package/css-walk.js +9 -12
- package/editable-css-paint.js +118 -0
- package/editable-dom-paint.js +217 -146
- package/editable-line-dom.js +241 -0
- package/engine.d.ts +28 -6
- package/engine.js +707 -144
- package/fence.d.ts +5 -5
- package/fence.js +196 -56
- package/highlight-theme.js +10 -4
- package/incremental-tokenize.js +95 -17
- package/package.json +1 -1
- package/preprocessor.js +16 -2
- package/scoped.js +15 -13
- package/search.d.ts +6 -4
- package/search.js +427 -85
- package/stream-final-highlight.js +52 -0
- package/stream-highlighted.js +17 -0
- package/stream-preview.js +7 -2
- package/stream-regenerate.js +262 -0
- package/style-registry.js +60 -0
- package/text-diff.js +66 -18
- package/tokenized-document.d.ts +8 -0
- package/tokenized-document.js +60 -15
- package/worker.js +14 -2
package/fence.d.ts
CHANGED
|
@@ -85,11 +85,11 @@ export type MarkdownSegment = TextSegment | FenceSegment;
|
|
|
85
85
|
*/
|
|
86
86
|
export interface FenceSplitter {
|
|
87
87
|
/**
|
|
88
|
-
* Appends `chunk` to the buffer. O(chunk):
|
|
89
|
-
* the last
|
|
90
|
-
*
|
|
91
|
-
* segment object keeps its identity, so a
|
|
92
|
-
* them.
|
|
88
|
+
* Appends `chunk` to the buffer. O(chunk): resumes scanning from the start
|
|
89
|
+
* of the last line not yet terminated by a newline, since only that line
|
|
90
|
+
* (and so only the last segment) can change. Only the last segment object
|
|
91
|
+
* may be replaced; every earlier segment object keeps its identity, so a
|
|
92
|
+
* keyed `{#each}` never re-mounts them.
|
|
93
93
|
*/
|
|
94
94
|
append(chunk: string): void;
|
|
95
95
|
/**
|
package/fence.js
CHANGED
|
@@ -9,8 +9,10 @@ import { ensureRegistered, registry } from "./registry.js";
|
|
|
9
9
|
|
|
10
10
|
const TOKEN_RE = /(\w+)="([^"]*)"|(\w+)=\{([^}]*)\}|\{([^}]*)\}|(\w+)/g;
|
|
11
11
|
const WHITESPACE_RE = /\s+/;
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
// A line here excludes its "\n" but keeps a CRLF line's "\r", which `.`
|
|
13
|
+
// doesn't match, so both allow one before the end.
|
|
14
|
+
const OPENING_FENCE_RE = /^( {0,3})(`{3,}|~{3,})(.*)\r?$/;
|
|
15
|
+
const CLOSING_FENCE_RE = /^( {0,3})(`+|~+)( *)\r?$/;
|
|
14
16
|
const INFO_SPLIT_RE = /^(\S*)\s*([\s\S]*)$/;
|
|
15
17
|
|
|
16
18
|
/**
|
|
@@ -138,6 +140,24 @@ export async function highlightFence({ code, lang, meta }) {
|
|
|
138
140
|
);
|
|
139
141
|
}
|
|
140
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Whether the line spanning `[lineStart, lineEnd)` of `text` can be a fence
|
|
145
|
+
* line at all: up to 3 spaces, then a backtick or tilde. A char-code check,
|
|
146
|
+
* so the (overwhelmingly common) prose or code line skips both the line
|
|
147
|
+
* slice and the fence regexes below - see bench/fence-splitter.bench.ts.
|
|
148
|
+
* @param {string} text
|
|
149
|
+
* @param {number} lineStart
|
|
150
|
+
* @param {number} lineEnd
|
|
151
|
+
* @returns {boolean}
|
|
152
|
+
*/
|
|
153
|
+
function mayBeFence(text, lineStart, lineEnd) {
|
|
154
|
+
const maxIndent = Math.min(lineStart + 3, lineEnd);
|
|
155
|
+
let p = lineStart;
|
|
156
|
+
while (p < maxIndent && text.charCodeAt(p) === 32 /* " " */) p += 1;
|
|
157
|
+
const code = text.charCodeAt(p);
|
|
158
|
+
return code === 96 /* "`" */ || code === 126 /* "~" */;
|
|
159
|
+
}
|
|
160
|
+
|
|
141
161
|
/**
|
|
142
162
|
* @param {string} line
|
|
143
163
|
* @returns {{ indent: number; char: "`" | "~"; len: number; info: string } | null}
|
|
@@ -167,14 +187,19 @@ function matchClosingFence(line, char, minLen) {
|
|
|
167
187
|
}
|
|
168
188
|
|
|
169
189
|
/**
|
|
170
|
-
*
|
|
190
|
+
* The line spanning `[lineStart, lineEnd)` of `text`, minus up to `indent`
|
|
191
|
+
* leading spaces.
|
|
192
|
+
* @param {string} text
|
|
193
|
+
* @param {number} lineStart
|
|
194
|
+
* @param {number} lineEnd
|
|
171
195
|
* @param {number} indent
|
|
172
196
|
* @returns {string}
|
|
173
197
|
*/
|
|
174
|
-
function stripIndent(
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
198
|
+
function stripIndent(text, lineStart, lineEnd, indent) {
|
|
199
|
+
const maxStrip = Math.min(lineStart + indent, lineEnd);
|
|
200
|
+
let p = lineStart;
|
|
201
|
+
while (p < maxStrip && text.charCodeAt(p) === 32 /* " " */) p += 1;
|
|
202
|
+
return text.slice(p, lineEnd);
|
|
178
203
|
}
|
|
179
204
|
|
|
180
205
|
/**
|
|
@@ -183,13 +208,75 @@ function stripIndent(line, indent) {
|
|
|
183
208
|
*/
|
|
184
209
|
|
|
185
210
|
/**
|
|
186
|
-
*
|
|
211
|
+
* An open fence being scanned, with `lineCount` newline-terminated content
|
|
212
|
+
* lines so far. An unindented fence's lines sit back to back in the text, so
|
|
213
|
+
* they are tracked as the span `[codeStart, codeEnd)` and sliced from the
|
|
214
|
+
* current buffer when needed. An indented fence strips each line, so `code`
|
|
215
|
+
* holds its lines already joined instead, and a resumed scan extends it
|
|
216
|
+
* rather than re-joining every line.
|
|
217
|
+
* @typedef {{ char: "`" | "~"; len: number; indent: number; info: string; start: number; code: string; codeStart: number; codeEnd: number; lineCount: number }} FenceContext
|
|
218
|
+
*/
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Where a scan stopped: `pos` is the start of the trailing line that wasn't
|
|
222
|
+
* newline-terminated yet (or the end of the text), and `proseStart`/`fence`
|
|
223
|
+
* describe the segment that line belongs to. Everything before `pos` is
|
|
224
|
+
* final - a newline-terminated line never changes on append - so `append`
|
|
225
|
+
* resumes from here instead of rescanning the whole last segment.
|
|
226
|
+
* @typedef {{ pos: number; proseStart: number; fence: FenceContext | null }} ScanState
|
|
227
|
+
*/
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* A copy of `slice` that owns only its own characters. A slice of the
|
|
231
|
+
* splitter's buffer is a substring that keeps the whole buffer alive - and
|
|
232
|
+
* since `append` grows a new buffer every chunk, each closed segment would
|
|
233
|
+
* pin its own full copy of the text as it stood when the segment closed
|
|
234
|
+
* (bench/markdown-stream.bench.ts --alloc: a finished 51 KB stream kept
|
|
235
|
+
* 1.26 MB alive, 865 KB with copies). `structuredClone` copies in both V8 and
|
|
236
|
+
* JSC at memcpy speed; rope tricks like `` ` ${s}`.slice(1) `` don't, since
|
|
237
|
+
* JSC slices a rope's fiber without flattening it.
|
|
238
|
+
* @param {string} slice
|
|
239
|
+
* @returns {string}
|
|
240
|
+
*/
|
|
241
|
+
function ownCopy(slice) {
|
|
242
|
+
return structuredClone(slice);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* `ctx`'s newline-terminated content lines, joined. Slicing an unindented
|
|
247
|
+
* fence's lines from `text` keeps only the current buffer alive. Joining
|
|
248
|
+
* them as they arrive would make a rope of slices of every buffer `append`
|
|
249
|
+
* grew while the fence was open, pinning a full copy of the text as it
|
|
250
|
+
* stood each time a chunk finished a line.
|
|
251
|
+
* @param {FenceContext} ctx
|
|
252
|
+
* @param {string} text
|
|
253
|
+
* @returns {string}
|
|
254
|
+
*/
|
|
255
|
+
function fenceCode(ctx, text) {
|
|
256
|
+
if (ctx.indent > 0) return ctx.code;
|
|
257
|
+
return ctx.lineCount === 0 ? "" : text.slice(ctx.codeStart, ctx.codeEnd);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* @param {number} offset
|
|
262
|
+
* @returns {ScanState}
|
|
263
|
+
*/
|
|
264
|
+
function scanStateAt(offset) {
|
|
265
|
+
return { pos: offset, proseStart: offset, fence: null };
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* @param {FenceContext} ctx
|
|
270
|
+
* @param {string} text
|
|
187
271
|
* @param {number} id
|
|
188
272
|
* @param {number} end
|
|
189
273
|
* @param {boolean} open
|
|
274
|
+
* @param {string | undefined} partial Trailing content line not yet
|
|
275
|
+
* terminated by a newline, if any.
|
|
190
276
|
* @returns {FenceSegment}
|
|
191
277
|
*/
|
|
192
|
-
function buildFenceSegment(ctx, id, end, open) {
|
|
278
|
+
function buildFenceSegment(ctx, text, id, end, open, partial) {
|
|
279
|
+
const code = fenceCode(ctx, text);
|
|
193
280
|
const infoMatch = /** @type {RegExpExecArray} */ (
|
|
194
281
|
INFO_SPLIT_RE.exec(ctx.info)
|
|
195
282
|
);
|
|
@@ -200,7 +287,16 @@ function buildFenceSegment(ctx, id, end, open) {
|
|
|
200
287
|
lang: resolveLanguageName(ctx.info),
|
|
201
288
|
info: ctx.info,
|
|
202
289
|
meta: parseMeta(meta),
|
|
203
|
-
code:
|
|
290
|
+
code:
|
|
291
|
+
partial === undefined
|
|
292
|
+
? open
|
|
293
|
+
? code
|
|
294
|
+
: // Final, so copied (see `ownCopy`): an unindented body is a
|
|
295
|
+
// slice of the buffer, an indented one a rope of slices.
|
|
296
|
+
ownCopy(code)
|
|
297
|
+
: ctx.lineCount === 0
|
|
298
|
+
? partial
|
|
299
|
+
: `${code}\n${partial}`,
|
|
204
300
|
open,
|
|
205
301
|
start: ctx.start,
|
|
206
302
|
end,
|
|
@@ -208,25 +304,30 @@ function buildFenceSegment(ctx, id, end, open) {
|
|
|
208
304
|
}
|
|
209
305
|
|
|
210
306
|
/**
|
|
211
|
-
* Parses `text` from `
|
|
212
|
-
* document (always begins in prose mode -
|
|
213
|
-
* never occurs mid-fence, only at a definite
|
|
307
|
+
* Parses `text` from `state.pos` onward. A fresh `scanStateAt(offset)`
|
|
308
|
+
* treats `offset` as the start of a document (always begins in prose mode -
|
|
309
|
+
* valid since a segment boundary never occurs mid-fence, only at a definite
|
|
310
|
+
* open/close/EOF point); a state returned by an earlier call resumes that
|
|
311
|
+
* scan where it stopped.
|
|
214
312
|
* @param {string} text
|
|
215
|
-
* @param {
|
|
313
|
+
* @param {ScanState} state
|
|
216
314
|
* @param {number} startId
|
|
217
|
-
* @returns {{ segments: MarkdownSegment[]; nextId: number }}
|
|
315
|
+
* @returns {{ segments: MarkdownSegment[]; nextId: number; state: ScanState }}
|
|
218
316
|
*/
|
|
219
|
-
function computeSegments(text,
|
|
317
|
+
function computeSegments(text, state, startId) {
|
|
220
318
|
/** @type {MarkdownSegment[]} */
|
|
221
319
|
const segments = [];
|
|
222
320
|
let id = startId;
|
|
223
321
|
const n = text.length;
|
|
224
|
-
let i =
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
322
|
+
let i = state.pos;
|
|
323
|
+
let proseStart = state.proseStart;
|
|
324
|
+
// Copied, since the content lines below extend it in place.
|
|
325
|
+
/** @type {FenceContext | null} */
|
|
326
|
+
let fenceCtx = state.fence === null ? null : { ...state.fence };
|
|
327
|
+
// Start of the trailing line not yet terminated by a newline, if any.
|
|
328
|
+
let resumePos = n;
|
|
329
|
+
/** @type {string | undefined} */
|
|
330
|
+
let partial;
|
|
230
331
|
|
|
231
332
|
while (i < n) {
|
|
232
333
|
const lineStart = i;
|
|
@@ -234,10 +335,12 @@ function computeSegments(text, offset, startId) {
|
|
|
234
335
|
const hasNewline = nlIndex !== -1;
|
|
235
336
|
const lineEnd = hasNewline ? nlIndex : n;
|
|
236
337
|
const lineEndIncl = hasNewline ? nlIndex + 1 : n;
|
|
237
|
-
|
|
338
|
+
if (!hasNewline) resumePos = lineStart;
|
|
238
339
|
|
|
239
|
-
if (
|
|
240
|
-
const open =
|
|
340
|
+
if (fenceCtx === null) {
|
|
341
|
+
const open = mayBeFence(text, lineStart, lineEnd)
|
|
342
|
+
? matchOpeningFence(text.slice(lineStart, lineEnd))
|
|
343
|
+
: null;
|
|
241
344
|
if (open !== null) {
|
|
242
345
|
if (!hasNewline) {
|
|
243
346
|
// Opening line not yet terminated by a newline: stays prose until
|
|
@@ -248,7 +351,11 @@ function computeSegments(text, offset, startId) {
|
|
|
248
351
|
segments.push({
|
|
249
352
|
id: id++,
|
|
250
353
|
kind: "text",
|
|
251
|
-
|
|
354
|
+
// Closed for good, so it gets its own copy (see `ownCopy`). The
|
|
355
|
+
// open tail below stays a cheap slice: it is rebuilt on every
|
|
356
|
+
// append and only ever pins the current buffer (as does an open
|
|
357
|
+
// unindented fence's code, see `fenceCode`).
|
|
358
|
+
text: ownCopy(text.slice(proseStart, lineStart)),
|
|
252
359
|
start: proseStart,
|
|
253
360
|
end: lineStart,
|
|
254
361
|
});
|
|
@@ -257,23 +364,33 @@ function computeSegments(text, offset, startId) {
|
|
|
257
364
|
char: open.char,
|
|
258
365
|
len: open.len,
|
|
259
366
|
indent: open.indent,
|
|
260
|
-
|
|
367
|
+
// Copied once (see `ownCopy`), so the info, meta and title of
|
|
368
|
+
// every segment built from this fence don't pin the buffer.
|
|
369
|
+
info: ownCopy(open.info),
|
|
261
370
|
start: lineStart,
|
|
262
|
-
|
|
371
|
+
code: "",
|
|
372
|
+
codeStart: 0,
|
|
373
|
+
codeEnd: 0,
|
|
374
|
+
lineCount: 0,
|
|
263
375
|
};
|
|
264
|
-
mode = "fence";
|
|
265
|
-
i = lineEndIncl;
|
|
266
|
-
continue;
|
|
267
376
|
}
|
|
268
377
|
i = lineEndIncl;
|
|
269
378
|
continue;
|
|
270
379
|
}
|
|
271
380
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
381
|
+
if (
|
|
382
|
+
hasNewline &&
|
|
383
|
+
mayBeFence(text, lineStart, lineEnd) &&
|
|
384
|
+
matchClosingFence(
|
|
385
|
+
text.slice(lineStart, lineEnd),
|
|
386
|
+
fenceCtx.char,
|
|
387
|
+
fenceCtx.len,
|
|
388
|
+
)
|
|
389
|
+
) {
|
|
390
|
+
segments.push(
|
|
391
|
+
buildFenceSegment(fenceCtx, text, id++, lineEndIncl, false, undefined),
|
|
392
|
+
);
|
|
393
|
+
fenceCtx = null;
|
|
277
394
|
proseStart = lineEndIncl;
|
|
278
395
|
i = lineEndIncl;
|
|
279
396
|
continue;
|
|
@@ -281,20 +398,24 @@ function computeSegments(text, offset, startId) {
|
|
|
281
398
|
// A same-line-terminated closer requires a newline (like an opener, a
|
|
282
399
|
// still-unterminated closer-looking line might grow into content or a
|
|
283
400
|
// longer/shorter run with the next chunk) - anything else is content.
|
|
284
|
-
|
|
285
|
-
if (!hasNewline)
|
|
401
|
+
const content = stripIndent(text, lineStart, lineEnd, fenceCtx.indent);
|
|
402
|
+
if (!hasNewline) {
|
|
403
|
+
partial = content;
|
|
404
|
+
break;
|
|
405
|
+
}
|
|
406
|
+
if (fenceCtx.indent > 0) {
|
|
407
|
+
fenceCtx.code =
|
|
408
|
+
fenceCtx.lineCount === 0 ? content : `${fenceCtx.code}\n${content}`;
|
|
409
|
+
} else {
|
|
410
|
+
if (fenceCtx.lineCount === 0) fenceCtx.codeStart = lineStart;
|
|
411
|
+
fenceCtx.codeEnd = lineEnd;
|
|
412
|
+
}
|
|
413
|
+
fenceCtx.lineCount += 1;
|
|
286
414
|
i = lineEndIncl;
|
|
287
415
|
}
|
|
288
416
|
|
|
289
|
-
if (
|
|
290
|
-
segments.push(
|
|
291
|
-
buildFenceSegment(
|
|
292
|
-
/** @type {NonNullable<typeof fenceCtx>} */ (fenceCtx),
|
|
293
|
-
id++,
|
|
294
|
-
n,
|
|
295
|
-
true,
|
|
296
|
-
),
|
|
297
|
-
);
|
|
417
|
+
if (fenceCtx !== null) {
|
|
418
|
+
segments.push(buildFenceSegment(fenceCtx, text, id++, n, true, partial));
|
|
298
419
|
} else if (proseStart < n) {
|
|
299
420
|
segments.push({
|
|
300
421
|
id: id++,
|
|
@@ -305,7 +426,11 @@ function computeSegments(text, offset, startId) {
|
|
|
305
426
|
});
|
|
306
427
|
}
|
|
307
428
|
|
|
308
|
-
return {
|
|
429
|
+
return {
|
|
430
|
+
segments,
|
|
431
|
+
nextId: id,
|
|
432
|
+
state: { pos: resumePos, proseStart, fence: fenceCtx },
|
|
433
|
+
};
|
|
309
434
|
}
|
|
310
435
|
|
|
311
436
|
/**
|
|
@@ -316,28 +441,41 @@ export function createFenceSplitter() {
|
|
|
316
441
|
/** @type {MarkdownSegment[]} */
|
|
317
442
|
let segments = [];
|
|
318
443
|
let nextId = 1;
|
|
444
|
+
/** @type {ScanState} */
|
|
445
|
+
let scanState = scanStateAt(0);
|
|
319
446
|
|
|
320
447
|
return {
|
|
321
448
|
append(chunk) {
|
|
322
449
|
if (chunk === "") return;
|
|
323
450
|
text += chunk;
|
|
324
451
|
|
|
452
|
+
// The segment `scanState` stopped in is the last one - unless the
|
|
453
|
+
// text ended right after a closing fence line, in which case the scan
|
|
454
|
+
// starts a brand-new segment and every existing one is final.
|
|
325
455
|
const last = segments[segments.length - 1];
|
|
326
|
-
const
|
|
327
|
-
|
|
456
|
+
const tailStart =
|
|
457
|
+
scanState.fence === null ? scanState.proseStart : scanState.fence.start;
|
|
458
|
+
const continuesLast = last !== undefined && last.start === tailStart;
|
|
459
|
+
// Skipping an id when not continuing keeps ids identical to a rescan
|
|
460
|
+
// from `last.start`, which re-emits `last` and discards its new id.
|
|
461
|
+
const result = computeSegments(
|
|
462
|
+
text,
|
|
463
|
+
scanState,
|
|
464
|
+
last !== undefined && !continuesLast ? nextId + 1 : nextId,
|
|
465
|
+
);
|
|
328
466
|
nextId = result.nextId;
|
|
467
|
+
scanState = result.state;
|
|
329
468
|
|
|
330
|
-
if (result.segments.length > 0 &&
|
|
469
|
+
if (result.segments.length > 0 && continuesLast) {
|
|
331
470
|
result.segments[0] = {
|
|
332
471
|
.../** @type {MarkdownSegment} */ (result.segments[0]),
|
|
333
472
|
id: last.id,
|
|
334
473
|
};
|
|
335
474
|
}
|
|
336
475
|
|
|
337
|
-
segments =
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
: [...segments.slice(0, -1), ...result.segments];
|
|
476
|
+
segments = continuesLast
|
|
477
|
+
? [...segments.slice(0, -1), ...result.segments]
|
|
478
|
+
: [...segments, ...result.segments];
|
|
341
479
|
},
|
|
342
480
|
|
|
343
481
|
set(newText) {
|
|
@@ -361,8 +499,9 @@ export function createFenceSplitter() {
|
|
|
361
499
|
reuseCandidate === undefined ? 0 : reuseCandidate.start;
|
|
362
500
|
|
|
363
501
|
text = newText;
|
|
364
|
-
const result = computeSegments(text, rebuildFrom, nextId);
|
|
502
|
+
const result = computeSegments(text, scanStateAt(rebuildFrom), nextId);
|
|
365
503
|
nextId = result.nextId;
|
|
504
|
+
scanState = result.state;
|
|
366
505
|
|
|
367
506
|
const first = result.segments[0];
|
|
368
507
|
if (
|
|
@@ -390,6 +529,7 @@ export function createFenceSplitter() {
|
|
|
390
529
|
text = "";
|
|
391
530
|
segments = [];
|
|
392
531
|
nextId = 1;
|
|
532
|
+
scanState = scanStateAt(0);
|
|
393
533
|
},
|
|
394
534
|
};
|
|
395
535
|
}
|
package/highlight-theme.js
CHANGED
|
@@ -97,37 +97,43 @@ export function highlightRules(theme) {
|
|
|
97
97
|
const css = match ? (match[2] ?? "") : theme;
|
|
98
98
|
|
|
99
99
|
let out = "";
|
|
100
|
+
// Comments are dropped from the prelude, so it's assembled from the
|
|
101
|
+
// slices between them (`runStart` marks the current one) rather than a
|
|
102
|
+
// char at a time (see bench/scoped.bench.ts).
|
|
100
103
|
let prelude = "";
|
|
104
|
+
let runStart = 0;
|
|
101
105
|
let i = 0;
|
|
102
106
|
while (i < css.length) {
|
|
103
107
|
const ch = css[i];
|
|
104
108
|
if (ch === "/" && css[i + 1] === "*") {
|
|
109
|
+
prelude += css.slice(runStart, i);
|
|
105
110
|
const end = css.indexOf("*/", i + 2);
|
|
106
111
|
i = end === -1 ? css.length : end + 2;
|
|
112
|
+
runStart = i;
|
|
107
113
|
continue;
|
|
108
114
|
}
|
|
109
115
|
if (ch === '"' || ch === "'") {
|
|
110
|
-
|
|
111
|
-
prelude += css.slice(i, stop);
|
|
112
|
-
i = stop;
|
|
116
|
+
i = findStringEnd(css, i);
|
|
113
117
|
continue;
|
|
114
118
|
}
|
|
115
119
|
if (ch === "{") {
|
|
120
|
+
prelude += css.slice(runStart, i);
|
|
116
121
|
const bodyEnd = findBlockEnd(css, i + 1);
|
|
117
122
|
if (!prelude.trim().startsWith("@")) {
|
|
118
123
|
out += highlightRulesFor(prelude, css.slice(i + 1, bodyEnd));
|
|
119
124
|
}
|
|
120
125
|
prelude = "";
|
|
121
126
|
i = bodyEnd + 1;
|
|
127
|
+
runStart = i;
|
|
122
128
|
continue;
|
|
123
129
|
}
|
|
124
130
|
if (ch === ";") {
|
|
125
131
|
// Blockless at-rule (@import, etc.): nothing to convert.
|
|
126
132
|
prelude = "";
|
|
127
133
|
i += 1;
|
|
134
|
+
runStart = i;
|
|
128
135
|
continue;
|
|
129
136
|
}
|
|
130
|
-
prelude += ch;
|
|
131
137
|
i += 1;
|
|
132
138
|
}
|
|
133
139
|
return out;
|
package/incremental-tokenize.js
CHANGED
|
@@ -16,11 +16,56 @@ import { diffText } from "./text-diff.js";
|
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
|
+
* What a reparse kept from the previous parse: `events[0, head)` and its
|
|
20
|
+
* last `tail` events are the same objects as `from`'s first `head` and last
|
|
21
|
+
* `tail`. Lets a consumer holding output derived from `from` (e.g.
|
|
22
|
+
* `patchLineHtml`) find the changed span without comparing the event
|
|
23
|
+
* arrays.
|
|
24
|
+
* @typedef {{
|
|
25
|
+
* from: ScopeEvent[],
|
|
26
|
+
* head: number,
|
|
27
|
+
* tail: number,
|
|
28
|
+
* }} EventReuse
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* How many leading and trailing events an edit left as they were: the first
|
|
33
|
+
* `prefix` and last `suffix` of `events` are the same objects as in
|
|
34
|
+
* `prevEvents`, and the two never overlap on either side. Taken from
|
|
35
|
+
* `reuse` when it is relative to `prevEvents`, else found by comparing the
|
|
36
|
+
* arrays, which is O(document).
|
|
37
|
+
* @param {ScopeEvent[]} prevEvents
|
|
38
|
+
* @param {ScopeEvent[]} events
|
|
39
|
+
* @param {EventReuse} [reuse]
|
|
40
|
+
* @returns {{ prefix: number, suffix: number }}
|
|
41
|
+
*/
|
|
42
|
+
export function sharedEvents(prevEvents, events, reuse) {
|
|
43
|
+
if (reuse !== undefined && reuse.from === prevEvents) {
|
|
44
|
+
return { prefix: reuse.head, suffix: reuse.tail };
|
|
45
|
+
}
|
|
46
|
+
const prevCount = prevEvents.length;
|
|
47
|
+
const count = events.length;
|
|
48
|
+
const shared = Math.min(prevCount, count);
|
|
49
|
+
let prefix = 0;
|
|
50
|
+
while (prefix < shared && prevEvents[prefix] === events[prefix]) prefix++;
|
|
51
|
+
let suffix = 0;
|
|
52
|
+
while (
|
|
53
|
+
suffix < shared - prefix &&
|
|
54
|
+
prevEvents[prevCount - 1 - suffix] === events[count - 1 - suffix]
|
|
55
|
+
) {
|
|
56
|
+
suffix++;
|
|
57
|
+
}
|
|
58
|
+
return { prefix, suffix };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* `reuse` is set by `reparseIncremental` when it resumed from `previous`.
|
|
19
63
|
* @typedef {{
|
|
20
64
|
* code: string,
|
|
21
65
|
* language: string,
|
|
22
66
|
* events: ScopeEvent[],
|
|
23
67
|
* checkpoints: Snapshot[],
|
|
68
|
+
* reuse?: EventReuse,
|
|
24
69
|
* }} IncrementalParse
|
|
25
70
|
*/
|
|
26
71
|
|
|
@@ -77,6 +122,13 @@ function framesEqual(a, b) {
|
|
|
77
122
|
*/
|
|
78
123
|
export const CHECKPOINT_INTERVAL = 32;
|
|
79
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Shortest text past the resume checkpoint, in characters, for which
|
|
127
|
+
* `reparseIncremental` windows its scans (see `from.windowed` in
|
|
128
|
+
* engine.js's `createSession`).
|
|
129
|
+
*/
|
|
130
|
+
const WINDOWED_MIN_TAIL = 8192;
|
|
131
|
+
|
|
80
132
|
/**
|
|
81
133
|
* Full parse with a line checkpoint every `CHECKPOINT_INTERVAL` lines (and
|
|
82
134
|
* always at the document end). First paint or language change.
|
|
@@ -143,6 +195,21 @@ function findResumeIndex(checkpoints, maxPos) {
|
|
|
143
195
|
return best;
|
|
144
196
|
}
|
|
145
197
|
|
|
198
|
+
/**
|
|
199
|
+
* Pushes `source[from..]` onto `target`. The result used to be assembled by
|
|
200
|
+
* spreading a sliced prefix (`[...prefix, ...tail]`), which copied the
|
|
201
|
+
* O(document) prefix twice per edit; slicing once and pushing the rest
|
|
202
|
+
* copies it once (see bench/incremental.bench.ts).
|
|
203
|
+
* @param {ScopeEvent[]} target
|
|
204
|
+
* @param {ScopeEvent[]} source
|
|
205
|
+
* @param {number} from
|
|
206
|
+
*/
|
|
207
|
+
function appendEvents(target, source, from) {
|
|
208
|
+
for (let i = from; i < source.length; i++) {
|
|
209
|
+
target.push(/** @type {ScopeEvent} */ (source[i]));
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
146
213
|
/**
|
|
147
214
|
* Re-tokenizes `code` given the previous parse of the same language.
|
|
148
215
|
* @param {Registry} registry
|
|
@@ -165,12 +232,22 @@ export function reparseIncremental(registry, language, previous, code) {
|
|
|
165
232
|
|
|
166
233
|
// Prefix up to the resume checkpoint is unchanged (within diffText's common
|
|
167
234
|
// prefix). restore() clears the session event log, so reattach prefix events
|
|
168
|
-
// separately.
|
|
169
|
-
const
|
|
235
|
+
// separately, when the result is assembled (see `appendEvents`).
|
|
236
|
+
const prefixLength = resumeCheckpoint.eventCount;
|
|
170
237
|
// The full document is loaded so the tail can be walked with `advance()`
|
|
171
238
|
// (see parseIncremental); tokenization still only proceeds line by line.
|
|
239
|
+
// The parse usually re-converges within a few lines, so over a long tail
|
|
240
|
+
// scans are windowed rather than run to the end of `code`: otherwise a
|
|
241
|
+
// rule with no match nearby scanned the whole tail, and a one-character
|
|
242
|
+
// edit cost time in proportion to the document. Over a short tail
|
|
243
|
+
// (typing at the end, say) plain scans are cheaper than the windowed
|
|
244
|
+
// bookkeeping. See bench/incremental.bench.ts.
|
|
172
245
|
const session = registry.createSession(language, {
|
|
173
|
-
from: {
|
|
246
|
+
from: {
|
|
247
|
+
code,
|
|
248
|
+
snapshot: resumeCheckpoint,
|
|
249
|
+
windowed: code.length - resumeCheckpoint.pos > WINDOWED_MIN_TAIL,
|
|
250
|
+
},
|
|
174
251
|
});
|
|
175
252
|
const checkpoints = previous.checkpoints.slice(0, resumeIndex + 1);
|
|
176
253
|
|
|
@@ -231,7 +308,7 @@ export function reparseIncremental(registry, language, previous, code) {
|
|
|
231
308
|
// snap.eventCount is session-local; shift to index the combined array.
|
|
232
309
|
checkpoints.push({
|
|
233
310
|
...snap,
|
|
234
|
-
eventCount: snap.eventCount +
|
|
311
|
+
eventCount: snap.eventCount + prefixLength,
|
|
235
312
|
});
|
|
236
313
|
linesSinceCheckpoint = 0;
|
|
237
314
|
}
|
|
@@ -242,13 +319,11 @@ export function reparseIncremental(registry, language, previous, code) {
|
|
|
242
319
|
const oldCheckpoint = /** @type {Snapshot} */ (
|
|
243
320
|
previous.checkpoints[convergedAtOldIndex]
|
|
244
321
|
);
|
|
245
|
-
const events =
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
...previous.events.slice(oldCheckpoint.eventCount),
|
|
249
|
-
];
|
|
322
|
+
const events = previous.events.slice(0, prefixLength);
|
|
323
|
+
appendEvents(events, session.events(), 0);
|
|
324
|
+
appendEvents(events, previous.events, oldCheckpoint.eventCount);
|
|
250
325
|
const eventOffset =
|
|
251
|
-
|
|
326
|
+
prefixLength + session.events().length - oldCheckpoint.eventCount;
|
|
252
327
|
for (
|
|
253
328
|
let i = convergedAtOldIndex + 1;
|
|
254
329
|
i < previous.checkpoints.length;
|
|
@@ -261,14 +336,17 @@ export function reparseIncremental(registry, language, previous, code) {
|
|
|
261
336
|
eventCount: old.eventCount + eventOffset,
|
|
262
337
|
});
|
|
263
338
|
}
|
|
264
|
-
|
|
339
|
+
const reuse = {
|
|
340
|
+
from: previous.events,
|
|
341
|
+
head: prefixLength,
|
|
342
|
+
tail: previous.events.length - oldCheckpoint.eventCount,
|
|
343
|
+
};
|
|
344
|
+
return { code, language, events, checkpoints, reuse };
|
|
265
345
|
}
|
|
266
346
|
|
|
267
347
|
const tail = session.finish();
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
checkpoints,
|
|
273
|
-
};
|
|
348
|
+
const events = previous.events.slice(0, prefixLength);
|
|
349
|
+
appendEvents(events, tail.events, 0);
|
|
350
|
+
const reuse = { from: previous.events, head: prefixLength, tail: 0 };
|
|
351
|
+
return { code, language, events, checkpoints, reuse };
|
|
274
352
|
}
|
package/package.json
CHANGED
package/preprocessor.js
CHANGED
|
@@ -508,6 +508,21 @@ export function highlightStatic(options = {}) {
|
|
|
508
508
|
.filter((match) => match !== null);
|
|
509
509
|
if (matches.length === 0) return;
|
|
510
510
|
|
|
511
|
+
// One import() per distinct language source per file. Every match on
|
|
512
|
+
// a file usually shares a few languages, and each import() call pays
|
|
513
|
+
// module-resolution overhead even when the module is cached. A
|
|
514
|
+
// rejected import still warns once per match, as before.
|
|
515
|
+
/** @type {Map<string, Promise<any>>} */
|
|
516
|
+
const languageModules = new Map();
|
|
517
|
+
const loadLanguageModule = (/** @type {string} */ source) => {
|
|
518
|
+
let loaded = languageModules.get(source);
|
|
519
|
+
if (!loaded) {
|
|
520
|
+
loaded = resolveLanguageModule(source, filename);
|
|
521
|
+
languageModules.set(source, loaded);
|
|
522
|
+
}
|
|
523
|
+
return loaded;
|
|
524
|
+
};
|
|
525
|
+
|
|
511
526
|
const htmlByMatch = await Promise.all(
|
|
512
527
|
matches.map(async (match) => {
|
|
513
528
|
// Only ever read by the warn() paths below, and locate() scans
|
|
@@ -518,9 +533,8 @@ export function highlightStatic(options = {}) {
|
|
|
518
533
|
/** @type {import("./languages").LanguageType<string>} */
|
|
519
534
|
let language;
|
|
520
535
|
try {
|
|
521
|
-
const languageModule = await
|
|
536
|
+
const languageModule = await loadLanguageModule(
|
|
522
537
|
match.languageSource,
|
|
523
|
-
filename,
|
|
524
538
|
);
|
|
525
539
|
language = languageModule.default ?? languageModule;
|
|
526
540
|
} catch (cause) {
|