renor-engine 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +27 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +3 -3
  4. package/dist/cli.js +1656 -490
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -548
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/preferenceExtractor.js +0 -150
  75. package/dist/extraction/secretGuard.js +0 -176
  76. package/dist/extraction/secretScanner.js +0 -665
  77. package/dist/extraction/typedFacts.js +0 -224
  78. package/dist/extraction/writePathAdjudication.js +0 -110
  79. package/dist/importing/apply.js +0 -672
  80. package/dist/importing/bundle-write.js +0 -80
  81. package/dist/importing/cli.js +0 -408
  82. package/dist/importing/existing.js +0 -51
  83. package/dist/importing/facts.js +0 -121
  84. package/dist/importing/forgetImported.js +0 -79
  85. package/dist/importing/hosted.js +0 -467
  86. package/dist/importing/journalLog.js +0 -96
  87. package/dist/importing/markdown.js +0 -98
  88. package/dist/importing/plan.js +0 -201
  89. package/dist/importing/secretScreen.js +0 -113
  90. package/dist/importing/serialize.js +0 -188
  91. package/dist/importing/source.js +0 -463
  92. package/dist/importing/types.js +0 -6
  93. package/dist/indexer/chunker.js +0 -319
  94. package/dist/indexer/dailyEntries.js +0 -63
  95. package/dist/indexer/hasher.js +0 -10
  96. package/dist/indexer/laneRuns.js +0 -48
  97. package/dist/indexer/o200k.js +0 -481
  98. package/dist/indexer/outsideRoot.js +0 -77
  99. package/dist/indexer/parser.js +0 -170
  100. package/dist/indexer/pipeline.js +0 -988
  101. package/dist/indexer/staleness.js +0 -201
  102. package/dist/indexer/tokenizer.js +0 -77
  103. package/dist/indexer/walker.js +0 -170
  104. package/dist/mcp/hive.js +0 -88
  105. package/dist/mcp/requestErrors.js +0 -47
  106. package/dist/mcp/server.js +0 -426
  107. package/dist/memory/agentLease.js +0 -43
  108. package/dist/memory/confinedWrite.js +0 -315
  109. package/dist/memory/controls.js +0 -73
  110. package/dist/memory/dailyNote.js +0 -265
  111. package/dist/memory/durableAppend.js +0 -57
  112. package/dist/memory/factConflicts.js +0 -203
  113. package/dist/memory/factTerms.js +0 -321
  114. package/dist/memory/handoff.js +0 -363
  115. package/dist/memory/hiveNameHeader.cjs +0 -10
  116. package/dist/memory/inventory.js +0 -55
  117. package/dist/memory/invisibleText.js +0 -20
  118. package/dist/memory/lifecycle.js +0 -1991
  119. package/dist/memory/limits.js +0 -13
  120. package/dist/memory/requests.js +0 -183
  121. package/dist/memory/sourceName.js +0 -276
  122. package/dist/memory/statusFacts.js +0 -61
  123. package/dist/memory/whatsNew.js +0 -158
  124. package/dist/memory/writer.js +0 -175
  125. package/dist/memory/writerConflicts.js +0 -85
  126. package/dist/memorymap/memoryMap.js +0 -1594
  127. package/dist/memorymap/replacedJournal.js +0 -58
  128. package/dist/memorymap/replacements.js +0 -410
  129. package/dist/mvp/activity.js +0 -86
  130. package/dist/mvp/agentMemory.js +0 -1800
  131. package/dist/mvp/answerCheck.js +0 -211
  132. package/dist/mvp/auth.js +0 -531
  133. package/dist/mvp/cli.js +0 -75
  134. package/dist/mvp/client-store.js +0 -462
  135. package/dist/mvp/compileCache.js +0 -242
  136. package/dist/mvp/compileSeed.js +0 -33
  137. package/dist/mvp/config.js +0 -288
  138. package/dist/mvp/connectionStatus.js +0 -80
  139. package/dist/mvp/contextDistinct.js +0 -53
  140. package/dist/mvp/contextProfile.js +0 -87
  141. package/dist/mvp/contextRelevance.js +0 -413
  142. package/dist/mvp/directorySurface.js +0 -68
  143. package/dist/mvp/engine.js +0 -1078
  144. package/dist/mvp/exportAll.js +0 -374
  145. package/dist/mvp/factLanePack.js +0 -77
  146. package/dist/mvp/factMarkers.js +0 -65
  147. package/dist/mvp/factsLane.js +0 -203
  148. package/dist/mvp/hiddenSpans.js +0 -585
  149. package/dist/mvp/history.js +0 -414
  150. package/dist/mvp/hitDates.js +0 -31
  151. package/dist/mvp/hitFloor.js +0 -36
  152. package/dist/mvp/identity.js +0 -18
  153. package/dist/mvp/idle.js +0 -113
  154. package/dist/mvp/log.js +0 -118
  155. package/dist/mvp/memoryDir.js +0 -98
  156. package/dist/mvp/posthogMcp.js +0 -210
  157. package/dist/mvp/readScope.js +0 -104
  158. package/dist/mvp/readUsefulness.js +0 -222
  159. package/dist/mvp/sanitize.js +0 -48
  160. package/dist/mvp/schemaRejections.js +0 -91
  161. package/dist/mvp/scope.js +0 -127
  162. package/dist/mvp/server.js +0 -2604
  163. package/dist/mvp/serverCard.js +0 -38
  164. package/dist/mvp/storedSize.js +0 -139
  165. package/dist/mvp/telemetry.js +0 -474
  166. package/dist/net/poolFetch.js +0 -174
  167. package/dist/recall/actionRecall.js +0 -282
  168. package/dist/recall/autoRecall.js +0 -447
  169. package/dist/recall/searchRecall.js +0 -189
  170. package/dist/retrieval/adviceLines.js +0 -195
  171. package/dist/retrieval/adviceQuestion.js +0 -44
  172. package/dist/retrieval/conversationShape.js +0 -44
  173. package/dist/retrieval/dailyEntryTime.js +0 -55
  174. package/dist/retrieval/entityLookup.js +0 -162
  175. package/dist/retrieval/factsFirst.js +0 -188
  176. package/dist/retrieval/keyword.js +0 -84
  177. package/dist/retrieval/ledgerEventTime.js +0 -51
  178. package/dist/retrieval/ledgerLiveness.js +0 -83
  179. package/dist/retrieval/ledgerPresentation.js +0 -161
  180. package/dist/retrieval/noteIdentity.js +0 -57
  181. package/dist/retrieval/orchestrator.js +0 -1171
  182. package/dist/retrieval/plainWording.js +0 -236
  183. package/dist/retrieval/queryHelpers.js +0 -142
  184. package/dist/retrieval/questionSpan.js +0 -475
  185. package/dist/retrieval/readerEvents.js +0 -690
  186. package/dist/retrieval/readerLayout.js +0 -422
  187. package/dist/retrieval/relativeTimeHints.js +0 -113
  188. package/dist/retrieval/semantic.js +0 -67
  189. package/dist/router/routerCache.js +0 -106
  190. package/dist/router/sessionRouter.js +0 -726
  191. package/dist/toolRefusal.js +0 -120
  192. package/dist/watcher/watcher.js +0 -77
  193. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,585 +0,0 @@
1
- "use strict";
2
- /**
3
- * The spans of a note's text that a rendered Markdown view does not show a reader, labelled in the text an agent
4
- * reads. A note is untrusted text: what the viewer hides (an HTML comment, a link title, a reference definition, image
5
- * alt text, a footnote, a collapsed <details> body, the info string of a code fence, text after a long run of spaces
6
- * or blank lines) reaches an agent verbatim, and nothing in the reply said it was hidden. The label keeps the text and
7
- * says where it was hidden, so the agent can tell what a reader of the note would not have seen.
8
- *
9
- * Only the agent-facing copy is changed: the stored note and export_memory keep every character. Text inside a fenced
10
- * code block or an inline code span is code, not hidden, and is left alone. A text with no hidden span comes back as
11
- * the same string, so a reply about such a note is the same reply it was before.
12
- *
13
- * The engine has no Markdown syntax tree, so this is a scanner for the constructs above. It reads the text in one
14
- * pass, as a renderer would: a backslash escape, a code span or fence is opaque, and each other construct is matched
15
- * where it starts. Its lookups are precomputed once per text (bracket pairs, next-character positions, backtick runs,
16
- * paragraph breaks), so the work is linear in the length of the text, and no construct is matched by a scan that can
17
- * run to the end of the text once per start. A collapsed <details> body or a link text that holds more labelled
18
- * constructs is scanned inside its own label, to MAX_NESTING levels; deeper, its text is kept as it is inside the label.
19
- *
20
- * Known gaps: a hidden span that the chunker cut across two chunks (a section over 400 tokens split at a blank line)
21
- * is labelled only where it opens; indented code blocks are not recognised as code; brackets inside a code span may pair
22
- * with brackets outside it.
23
- */
24
- Object.defineProperty(exports, "__esModule", { value: true });
25
- exports.MAX_NESTING = exports.LONG_BLANK_RUN = exports.LONG_SPACE_RUN = void 0;
26
- exports.labelHiddenSpans = labelHiddenSpans;
27
- exports.labelMemoryText = labelMemoryText;
28
- /** A run of this many spaces, after text on the same line and before text on that line, hides what follows it. */
29
- exports.LONG_SPACE_RUN = 40;
30
- /** A run of this many blank lines, between text, hides what follows it. */
31
- exports.LONG_BLANK_RUN = 10;
32
- /** How deep a labelled body or link text is scanned inside its own label. Past it the text is kept as it is. */
33
- exports.MAX_NESTING = 32;
34
- const PUNCTUATION = /[!-\/:-@\[-`{-~]/;
35
- const FENCE_OPEN = /( {0,3})(`{3,}|~{3,})([^\n]*)/y;
36
- const FOOTNOTE_CONTINUE = /\n(?:[ \t\r]*\n)*(?: {4}|\t)[^\n]*/y;
37
- const SUMMARY = /^\s*<summary\b[^>]*>[\s\S]*?<\/summary\s*>/i;
38
- const DETAILS_TAG = /<(\/?)details\b/gi;
39
- const OPEN_ATTRIBUTE = /(?:^|\s)open(?=[\s=/]|$)/i;
40
- function verbatim(text, from, to) {
41
- return { end: to, out: text.slice(from, to), labelled: false };
42
- }
43
- function label(kind, body) {
44
- const inner = body.trim();
45
- return inner ? `[${kind}: ${inner}]` : `[${kind}]`;
46
- }
47
- function isBlank(c) {
48
- return c === ' ' || c === '\t' || c === '\r';
49
- }
50
- /**
51
- * What a text must hold for anything in it to be labelled. Each labelled step starts at one of these: a `<` (a comment, a
52
- * <details> block), a `[` (a link title, an image, a reference definition, a footnote), a backtick or a tilde (a code fence,
53
- * whose info string can hold hidden words), forty spaces in a row, or eleven line breaks with nothing but blanks between them
54
- * (ten blank lines). A text with none of them has no hidden span, and comes back as it is: found by this one test, instead of
55
- * by the scanner's step object and string for every character of it (about 17% of what a `get_context` allocated).
56
- */
57
- const MAY_HIDE = new RegExp(`[<\\[\`~]| {${exports.LONG_SPACE_RUN}}|\\n(?:[ \\t\\r]*\\n){${exports.LONG_BLANK_RUN}}`);
58
- /**
59
- * The agent-facing copy of `text`: each hidden span labelled in place, the rest unchanged. Returns `text` itself when
60
- * nothing in it is hidden.
61
- */
62
- function labelHiddenSpans(text) {
63
- if (!MAY_HIDE.test(text))
64
- return text;
65
- return new Labeller(text, 0).run();
66
- }
67
- /** A memory's text labelled; the memory itself when nothing in its text is hidden. */
68
- function labelMemoryText(memory) {
69
- if (typeof memory?.text !== 'string')
70
- return memory;
71
- const text = labelHiddenSpans(memory.text);
72
- return text === memory.text ? memory : { ...memory, text };
73
- }
74
- function labelIn(text, depth) {
75
- return new Labeller(text, depth).run();
76
- }
77
- class Labeller {
78
- text;
79
- depth;
80
- n;
81
- tables = new Map();
82
- escaped;
83
- closers;
84
- breaks;
85
- ticks;
86
- fences;
87
- constructor(text, depth) {
88
- this.text = text;
89
- this.depth = depth;
90
- this.n = text.length;
91
- }
92
- run() {
93
- const text = this.text;
94
- let out = '';
95
- let changed = false;
96
- let i = 0;
97
- while (i < this.n) {
98
- if (i === 0 || text[i - 1] === '\n') {
99
- const block = this.lineStart(i);
100
- if (block) {
101
- out += block.out;
102
- changed ||= block.labelled;
103
- i = block.end;
104
- continue;
105
- }
106
- }
107
- const step = this.inline(i);
108
- out += step.out;
109
- changed ||= step.labelled;
110
- i = step.end;
111
- }
112
- return changed ? out : text;
113
- }
114
- /** What starts at `at` on a line that is not the start of a block. Anything that opens no hidden span is copied. */
115
- inline(at) {
116
- const t = this.text;
117
- const c = t[at];
118
- if (c === '\\') {
119
- const next = t[at + 1];
120
- return verbatim(t, at, next !== undefined && PUNCTUATION.test(next) ? at + 2 : at + 1);
121
- }
122
- if (c === '\n')
123
- return this.newlines(at);
124
- if (c === ' ')
125
- return this.spaces(at);
126
- if (c === '`')
127
- return this.codeSpan(at);
128
- if (c === '<') {
129
- if (t.startsWith('<!--', at))
130
- return this.comment(at);
131
- if (this.isDetailsOpen(at))
132
- return this.details(at);
133
- return verbatim(t, at, at + 1);
134
- }
135
- if (c === '[')
136
- return this.link(at, at, false);
137
- if (c === '!' && t[at + 1] === '[')
138
- return this.link(at, at + 1, true);
139
- return verbatim(t, at, at + 1);
140
- }
141
- /** A run of blank lines that hides the text after it: labelled with a marker, the text is kept. Scanned once. */
142
- newlines(at) {
143
- const t = this.text;
144
- let end = at + 1;
145
- let blank = 0;
146
- for (;;) {
147
- let k = end;
148
- while (k < this.n && isBlank(t[k]))
149
- k++;
150
- if (k < this.n && t[k] === '\n') {
151
- blank++;
152
- end = k + 1;
153
- }
154
- else
155
- break;
156
- }
157
- if (blank >= exports.LONG_BLANK_RUN) {
158
- let k = end;
159
- while (k < this.n && isBlank(t[k]))
160
- k++;
161
- if (k < this.n)
162
- return { end, out: '\n[hidden: long run of blank lines]\n', labelled: true };
163
- }
164
- return verbatim(t, at, end);
165
- }
166
- /** A run of spaces after text on one line, before text on that line: a marker when it is long, the text is kept. */
167
- spaces(at) {
168
- const t = this.text;
169
- if (at > 0 && (t[at - 1] === ' ' || t[at - 1] === '\t' || t[at - 1] === '\n'))
170
- return verbatim(t, at, at + 1);
171
- let end = at;
172
- while (end < this.n && t[end] === ' ')
173
- end++;
174
- const next = t[end];
175
- if (end - at >= exports.LONG_SPACE_RUN && next !== undefined && !/\s/.test(next)) {
176
- return { end, out: ' [hidden: long run of spaces] ', labelled: true };
177
- }
178
- return verbatim(t, at, end);
179
- }
180
- /** An inline code span is code: copied as it is. Unclosed backticks are plain text. */
181
- codeSpan(at) {
182
- const t = this.text;
183
- let end = at;
184
- while (end < this.n && t[end] === '`')
185
- end++;
186
- const length = end - at;
187
- const starts = this.tickRuns().get(length) ?? [];
188
- let lo = 0;
189
- let hi = starts.length;
190
- while (lo < hi) {
191
- const mid = (lo + hi) >> 1;
192
- if (starts[mid] < end)
193
- lo = mid + 1;
194
- else
195
- hi = mid;
196
- }
197
- const start = starts[lo];
198
- if (start !== undefined && this.breakFrom(end) >= start)
199
- return verbatim(t, at, start + length);
200
- return verbatim(t, at, end);
201
- }
202
- comment(at) {
203
- const close = this.text.indexOf('-->', at + 4);
204
- const end = close < 0 ? this.n : close + 3;
205
- const inner = this.text.slice(at + 4, close < 0 ? this.n : close).trim();
206
- const out = inner ? `[hidden HTML comment: ${inner}]` : '[hidden HTML comment]';
207
- return { end, out, labelled: true };
208
- }
209
- isDetailsOpen(at) {
210
- const t = this.text;
211
- return t.slice(at, at + 8).toLowerCase() === '<details' && !/\w/.test(t[at + 8] ?? '');
212
- }
213
- /**
214
- * A <details> block that is collapsed: the summary is shown, the body is not, so the body is labelled and its own
215
- * hidden spans inside it are labelled the same way. A block marked open shows its body and is copied as it is.
216
- */
217
- details(at) {
218
- const t = this.text;
219
- const gt = this.nextOf('>', c => c === '>', at);
220
- if (gt >= this.n)
221
- return verbatim(t, at, at + 1);
222
- const tagEnd = gt + 1;
223
- if (OPEN_ATTRIBUTE.test(t.slice(at + 8, gt)))
224
- return verbatim(t, at, tagEnd);
225
- const close = this.detailsClose(tagEnd);
226
- const bodyEnd = close ? close.at : this.n;
227
- const body = t.slice(tagEnd, bodyEnd);
228
- const head = SUMMARY.exec(body)?.[0] ?? '';
229
- const core = body.slice(head.length).trim();
230
- if (!core)
231
- return verbatim(t, at, tagEnd);
232
- const inner = this.depth < exports.MAX_NESTING ? labelIn(core, this.depth + 1) : core;
233
- const closing = close ? t.slice(close.at, close.end) : '';
234
- const out = `${t.slice(at, tagEnd)}${head}${head ? ' ' : ''}[collapsed details: ${inner}]${closing}`;
235
- return { end: close ? close.end : this.n, out, labelled: true };
236
- }
237
- /**
238
- * The </details> that closes a block whose body starts at `from`: the one that brings the nesting back to zero, so a
239
- * nested block closes with its own end. A </details> inside a fenced code block is code and does not count.
240
- */
241
- detailsClose(from) {
242
- const re = new RegExp(DETAILS_TAG.source, 'gi');
243
- re.lastIndex = from;
244
- let depth = 1;
245
- for (let m = re.exec(this.text); m !== null; m = re.exec(this.text)) {
246
- if (this.inFence(m.index))
247
- continue;
248
- if (m[1] === '/') {
249
- const gt = this.nextOf('>', c => c === '>', m.index);
250
- if (gt >= this.n)
251
- continue;
252
- depth--;
253
- if (depth === 0)
254
- return { at: m.index, end: gt + 1 };
255
- }
256
- else {
257
- depth++;
258
- }
259
- }
260
- return null;
261
- }
262
- /** A link (or, with `start` before `lb`, an image) whose text is labelled: its alt text, or its title. */
263
- link(start, lb, image) {
264
- const t = this.text;
265
- const close = this.closeOf(lb);
266
- if (close < 0 || t[close + 1] !== '(')
267
- return verbatim(t, start, start + 1);
268
- const tail = this.linkTail(close + 2);
269
- if (!tail)
270
- return verbatim(t, start, start + 1);
271
- const words = t.slice(lb + 1, close);
272
- if (image) {
273
- const alt = words.trim();
274
- if (!alt && tail.title === undefined)
275
- return verbatim(t, start, start + 1);
276
- const head = alt ? `[image alt text: ${alt}]` : '![]';
277
- const title = tail.title === undefined ? '' : ` ${label('link title', tail.title)}`;
278
- return { end: tail.end, out: `${head}(${tail.dest})${title}`, labelled: true };
279
- }
280
- if (tail.title === undefined)
281
- return verbatim(t, start, start + 1);
282
- const inner = this.depth < exports.MAX_NESTING ? labelIn(words, this.depth + 1) : words;
283
- return { end: tail.end, out: `[${inner}](${tail.dest}) ${label('link title', tail.title)}`, labelled: true };
284
- }
285
- /** The destination and title of an inline link whose '(' is before `p`, and the position after its ')'. */
286
- linkTail(p) {
287
- const t = this.text;
288
- const k = this.skipWhitespace(p);
289
- const d = this.destination(k);
290
- if (!d)
291
- return null;
292
- let kk = this.skipWhitespace(d.end);
293
- let title;
294
- if (kk > d.end && this.isTitleOpener(kk)) {
295
- const r = this.title(kk);
296
- if (!r)
297
- return null;
298
- title = t.slice(r.start, r.close);
299
- kk = this.skipWhitespace(r.end);
300
- }
301
- if (t[kk] !== ')')
302
- return null;
303
- return { dest: t.slice(k, d.end), title, end: kk + 1 };
304
- }
305
- /** A destination: a <...> form on one line, or a run up to whitespace or ')'. */
306
- destination(k) {
307
- const t = this.text;
308
- if (t[k] === '<') {
309
- const gt = this.nextOf('>', c => c === '>', k + 1);
310
- if (gt >= this.n)
311
- return null;
312
- if (this.nextOf('\n', c => c === '\n', k) < gt)
313
- return null;
314
- return { end: gt + 1 };
315
- }
316
- return { end: this.nextOf('dest', c => /[\s)]/.test(c), k) };
317
- }
318
- /** A title in quotes or parentheses, which may run across lines: the text between its marks. */
319
- title(k) {
320
- const open = this.text[k];
321
- if (!this.isTitleOpener(k))
322
- return null;
323
- const closer = open === '(' ? ')' : open;
324
- let c = this.nextOf(closer, x => x === closer, k + 1);
325
- while (c < this.n && this.isEscaped(c))
326
- c = this.nextOf(closer, x => x === closer, c + 1);
327
- if (c >= this.n)
328
- return null;
329
- return { start: k + 1, close: c, end: c + 1 };
330
- }
331
- isTitleOpener(k) {
332
- const c = this.text[k];
333
- return c === '"' || c === "'" || c === '(';
334
- }
335
- /**
336
- * A reference definition at the start of a line: `[label]: destination` with an optional title on the same or the next
337
- * line. The whole definition is labelled, since a renderer shows none of it.
338
- */
339
- definition(p) {
340
- const t = this.text;
341
- const close = this.closeOf(p);
342
- if (close < 0 || t[close + 1] !== ':')
343
- return null;
344
- let k = close + 2;
345
- while (k < this.n && isBlank(t[k]))
346
- k++;
347
- if (t[k] === '\n') {
348
- k++;
349
- while (k < this.n && isBlank(t[k]))
350
- k++;
351
- }
352
- const dest = this.destination(k);
353
- if (!dest || dest.end === k)
354
- return null;
355
- let blockEnd = -1;
356
- const next = this.skipWhitespace(dest.end);
357
- if (next > dest.end && this.isTitleOpener(next)) {
358
- const r = this.title(next);
359
- if (r) {
360
- let e = r.end;
361
- while (e < this.n && isBlank(t[e]))
362
- e++;
363
- if (e >= this.n || t[e] === '\n')
364
- blockEnd = e;
365
- }
366
- }
367
- if (blockEnd < 0) {
368
- let e = dest.end;
369
- while (e < this.n && isBlank(t[e]))
370
- e++;
371
- if (e < this.n && t[e] !== '\n')
372
- return null;
373
- blockEnd = e;
374
- }
375
- const raw = t.slice(p, blockEnd).trim();
376
- return { end: blockEnd, out: `[reference definition: ${raw}]`, labelled: true };
377
- }
378
- /** A footnote definition at the start of a line: its text and the lines indented under it, labelled. */
379
- footnote(p) {
380
- const t = this.text;
381
- const close = this.closeOf(p);
382
- if (close < 0 || t[close + 1] !== ':')
383
- return null;
384
- let end = t.indexOf('\n', close + 2);
385
- if (end < 0)
386
- end = this.n;
387
- const parts = [t.slice(close + 2, end)];
388
- for (;;) {
389
- FOOTNOTE_CONTINUE.lastIndex = end;
390
- const more = FOOTNOTE_CONTINUE.exec(t);
391
- if (!more)
392
- break;
393
- parts.push(more[0]);
394
- end += more[0].length;
395
- }
396
- const body = parts.map(part => part.trim()).filter(Boolean).join(' ');
397
- if (!body)
398
- return null;
399
- return { end, out: `[footnote: ${body}]`, labelled: true };
400
- }
401
- /**
402
- * What starts a line: a code fence (whose content is code, and whose info string is labelled after the language word,
403
- * since a renderer shows the language and not the rest), a footnote, or a reference definition. Null when none does.
404
- */
405
- lineStart(at) {
406
- const t = this.text;
407
- FENCE_OPEN.lastIndex = at;
408
- const fence = FENCE_OPEN.exec(t);
409
- if (fence && !(fence[2][0] === '`' && fence[3].includes('`')))
410
- return this.fenced(at, fence);
411
- let p = at;
412
- while (p - at < 3 && t[p] === ' ')
413
- p++;
414
- if (t[p] === '[')
415
- return t[p + 1] === '^' ? this.footnote(p) : this.definition(p);
416
- return null;
417
- }
418
- fenced(at, open) {
419
- const t = this.text;
420
- const [, indent, marker, info] = open;
421
- const lineEnd = t.indexOf('\n', at);
422
- const trimmed = info.trim();
423
- const cut = trimmed.search(/\s/);
424
- const lang = cut < 0 ? trimmed : trimmed.slice(0, cut);
425
- const hidden = cut < 0 ? '' : trimmed.slice(cut).trim();
426
- const head = hidden
427
- ? `${indent}${marker}${info.match(/^\s*/)[0]}${lang} [code fence info: ${hidden}]`
428
- : open[0];
429
- if (lineEnd < 0)
430
- return { end: this.n, out: head, labelled: hidden !== '' };
431
- const ch = marker[0] === '`' ? '`' : '~';
432
- const closing = new RegExp('^ {0,3}' + ch + '{' + marker.length + ',}[ \\t\\r]*$');
433
- let end = this.n;
434
- for (let p = lineEnd + 1; p < this.n;) {
435
- let nl = t.indexOf('\n', p);
436
- if (nl < 0)
437
- nl = this.n;
438
- if (closing.test(t.slice(p, nl))) {
439
- end = nl;
440
- break;
441
- }
442
- p = nl + 1;
443
- }
444
- return { end, out: head + t.slice(at + open[0].length, end), labelled: hidden !== '' };
445
- }
446
- /** The position of the ']' that closes the '[' at `open`, or -1. Computed for the whole text once, in one pass. */
447
- closeOf(open) {
448
- if (!this.closers) {
449
- const pairs = new Int32Array(this.n).fill(-1);
450
- const stack = [];
451
- for (let i = 0; i < this.n; i++) {
452
- const c = this.text[i];
453
- if (c === '[' && !this.isEscaped(i))
454
- stack.push(i);
455
- else if (c === ']' && !this.isEscaped(i) && stack.length > 0)
456
- pairs[stack.pop()] = i;
457
- }
458
- this.closers = pairs;
459
- }
460
- return this.closers[open];
461
- }
462
- /** Whether the character at `at` is preceded by an odd number of backslashes, so it is escaped. */
463
- isEscaped(at) {
464
- if (!this.escaped) {
465
- const escaped = new Uint8Array(this.n);
466
- let run = 0;
467
- for (let i = 0; i < this.n; i++) {
468
- escaped[i] = run % 2;
469
- run = this.text[i] === '\\' ? run + 1 : 0;
470
- }
471
- this.escaped = escaped;
472
- }
473
- return this.escaped[at] === 1;
474
- }
475
- /** The first position at or after `from` whose character passes `test`, or the length of the text when none does. */
476
- nextOf(key, test, from) {
477
- let table = this.tables.get(key);
478
- if (table === undefined) {
479
- table = new Int32Array(this.n + 1);
480
- table[this.n] = this.n;
481
- for (let i = this.n - 1; i >= 0; i--)
482
- table[i] = test(this.text[i]) ? i : table[i + 1];
483
- this.tables.set(key, table);
484
- }
485
- return table[Math.min(from, this.n)];
486
- }
487
- skipWhitespace(k) {
488
- const t = this.text;
489
- while (k < this.n && (isBlank(t[k]) || t[k] === '\n'))
490
- k++;
491
- return k;
492
- }
493
- /** The first paragraph break at or after `at`: a '\n' that starts a blank line. The length of the text when none. */
494
- breakFrom(at) {
495
- if (!this.breaks) {
496
- const t = this.text;
497
- const isBreak = new Uint8Array(this.n);
498
- for (let i = 0; i < this.n; i++) {
499
- if (t[i] !== '\n')
500
- continue;
501
- let k = i + 1;
502
- while (k < this.n && isBlank(t[k]))
503
- k++;
504
- if (k < this.n && t[k] === '\n')
505
- isBreak[i] = 1;
506
- }
507
- const breaks = new Int32Array(this.n + 1);
508
- breaks[this.n] = this.n;
509
- for (let i = this.n - 1; i >= 0; i--)
510
- breaks[i] = isBreak[i] ? i : breaks[i + 1];
511
- this.breaks = breaks;
512
- }
513
- return this.breaks[Math.min(at, this.n)];
514
- }
515
- /** The start of each maximal backtick run, by its length, in order. */
516
- tickRuns() {
517
- if (!this.ticks) {
518
- const runs = new Map();
519
- const t = this.text;
520
- for (let i = 0; i < this.n;) {
521
- if (t[i] !== '`') {
522
- i++;
523
- continue;
524
- }
525
- let end = i;
526
- while (end < this.n && t[end] === '`')
527
- end++;
528
- const starts = runs.get(end - i) ?? [];
529
- starts.push(i);
530
- runs.set(end - i, starts);
531
- i = end;
532
- }
533
- this.ticks = runs;
534
- }
535
- return this.ticks;
536
- }
537
- /** Whether `at` lies inside a fenced code block. The blocks are the fence lines and what they enclose. */
538
- inFence(at) {
539
- if (!this.fences)
540
- this.fences = this.fenceRanges();
541
- const f = this.fences;
542
- let lo = 0;
543
- let hi = f.length / 2;
544
- while (lo < hi) {
545
- const mid = (lo + hi) >> 1;
546
- if (f[2 * mid] <= at)
547
- lo = mid + 1;
548
- else
549
- hi = mid;
550
- }
551
- const k = lo - 1;
552
- return k >= 0 && at < f[2 * k + 1];
553
- }
554
- /** The fenced blocks of the text, as pairs of start and end positions. */
555
- fenceRanges() {
556
- const t = this.text;
557
- const out = [];
558
- let open = null;
559
- let s = 0;
560
- for (;;) {
561
- let e = t.indexOf('\n', s);
562
- if (e < 0)
563
- e = this.n;
564
- const line = t.slice(s, e);
565
- if (open === null) {
566
- const m = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
567
- if (m && !(m[1][0] === '`' && m[2].includes('`')))
568
- open = { start: s, ch: m[1][0], len: m[1].length };
569
- }
570
- else {
571
- const m = /^ {0,3}(`+|~+)[ \t\r]*$/.exec(line);
572
- if (m && m[1][0] === open.ch && m[1].length >= open.len) {
573
- out.push(open.start, e);
574
- open = null;
575
- }
576
- }
577
- if (e >= this.n)
578
- break;
579
- s = e + 1;
580
- }
581
- if (open !== null)
582
- out.push(open.start, this.n);
583
- return out;
584
- }
585
- }