sensemaking 0.22.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -8
- package/dist/cjs/chunk/extract.d.cts +3 -3
- package/dist/cjs/chunk/extract.d.ts +3 -3
- package/dist/cjs/chunk/extract.js +369 -72
- package/dist/cjs/chunk/extract.js.map +1 -1
- package/dist/cjs/chunk/group.js +2 -2
- package/dist/cjs/chunk/group.js.map +1 -1
- package/dist/cjs/chunk/parse.js +63 -67
- package/dist/cjs/chunk/parse.js.map +1 -1
- package/dist/cjs/chunk/parser.d.cts +2 -0
- package/dist/cjs/chunk/parser.d.ts +2 -0
- package/dist/cjs/chunk/parser.js +40 -0
- package/dist/cjs/chunk/parser.js.map +1 -0
- package/dist/cjs/chunk/types.d.cts +2 -2
- package/dist/cjs/chunk/types.d.ts +2 -2
- package/dist/cjs/chunk/version.d.cts +1 -1
- package/dist/cjs/chunk/version.d.ts +1 -1
- package/dist/cjs/chunk/version.js +1 -1
- package/dist/cjs/chunk/version.js.map +1 -1
- package/dist/cjs/features/links.js +67 -71
- package/dist/cjs/features/links.js.map +1 -1
- package/dist/cjs/features/sections.js +1 -1
- package/dist/cjs/features/sections.js.map +1 -1
- package/dist/cjs/scan/pool.js +1 -1
- package/dist/cjs/scan/pool.js.map +1 -1
- package/dist/cjs/store/builder.js +2 -3
- package/dist/cjs/store/builder.js.map +1 -1
- package/dist/cjs/store/duckdb/open.d.cts +1 -1
- package/dist/cjs/store/duckdb/open.d.ts +1 -1
- package/dist/cjs/store/duckdb/open.js +1 -1
- package/dist/cjs/store/duckdb/open.js.map +1 -1
- package/dist/cjs/store/sqlite/open.d.cts +1 -1
- package/dist/cjs/store/sqlite/open.d.ts +1 -1
- package/dist/cjs/store/sqlite/open.js +1 -1
- package/dist/cjs/store/sqlite/open.js.map +1 -1
- package/dist/cjs/store/turso/connection.d.cts +1 -0
- package/dist/cjs/store/turso/connection.d.ts +1 -0
- package/dist/cjs/store/turso/connection.js +134 -13
- package/dist/cjs/store/turso/connection.js.map +1 -1
- package/dist/cjs/store/turso/open.d.cts +1 -1
- package/dist/cjs/store/turso/open.d.ts +1 -1
- package/dist/cjs/store/turso/open.js +8 -2
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/store.js +8 -1
- package/dist/cjs/store/turso/store.js.map +1 -1
- package/dist/cjs/text/strip.js +1 -3
- package/dist/cjs/text/strip.js.map +1 -1
- package/dist/cjs/workers/parse.js.map +1 -1
- package/dist/esm/chunk/extract.d.ts +3 -3
- package/dist/esm/chunk/extract.js +276 -63
- package/dist/esm/chunk/extract.js.map +1 -1
- package/dist/esm/chunk/group.js +2 -2
- package/dist/esm/chunk/group.js.map +1 -1
- package/dist/esm/chunk/parse.js +65 -64
- package/dist/esm/chunk/parse.js.map +1 -1
- package/dist/esm/chunk/parser.d.ts +2 -0
- package/dist/esm/chunk/parser.js +26 -0
- package/dist/esm/chunk/parser.js.map +1 -0
- package/dist/esm/chunk/types.d.ts +2 -2
- package/dist/esm/chunk/types.js.map +1 -1
- package/dist/esm/chunk/version.d.ts +1 -1
- package/dist/esm/chunk/version.js +1 -1
- package/dist/esm/chunk/version.js.map +1 -1
- package/dist/esm/features/links.js +59 -22
- package/dist/esm/features/links.js.map +1 -1
- package/dist/esm/features/sections.js +1 -1
- package/dist/esm/features/sections.js.map +1 -1
- package/dist/esm/scan/pool.js +1 -1
- package/dist/esm/scan/pool.js.map +1 -1
- package/dist/esm/store/builder.js +2 -3
- package/dist/esm/store/builder.js.map +1 -1
- package/dist/esm/store/duckdb/open.d.ts +1 -1
- package/dist/esm/store/duckdb/open.js +1 -1
- package/dist/esm/store/duckdb/open.js.map +1 -1
- package/dist/esm/store/sqlite/open.d.ts +1 -1
- package/dist/esm/store/sqlite/open.js +1 -1
- package/dist/esm/store/sqlite/open.js.map +1 -1
- package/dist/esm/store/turso/connection.d.ts +1 -0
- package/dist/esm/store/turso/connection.js +21 -8
- package/dist/esm/store/turso/connection.js.map +1 -1
- package/dist/esm/store/turso/open.d.ts +1 -1
- package/dist/esm/store/turso/open.js +3 -2
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/store.js +2 -0
- package/dist/esm/store/turso/store.js.map +1 -1
- package/dist/esm/text/strip.js +1 -1
- package/dist/esm/text/strip.js.map +1 -1
- package/dist/esm/workers/parse.js.map +1 -1
- package/package.json +5 -19
- package/skills/sense-setup/SKILL.md +1 -1
package/dist/esm/chunk/parse.js
CHANGED
|
@@ -1,71 +1,72 @@
|
|
|
1
|
-
import Module from 'node:module';
|
|
2
1
|
import { extractText } from './extract.js';
|
|
3
|
-
|
|
4
|
-
//
|
|
5
|
-
|
|
2
|
+
import { parser } from './parser.js';
|
|
3
|
+
// Opening token type to Block type. Everything else (rules, raw html, footnote definitions) is
|
|
4
|
+
// 'other', as mdast's BLOCK_TYPES mapped no entry for those nodes.
|
|
6
5
|
const BLOCK_TYPES = {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
6
|
+
heading_open: 'heading',
|
|
7
|
+
paragraph_open: 'paragraph',
|
|
8
|
+
fence: 'code',
|
|
9
|
+
code_block: 'code',
|
|
10
|
+
table_open: 'table',
|
|
11
|
+
ordered_list_open: 'list',
|
|
12
|
+
bullet_list_open: 'list',
|
|
13
|
+
blockquote_open: 'blockquote'
|
|
13
14
|
};
|
|
14
|
-
|
|
15
|
-
//
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
mdastExtensions: [
|
|
41
|
-
gfmAutolinkLiteralFromMarkdown(),
|
|
42
|
-
gfmFootnoteFromMarkdown(),
|
|
43
|
-
gfmStrikethroughFromMarkdown(),
|
|
44
|
-
gfmTableFromMarkdown(),
|
|
45
|
-
gfmTaskListItemFromMarkdown()
|
|
46
|
-
]
|
|
15
|
+
// Top-level blocks of a markdown body, typed and line-extent bounded from markdown-it's own
|
|
16
|
+
// token maps (never a regex guess). html and linkify on, with the footnote and task-list plugins.
|
|
17
|
+
export function parse(body) {
|
|
18
|
+
const tokens = parser().parse(body, {});
|
|
19
|
+
const lines = body.split('\n');
|
|
20
|
+
const blocks = [];
|
|
21
|
+
let i = 0;
|
|
22
|
+
while(i < tokens.length){
|
|
23
|
+
const token = tokens[i];
|
|
24
|
+
if (token.nesting === 0) {
|
|
25
|
+
var _BLOCK_TYPES_token_type;
|
|
26
|
+
blocks.push(makeBlock((_BLOCK_TYPES_token_type = BLOCK_TYPES[token.type]) !== null && _BLOCK_TYPES_token_type !== void 0 ? _BLOCK_TYPES_token_type : 'other', tokens, i, i + 1, lines));
|
|
27
|
+
i += 1;
|
|
28
|
+
} else if (token.nesting === 1) {
|
|
29
|
+
var _BLOCK_TYPES_token_type1;
|
|
30
|
+
let depth = 1;
|
|
31
|
+
let j = i + 1;
|
|
32
|
+
while(j < tokens.length && depth > 0){
|
|
33
|
+
depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;
|
|
34
|
+
j += 1;
|
|
35
|
+
}
|
|
36
|
+
blocks.push(makeBlock((_BLOCK_TYPES_token_type1 = BLOCK_TYPES[token.type]) !== null && _BLOCK_TYPES_token_type1 !== void 0 ? _BLOCK_TYPES_token_type1 : 'other', tokens, i, j, lines));
|
|
37
|
+
i = j;
|
|
38
|
+
} else {
|
|
39
|
+
// Defensive: the balanced scan above consumes every close belonging to an open.
|
|
40
|
+
i += 1;
|
|
47
41
|
}
|
|
48
|
-
}
|
|
49
|
-
return
|
|
42
|
+
}
|
|
43
|
+
return blocks;
|
|
50
44
|
}
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
endLine: position ? position.end.line : 1,
|
|
63
|
-
node
|
|
64
|
-
};
|
|
65
|
-
if (node.type === 'heading') {
|
|
66
|
-
block.depth = node.depth;
|
|
67
|
-
block.text = extractText(node);
|
|
45
|
+
// 1-based inclusive extent from the min/max of the tokens' maps (0-based half-open); a block
|
|
46
|
+
// whose tokens carry no map (an empty footnote definition) falls back to line 1. Trailing
|
|
47
|
+
// blank lines trim so a list's endLine lands where mdast's position ended it.
|
|
48
|
+
function makeBlock(type, tokens, i, j, lines) {
|
|
49
|
+
let start = Infinity;
|
|
50
|
+
let end = 0;
|
|
51
|
+
for(let k = i; k < j; k++){
|
|
52
|
+
const map = tokens[k].map;
|
|
53
|
+
if (map) {
|
|
54
|
+
start = Math.min(start, map[0]);
|
|
55
|
+
end = Math.max(end, map[1]);
|
|
68
56
|
}
|
|
69
|
-
|
|
70
|
-
|
|
57
|
+
}
|
|
58
|
+
const startLine = Number.isFinite(start) ? start + 1 : 1;
|
|
59
|
+
let endLine = end > start ? end : startLine;
|
|
60
|
+
while(endLine > startLine && lines[endLine - 1].trim() === '')endLine--;
|
|
61
|
+
const block = {
|
|
62
|
+
type,
|
|
63
|
+
startLine,
|
|
64
|
+
endLine,
|
|
65
|
+
node: tokens.slice(i, j)
|
|
66
|
+
};
|
|
67
|
+
if (type === 'heading') {
|
|
68
|
+
block.depth = Number(tokens[i].tag.slice(1));
|
|
69
|
+
block.text = extractText(block);
|
|
70
|
+
}
|
|
71
|
+
return block;
|
|
71
72
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import type { Token } from 'markdown-it';\nimport { extractText } from './extract.ts';\nimport { parser } from './parser.ts';\nimport type { Block, BlockType } from './types.ts';\n\n// Opening token type to Block type. Everything else (rules, raw html, footnote definitions) is\n// 'other', as mdast's BLOCK_TYPES mapped no entry for those nodes.\nconst BLOCK_TYPES: Record<string, BlockType> = {\n heading_open: 'heading',\n paragraph_open: 'paragraph',\n fence: 'code',\n code_block: 'code',\n table_open: 'table',\n ordered_list_open: 'list',\n bullet_list_open: 'list',\n blockquote_open: 'blockquote',\n};\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from markdown-it's own\n// token maps (never a regex guess). html and linkify on, with the footnote and task-list plugins.\nexport function parse(body: string): Block[] {\n const tokens = parser().parse(body, {});\n const lines = body.split('\\n');\n const blocks: Block[] = [];\n let i = 0;\n while (i < tokens.length) {\n const token = tokens[i];\n if (token.nesting === 0) {\n blocks.push(makeBlock(BLOCK_TYPES[token.type] ?? 'other', tokens, i, i + 1, lines));\n i += 1;\n } else if (token.nesting === 1) {\n let depth = 1;\n let j = i + 1;\n while (j < tokens.length && depth > 0) {\n depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;\n j += 1;\n }\n blocks.push(makeBlock(BLOCK_TYPES[token.type] ?? 'other', tokens, i, j, lines));\n i = j;\n } else {\n // Defensive: the balanced scan above consumes every close belonging to an open.\n i += 1;\n }\n }\n return blocks;\n}\n\n// 1-based inclusive extent from the min/max of the tokens' maps (0-based half-open); a block\n// whose tokens carry no map (an empty footnote definition) falls back to line 1. Trailing\n// blank lines trim so a list's endLine lands where mdast's position ended it.\nfunction makeBlock(type: BlockType, tokens: Token[], i: number, j: number, lines: string[]): Block {\n let start = Infinity;\n let end = 0;\n for (let k = i; k < j; k++) {\n const map = tokens[k].map;\n if (map) {\n start = Math.min(start, map[0]);\n end = Math.max(end, map[1]);\n }\n }\n const startLine = Number.isFinite(start) ? start + 1 : 1;\n let endLine = end > start ? end : startLine;\n while (endLine > startLine && lines[endLine - 1].trim() === '') endLine--;\n const block: Block = { type, startLine, endLine, node: tokens.slice(i, j) };\n if (type === 'heading') {\n block.depth = Number(tokens[i].tag.slice(1));\n block.text = extractText(block);\n }\n return block;\n}\n"],"names":["extractText","parser","BLOCK_TYPES","heading_open","paragraph_open","fence","code_block","table_open","ordered_list_open","bullet_list_open","blockquote_open","parse","body","tokens","lines","split","blocks","i","length","token","nesting","push","makeBlock","type","depth","j","start","Infinity","end","k","map","Math","min","max","startLine","Number","isFinite","endLine","trim","block","node","slice","tag","text"],"mappings":"AACA,SAASA,WAAW,QAAQ,eAAe;AAC3C,SAASC,MAAM,QAAQ,cAAc;AAGrC,+FAA+F;AAC/F,mEAAmE;AACnE,MAAMC,cAAyC;IAC7CC,cAAc;IACdC,gBAAgB;IAChBC,OAAO;IACPC,YAAY;IACZC,YAAY;IACZC,mBAAmB;IACnBC,kBAAkB;IAClBC,iBAAiB;AACnB;AAEA,4FAA4F;AAC5F,kGAAkG;AAClG,OAAO,SAASC,MAAMC,IAAY;IAChC,MAAMC,SAASZ,SAASU,KAAK,CAACC,MAAM,CAAC;IACrC,MAAME,QAAQF,KAAKG,KAAK,CAAC;IACzB,MAAMC,SAAkB,EAAE;IAC1B,IAAIC,IAAI;IACR,MAAOA,IAAIJ,OAAOK,MAAM,CAAE;QACxB,MAAMC,QAAQN,MAAM,CAACI,EAAE;QACvB,IAAIE,MAAMC,OAAO,KAAK,GAAG;gBACDlB;YAAtBc,OAAOK,IAAI,CAACC,WAAUpB,0BAAAA,WAAW,CAACiB,MAAMI,IAAI,CAAC,cAAvBrB,qCAAAA,0BAA2B,SAASW,QAAQI,GAAGA,IAAI,GAAGH;YAC5EG,KAAK;QACP,OAAO,IAAIE,MAAMC,OAAO,KAAK,GAAG;gBAORlB;YANtB,IAAIsB,QAAQ;YACZ,IAAIC,IAAIR,IAAI;YACZ,MAAOQ,IAAIZ,OAAOK,MAAM,IAAIM,QAAQ,EAAG;gBACrCA,SAASX,MAAM,CAACY,EAAE,CAACL,OAAO,KAAK,IAAI,IAAIP,MAAM,CAACY,EAAE,CAACL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI;gBACvEK,KAAK;YACP;YACAT,OAAOK,IAAI,CAACC,WAAUpB,2BAAAA,WAAW,CAACiB,MAAMI,IAAI,CAAC,cAAvBrB,sCAAAA,2BAA2B,SAASW,QAAQI,GAAGQ,GAAGX;YACxEG,IAAIQ;QACN,OAAO;YACL,gFAAgF;YAChFR,KAAK;QACP;IACF;IACA,OAAOD;AACT;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,8EAA8E;AAC9E,SAASM,UAAUC,IAAe,EAAEV,MAAe,EAAEI,CAAS,EAAEQ,CAAS,EAAEX,KAAe;IACxF,IAAIY,QAAQC;IACZ,IAAIC,MAAM;IACV,IAAK,IAAIC,IAAIZ,GAAGY,IAAIJ,GAAGI,IAAK;QAC1B,MAAMC,MAAMjB,MAAM,CAACgB,EAAE,CAACC,GAAG;QACzB,IAAIA,KAAK;YACPJ,QAAQK,KAAKC,GAAG,CAACN,OAAOI,GAAG,CAAC,EAAE;YAC9BF,MAAMG,KAAKE,GAAG,CAACL,KAAKE,GAAG,CAAC,EAAE;QAC5B;IACF;IACA,MAAMI,YAAYC,OAAOC,QAAQ,CAACV,SAASA,QAAQ,IAAI;IACvD,IAAIW,UAAUT,MAAMF,QAAQE,MAAMM;IAClC,MAAOG,UAAUH,aAAapB,KAAK,CAACuB,UAAU,EAAE,CAACC,IAAI,OAAO,GAAID;IAChE,MAAME,QAAe;QAAEhB;QAAMW;QAAWG;QAASG,MAAM3B,OAAO4B,KAAK,CAACxB,GAAGQ;IAAG;IAC1E,IAAIF,SAAS,WAAW;QACtBgB,MAAMf,KAAK,GAAGW,OAAOtB,MAAM,CAACI,EAAE,CAACyB,GAAG,CAACD,KAAK,CAAC;QACzCF,MAAMI,IAAI,GAAG3C,YAAYuC;IAC3B;IACA,OAAOA;AACT"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import Module from 'node:module';
|
|
2
|
+
// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never
|
|
3
|
+
// parses, so every store-opening command paid for them until a file actually changed.
|
|
4
|
+
const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
|
|
5
|
+
let cached;
|
|
6
|
+
// The one shared parser: html and linkify on, fuzzy links, footnotes and task lists, with the
|
|
7
|
+
// footnote rules disabled for GFM parity (definitions stay spans; ^[...] stays inert text).
|
|
8
|
+
export function parser() {
|
|
9
|
+
if (cached) return cached;
|
|
10
|
+
const Ctor = _require('markdown-it');
|
|
11
|
+
const footnote = _require('markdown-it-footnote');
|
|
12
|
+
const taskLists = _require('markdown-it-task-lists');
|
|
13
|
+
const md = new Ctor({
|
|
14
|
+
html: true,
|
|
15
|
+
linkify: true
|
|
16
|
+
}).use(footnote).use(taskLists);
|
|
17
|
+
// Fuzzy linking is what links www. domains and emails (the scheme matcher does neither); its
|
|
18
|
+
// bare-domain over-matching vs GFM is reined in at extraction, where the text is kept.
|
|
19
|
+
md.linkify.set({
|
|
20
|
+
fuzzyLink: true
|
|
21
|
+
});
|
|
22
|
+
md.core.ruler.disable('footnote_tail');
|
|
23
|
+
md.inline.ruler.disable('footnote_inline');
|
|
24
|
+
cached = md;
|
|
25
|
+
return cached;
|
|
26
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parser.ts"],"sourcesContent":["import Module from 'node:module';\nimport type { MarkdownIt } from 'markdown-it';\nimport type footnotePlugin from 'markdown-it-footnote';\nimport type taskListsPlugin from 'markdown-it-task-lists';\n\n// Tier-2, as embed/static.ts: the parser's packages cost ~19 ms to load and a warm tree never\n// parses, so every store-opening command paid for them until a file actually changed.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// The CJS build resolves markdown-it's export= types, so the constructor is named locally.\ntype Ctor = new (options?: { html?: boolean; linkify?: boolean }) => MarkdownIt;\n\nlet cached: MarkdownIt | undefined;\n\n// The one shared parser: html and linkify on, fuzzy links, footnotes and task lists, with the\n// footnote rules disabled for GFM parity (definitions stay spans; ^[...] stays inert text).\nexport function parser(): MarkdownIt {\n if (cached) return cached;\n const Ctor = _require('markdown-it') as Ctor;\n const footnote = _require('markdown-it-footnote') as typeof footnotePlugin;\n const taskLists = _require('markdown-it-task-lists') as typeof taskListsPlugin;\n const md = new Ctor({ html: true, linkify: true }).use(footnote).use(taskLists);\n // Fuzzy linking is what links www. domains and emails (the scheme matcher does neither); its\n // bare-domain over-matching vs GFM is reined in at extraction, where the text is kept.\n md.linkify.set({ fuzzyLink: true });\n md.core.ruler.disable('footnote_tail');\n md.inline.ruler.disable('footnote_inline');\n cached = md;\n return cached;\n}\n"],"names":["Module","_require","require","createRequire","url","cached","parser","Ctor","footnote","taskLists","md","html","linkify","use","set","fuzzyLink","core","ruler","disable","inline"],"mappings":"AAAA,OAAOA,YAAY,cAAc;AAKjC,8FAA8F;AAC9F,sFAAsF;AACtF,MAAMC,WAAW,OAAOC,YAAY,cAAcF,OAAOG,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAK1F,IAAIG;AAEJ,8FAA8F;AAC9F,4FAA4F;AAC5F,OAAO,SAASC;IACd,IAAID,QAAQ,OAAOA;IACnB,MAAME,OAAON,SAAS;IACtB,MAAMO,WAAWP,SAAS;IAC1B,MAAMQ,YAAYR,SAAS;IAC3B,MAAMS,KAAK,IAAIH,KAAK;QAAEI,MAAM;QAAMC,SAAS;IAAK,GAAGC,GAAG,CAACL,UAAUK,GAAG,CAACJ;IACrE,6FAA6F;IAC7F,uFAAuF;IACvFC,GAAGE,OAAO,CAACE,GAAG,CAAC;QAAEC,WAAW;IAAK;IACjCL,GAAGM,IAAI,CAACC,KAAK,CAACC,OAAO,CAAC;IACtBR,GAAGS,MAAM,CAACF,KAAK,CAACC,OAAO,CAAC;IACxBb,SAASK;IACT,OAAOL;AACT"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Token } from 'markdown-it';
|
|
2
2
|
export type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';
|
|
3
3
|
export interface Block {
|
|
4
4
|
type: BlockType;
|
|
@@ -6,7 +6,7 @@ export interface Block {
|
|
|
6
6
|
endLine: number;
|
|
7
7
|
depth?: number;
|
|
8
8
|
text?: string;
|
|
9
|
-
node:
|
|
9
|
+
node: Token[];
|
|
10
10
|
}
|
|
11
11
|
export interface ChunkOptions {
|
|
12
12
|
targetTokens?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/types.ts"],"sourcesContent":["import type {
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/types.ts"],"sourcesContent":["import type { Token } from 'markdown-it';\n\nexport type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';\n\n// One top-level markdown-it block plus its 1-indexed source line extent, heading depth/text if any.\n// `node` is the block's token slice, so extract.ts and group.ts (W2) read the same parse.\nexport interface Block {\n type: BlockType;\n startLine: number;\n endLine: number;\n depth?: number;\n text?: string;\n node: Token[];\n}\n\nexport interface ChunkOptions {\n // D3/W3b: the pgc pairing cap and the sole owner lever (embed.chunkTokens); default 500.\n targetTokens?: number;\n // D9: 'raw' (source lines verbatim) is the shipped default; 'extracted' (flavor-resolved)\n // stays available for a future measured revisit, not currently shipped.\n text?: 'extracted' | 'raw';\n}\n\n// One grouped, extracted unit: 1-indexed line extent over the body, flavor-resolved text.\nexport interface Chunk {\n startLine: number;\n endLine: number;\n text: string;\n}\n"],"names":[],"mappings":"AAuBA,0FAA0F;AAC1F,WAIC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const CHUNK_VERSION = "chunk:
|
|
1
|
+
export declare const CHUNK_VERSION = "chunk:v6";
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in
|
|
2
2
|
// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.
|
|
3
|
-
export const CHUNK_VERSION = 'chunk:
|
|
3
|
+
export const CHUNK_VERSION = 'chunk:v6';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/version.ts"],"sourcesContent":["// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in\n// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.\nexport const CHUNK_VERSION = 'chunk:
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/version.ts"],"sourcesContent":["// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in\n// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.\nexport const CHUNK_VERSION = 'chunk:v6';\n"],"names":["CHUNK_VERSION"],"mappings":"AAAA,wFAAwF;AACxF,iGAAiG;AACjG,OAAO,MAAMA,gBAAgB,WAAW"}
|
|
@@ -120,17 +120,6 @@ async function writeResolved(db, rows) {
|
|
|
120
120
|
r.target
|
|
121
121
|
]));
|
|
122
122
|
}
|
|
123
|
-
// A new or deleted file can change any note's resolution, so re-resolve the whole table.
|
|
124
|
-
// Fallback for cold builds and large deltas -- see afterReconcile's threshold.
|
|
125
|
-
async function resolveAll(db, files) {
|
|
126
|
-
const pathSet = new Set(files.map((f)=>f.relPath));
|
|
127
|
-
const byBase = buildByBase(files);
|
|
128
|
-
const stmt = await db.prepare('SELECT src, target, dst, embed FROM links');
|
|
129
|
-
const rows = await stmt.all();
|
|
130
|
-
const changed = resolveRows(rows, pathSet, byBase);
|
|
131
|
-
await writeResolved(db, changed);
|
|
132
|
-
return changed.length > 0;
|
|
133
|
-
}
|
|
134
123
|
// Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable limit
|
|
135
124
|
// (SQLITE_MAX_VARIABLE_NUMBER, 32766); harmless headroom on DuckDB, which has no such cap.
|
|
136
125
|
async function selectIn(db, column, keys) {
|
|
@@ -186,8 +175,11 @@ export async function linkEdges(db) {
|
|
|
186
175
|
// Records whether remove()/store() deleted any link row, since afterReconcile runs after both
|
|
187
176
|
// and dstChanged alone misses the reparsed-to-zero-links case. Keyed on delta so state dies with it.
|
|
188
177
|
const removedWithLinks = new WeakMap();
|
|
189
|
-
//
|
|
190
|
-
//
|
|
178
|
+
// Cold build only: whether store() resolved at least one row to a non-null dst, so afterReconcile
|
|
179
|
+
// can report linksChanged without re-reading the table it just wrote.
|
|
180
|
+
const coldResolved = new WeakMap();
|
|
181
|
+
// dst stays NULL here on an incremental store (resolution runs afterward); a cold build resolves
|
|
182
|
+
// it inline in store() instead, through the *_RESOLVED column list and SQL below.
|
|
191
183
|
const LINK_COLUMNS = [
|
|
192
184
|
'src',
|
|
193
185
|
'target',
|
|
@@ -195,6 +187,14 @@ const LINK_COLUMNS = [
|
|
|
195
187
|
'embed'
|
|
196
188
|
];
|
|
197
189
|
const UPSERT_LINK_SQL = 'INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base';
|
|
190
|
+
const LINK_COLUMNS_RESOLVED = [
|
|
191
|
+
'src',
|
|
192
|
+
'target',
|
|
193
|
+
'target_base',
|
|
194
|
+
'dst',
|
|
195
|
+
'embed'
|
|
196
|
+
];
|
|
197
|
+
const UPSERT_LINK_SQL_RESOLVED = 'INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, ?, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base';
|
|
198
198
|
export const links = {
|
|
199
199
|
name: 'links',
|
|
200
200
|
async schema (db) {
|
|
@@ -238,8 +238,18 @@ export const links = {
|
|
|
238
238
|
]));
|
|
239
239
|
removedWithLinks.set(delta, true);
|
|
240
240
|
}
|
|
241
|
-
//
|
|
242
|
-
//
|
|
241
|
+
// Cold build: every path is in `added`, so pathSet/byBase cover the same complete tree
|
|
242
|
+
// afterReconcile's large branch would otherwise re-derive; resolve dst here instead.
|
|
243
|
+
const cold = delta.files.length === 0 || delta.added.length === delta.files.length;
|
|
244
|
+
let resolveNew;
|
|
245
|
+
if (cold) {
|
|
246
|
+
const pathSet = new Set(delta.files.map((f)=>f.relPath));
|
|
247
|
+
const byBase = buildByBase(delta.files);
|
|
248
|
+
resolveNew = (src, target)=>resolveTarget(src, target, pathSet, byBase);
|
|
249
|
+
}
|
|
250
|
+
let resolvedLink = false;
|
|
251
|
+
// Upsert preserves dst on surviving rows; a cold build resolves new rows, else they start NULL.
|
|
252
|
+
// A same-note anchor gets no target_base, so SQL's `IN` excludes it from resolveIncremental's lookup.
|
|
243
253
|
// Split, unlike the other features: this hook has no remove() pass behind it (see above), so
|
|
244
254
|
// only a path in `added` is guaranteed to have no row yet. The rest genuinely conflict and the
|
|
245
255
|
// upsert is what preserves their `dst`.
|
|
@@ -247,18 +257,39 @@ export const links = {
|
|
|
247
257
|
const upsertRows = [];
|
|
248
258
|
for (const [path, targets] of byPath){
|
|
249
259
|
for (const { target, embed } of targets){
|
|
250
|
-
const
|
|
260
|
+
const targetBase = target.startsWith('#') ? null : baseKey(cleanTarget(target));
|
|
261
|
+
if (!addedSet.has(path)) {
|
|
262
|
+
upsertRows.push([
|
|
263
|
+
path,
|
|
264
|
+
target,
|
|
265
|
+
targetBase,
|
|
266
|
+
embed ? 1 : 0
|
|
267
|
+
]);
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
if (!resolveNew) {
|
|
271
|
+
appendable.push([
|
|
272
|
+
path,
|
|
273
|
+
target,
|
|
274
|
+
targetBase,
|
|
275
|
+
embed ? 1 : 0
|
|
276
|
+
]);
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
const dst = resolveNew(path, target);
|
|
280
|
+
if (dst !== null) resolvedLink = true;
|
|
281
|
+
appendable.push([
|
|
251
282
|
path,
|
|
252
283
|
target,
|
|
253
|
-
|
|
284
|
+
targetBase,
|
|
285
|
+
dst,
|
|
254
286
|
embed ? 1 : 0
|
|
255
|
-
];
|
|
256
|
-
if (addedSet.has(path)) appendable.push(row);
|
|
257
|
-
else upsertRows.push(row);
|
|
287
|
+
]);
|
|
258
288
|
}
|
|
259
289
|
}
|
|
260
|
-
await appendRows(db, 'links', LINK_COLUMNS, UPSERT_LINK_SQL, appendable);
|
|
290
|
+
await appendRows(db, 'links', resolveNew ? LINK_COLUMNS_RESOLVED : LINK_COLUMNS, resolveNew ? UPSERT_LINK_SQL_RESOLVED : UPSERT_LINK_SQL, appendable);
|
|
261
291
|
if (upsertRows.length > 0) await db.runBatch(UPSERT_LINK_SQL, upsertRows);
|
|
292
|
+
if (cold) coldResolved.set(delta, resolvedLink);
|
|
262
293
|
},
|
|
263
294
|
async afterReconcile (db, delta) {
|
|
264
295
|
var _removedWithLinks_get;
|
|
@@ -268,7 +299,13 @@ export const links = {
|
|
|
268
299
|
// A cold build is the only case a full pass wins. Measured 2026-09-01 on 6,566 notes, resolve
|
|
269
300
|
// time alone: full is flat near 90ms at any churn, incremental 7ms at 10 files and 30ms at 2,000.
|
|
270
301
|
const large = delta.files.length === 0 || delta.added.length === delta.files.length;
|
|
271
|
-
|
|
302
|
+
// store() already resolved dst inline on this cold build (same test), so nothing is left to update.
|
|
303
|
+
if (large) {
|
|
304
|
+
var _coldResolved_get;
|
|
305
|
+
delta.linksChanged = ((_coldResolved_get = coldResolved.get(delta)) !== null && _coldResolved_get !== void 0 ? _coldResolved_get : false) || removeHadLinks;
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
const dstChanged = await resolveIncremental(db, delta);
|
|
272
309
|
delta.linksChanged = dstChanged || removeHadLinks;
|
|
273
310
|
}
|
|
274
311
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { FileStat } from '../scan/index.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Connection } from '../store/types.ts';\nimport { maskRegions } from './fences.ts';\nimport type { ExtractedDoc, Feature, ReconcileDelta } from './types.ts';\n\n// links(src, target, target_base, dst, embed): target as written, target_base its indexed baseKey, dst the resolved path or NULL (a queryable dead link).\n// embed distinguishes `![[x]]`/`` from `[[x]]`/`[](x.md)` -- Obsidian's grain: a target both linked and embedded in one note is two rows.\n\n// Alias stripped after |; a leading # keeps the anchor as the target (self-link needs a\n// heading name), else the anchor splits off. Null = not a link.\nfunction parseWikilinkInner(inner: string): string | null {\n const beforeAlias = inner.split('|')[0].trim();\n if (beforeAlias.startsWith('#')) return beforeAlias.length > 1 ? beforeAlias : null;\n const target = beforeAlias.split('#')[0].trim();\n return target || null;\n}\n\n// A frontmatter value counts only when it is exactly a wikilink (\"[[X]]\") -- mid-string and\n// ![[...]] forms don't count there, per Obsidian's rule (probe-verified).\nfunction extract(_raw: string, rawBody: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): Array<{ target: string; embed: boolean }> {\n const body = /\\[\\[|\\]\\(/.test(rawBody) ? maskRegions(rawBody) : rawBody;\n const seen = new Set<string>();\n const results: Array<{ target: string; embed: boolean }> = [];\n const add = (target: string, embed: boolean) => {\n const key = `${target}\\0${embed ? '1' : '0'}`;\n if (seen.has(key)) return;\n seen.add(key);\n results.push({ target, embed });\n };\n // To the first ]], as Obsidian parses it, so a heading or alias holding a lone ] still\n // matches; anchor and alias split off in parseWikilinkInner.\n for (const m of body.matchAll(/\\[\\[(.*?)\\]\\]/g)) {\n const target = parseWikilinkInner(m[1]);\n if (target) add(target, body[(m.index ?? 0) - 1] === '!');\n }\n // Any internal destination, not only .md-suffixed: \\](Zektor) resolves like [[Zektor]] and\n // \\](#anchor) is a self-edge. External URLs and titled links (dest stops at whitespace) skip.\n for (const m of body.matchAll(/(!)?\\[[^\\]]*\\]\\(((?:[^()\\s]|\\([^()\\s]*\\))+)(?:\\s+(?:\"[^)]*\"|'[^)]*'))?\\)/g)) {\n const dest = m[2].trim();\n // External: a URI scheme prefix (mailto:, tel:, data:, https:...), protocol-relative\n // //host, a www. shorthand, or a scheme anywhere (the malformed double-paren case).\n if (!dest || dest.includes('://')) continue;\n if (/^([a-z][a-z0-9+.-]*:|\\/\\/|www\\.)/i.test(dest)) continue;\n const target = dest.startsWith('#') ? dest : dest.split('#')[0];\n if (target) add(target, m[1] === '!');\n }\n for (const value of Object.values(data ?? {})) {\n for (const item of Array.isArray(value) ? value : [value]) {\n if (typeof item !== 'string') continue;\n const m = /^\\[\\[(.*)\\]\\]$/.exec(item.trim());\n if (!m) continue;\n const target = parseWikilinkInner(m[1]);\n if (target) add(target, false);\n }\n }\n return results;\n}\n\nfunction cleanTarget(target: string): string {\n return target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n}\n\nfunction baseKey(path: string): string {\n return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking note's\n// directory, then basename match (shortest path wins ties, verified against a cold-loaded vault).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n // [[#Heading]]: a same-note anchor, always the note itself -- never depends on other files.\n if (target.startsWith('#')) return src;\n const clean = cleanTarget(target);\n const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n if (pathSet.has(candidate)) return candidate;\n }\n const candidates = byBase.get(baseKey(clean));\n if (!candidates) return null;\n // The linking note itself wins a basename collision (Obsidian resolves to self before the\n // shortest-path rule -- cold-load verified on the hub corpus).\n return candidates.includes(src) ? src : candidates[0];\n}\n\n// Shortest path wins a basename collision; equal lengths fall back to lexicographic, our own\n// deterministic tiebreak for a case Obsidian itself leaves registration-order-dependent.\nfunction buildByBase(files: FileStat[]): Map<string, string[]> {\n const byBase = new Map<string, string[]>();\n const paths = files.map((f) => f.relPath).sort((a, b) => a.length - b.length || (a < b ? -1 : a > b ? 1 : 0));\n for (const path of paths) {\n const key = baseKey(path);\n const list = byBase.get(key);\n if (list) list.push(path);\n else byBase.set(key, [path]);\n }\n return byBase;\n}\n\ninterface LinkRow {\n src: string;\n target: string;\n dst: string | null;\n embed: number;\n}\n\n// resolveTarget applied to `rows`, returning only the rows whose dst changed, so the caller\n// can write them in one runBatch call.\nfunction resolveRows(rows: LinkRow[], pathSet: Set<string>, byBase: Map<string, string[]>): LinkRow[] {\n const changed: LinkRow[] = [];\n for (const row of rows) {\n const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n if (dst !== row.dst) changed.push({ ...row, dst });\n }\n return changed;\n}\n\nasync function writeResolved(db: Connection, rows: LinkRow[]): Promise<void> {\n if (rows.length === 0) return;\n await db.runBatch(\n 'UPDATE links SET dst = ? WHERE src = ? AND target = ?',\n rows.map((r) => [r.dst, r.src, r.target])\n );\n}\n\n// A new or deleted file can change any note's resolution, so re-resolve the whole table.\n// Fallback for cold builds and large deltas -- see afterReconcile's threshold.\nasync function resolveAll(db: Connection, files: FileStat[]): Promise<boolean> {\n const pathSet = new Set(files.map((f) => f.relPath));\n const byBase = buildByBase(files);\n const stmt = await db.prepare('SELECT src, target, dst, embed FROM links');\n const rows = (await stmt.all()) as unknown as LinkRow[];\n const changed = resolveRows(rows, pathSet, byBase);\n await writeResolved(db, changed);\n return changed.length > 0;\n}\n\n// Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable limit\n// (SQLITE_MAX_VARIABLE_NUMBER, 32766); harmless headroom on DuckDB, which has no such cap.\nasync function selectIn(db: Connection, column: string, keys: string[]): Promise<LinkRow[]> {\n const rows: LinkRow[] = [];\n for (let i = 0; i < keys.length; i += 500) {\n const chunk = keys.slice(i, i + 500);\n const stmt = await db.prepare(`SELECT src, target, dst, embed FROM links WHERE ${column} IN (${chunk.map(() => '?').join(', ')})`);\n rows.push(...((await stmt.all(...chunk)) as unknown as LinkRow[]));\n }\n return rows;\n}\n\n// Re-resolves only what this reconcile could have touched: re-stored sources, links whose\n// target basename matches an added/vanished file, and links whose dst vanished -- all indexed.\nasync function resolveIncremental(db: Connection, delta: ReconcileDelta): Promise<boolean> {\n const pathSet = new Set(delta.files.map((f) => f.relPath));\n const byBase = buildByBase(delta.files);\n\n const changedBasenames = new Set<string>();\n for (const p of delta.added) changedBasenames.add(baseKey(p));\n for (const p of delta.vanished) changedBasenames.add(baseKey(p));\n\n // \\0 can't appear in a path or target, so the key never collides; a space can.\n const candidates = new Map<string, LinkRow>();\n const collect = (rows: LinkRow[]) => {\n for (const row of rows) candidates.set(`${row.src}\\0${row.target}\\0${row.embed}`, row);\n };\n collect(await selectIn(db, 'src', delta.reparsed));\n collect(await selectIn(db, 'target_base', [...changedBasenames]));\n collect(await selectIn(db, 'dst', delta.vanished));\n\n const changed = resolveRows([...candidates.values()], pathSet, byBase);\n await writeResolved(db, changed);\n return changed.length > 0;\n}\n\n// One edge per (src, target); an embed row is excluded only when the same pair also exists as\n// a link. Self-edges are dropped so PageRank mass is not self-recycled.\nexport const LINK_EDGES_SQL = `SELECT src, dst FROM links WHERE dst IS NOT NULL AND embed = 0 AND src != dst\n UNION ALL\n SELECT src, dst FROM links l WHERE dst IS NOT NULL AND embed = 1 AND src != dst\n AND NOT EXISTS (SELECT 1 FROM links l0 WHERE l0.src = l.src AND l0.target = l.target AND l0.embed = 0)`;\n\nexport function toEdges(rows: Array<{ src: string; dst: string }>): [string, string][] {\n return rows.map((r) => [r.src, r.dst]);\n}\n\n// Resolved edges, for rank (inside reconcile's afterReconcile pass).\nexport async function linkEdges(db: Connection): Promise<[string, string][]> {\n const stmt = await db.prepare(LINK_EDGES_SQL);\n return toEdges((await stmt.all()) as Array<{ src: string; dst: string }>);\n}\n\n// Records whether remove()/store() deleted any link row, since afterReconcile runs after both\n// and dstChanged alone misses the reparsed-to-zero-links case. Keyed on delta so state dies with it.\nconst removedWithLinks = new WeakMap<ReconcileDelta, boolean>();\n\n// dst is written by resolution, not by store(), so it is absent from the column list and takes the\n// table's default on the append path.\nconst LINK_COLUMNS = ['src', 'target', 'target_base', 'embed'];\nconst UPSERT_LINK_SQL = 'INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base';\n\nexport const links: Feature = {\n name: 'links',\n async schema(db) {\n await db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, embed INTEGER, PRIMARY KEY (src, target, embed))');\n await db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n await db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');\n },\n extract,\n // Reparsed docs are diffed in store() instead of wiped here: delete-and-reinsert resets dst\n // to NULL, making every touch-only reparse look like an edge change and recompute PageRank.\n async remove(db, paths, delta) {\n const vanishedSet = new Set(delta.vanished);\n const vanished = paths.filter((p) => vanishedSet.has(p));\n if (vanished.length === 0) return;\n const existingRows = await selectIn(db, 'src', vanished);\n await db.runBatch(\n 'DELETE FROM links WHERE src = ?',\n vanished.map((p) => [p])\n );\n if (existingRows.length > 0) removedWithLinks.set(delta, true);\n },\n async store(db, docs: ExtractedDoc[], delta) {\n const byPath = new Map(docs.map((d) => [d.path, d.extracted as Array<{ target: string; embed: boolean }>]));\n const addedSet = new Set(delta.added);\n const reparsedExisting = docs.map((d) => d.path).filter((p) => !addedSet.has(p));\n\n // Stale rows (target/embed pairs missing from the new parse) are diffed in JS from one\n // bulk SELECT, so this pass costs one query plus one batched DELETE regardless of file count.\n const existingRows = await selectIn(db, 'src', reparsedExisting);\n const staleRows = existingRows.filter((row) => {\n const targets = byPath.get(row.src) ?? [];\n return !targets.some((t) => t.target === row.target && (t.embed ? 1 : 0) === row.embed);\n });\n if (staleRows.length > 0) {\n await db.runBatch(\n 'DELETE FROM links WHERE src = ? AND target = ? AND embed = ?',\n staleRows.map((r) => [r.src, r.target, r.embed])\n );\n removedWithLinks.set(delta, true);\n }\n\n // Upsert preserves dst on surviving rows; only new rows start at NULL. A same-note anchor\n // gets no target_base, so SQL's `IN` excludes it from resolveIncremental's lookup.\n // Split, unlike the other features: this hook has no remove() pass behind it (see above), so\n // only a path in `added` is guaranteed to have no row yet. The rest genuinely conflict and the\n // upsert is what preserves their `dst`.\n const appendable: unknown[][] = [];\n const upsertRows: unknown[][] = [];\n for (const [path, targets] of byPath) {\n for (const { target, embed } of targets) {\n const row = [path, target, target.startsWith('#') ? null : baseKey(cleanTarget(target)), embed ? 1 : 0];\n if (addedSet.has(path)) appendable.push(row);\n else upsertRows.push(row);\n }\n }\n await appendRows(db, 'links', LINK_COLUMNS, UPSERT_LINK_SQL, appendable);\n if (upsertRows.length > 0) await db.runBatch(UPSERT_LINK_SQL, upsertRows);\n },\n async afterReconcile(db, delta) {\n // Any deleted rows mean the edge set shrank -- whether the file vanished or was reparsed\n // down to fewer/no links.\n const removeHadLinks = removedWithLinks.get(delta) ?? false;\n\n // A cold build is the only case a full pass wins. Measured 2026-09-01 on 6,566 notes, resolve\n // time alone: full is flat near 90ms at any churn, incremental 7ms at 10 files and 30ms at 2,000.\n const large = delta.files.length === 0 || delta.added.length === delta.files.length;\n\n const dstChanged = large ? await resolveAll(db, delta.files) : await resolveIncremental(db, delta);\n delta.linksChanged = dstChanged || removeHadLinks;\n },\n};\n"],"names":["posix","appendRows","maskRegions","parseWikilinkInner","inner","beforeAlias","split","trim","startsWith","length","target","extract","_raw","rawBody","_search","data","body","test","seen","Set","results","add","embed","key","has","push","m","matchAll","index","dest","includes","value","Object","values","item","Array","isArray","exec","cleanTarget","replace","baseKey","path","basename","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","candidates","get","buildByBase","files","Map","paths","map","f","relPath","sort","a","b","list","set","resolveRows","rows","changed","row","dst","writeResolved","db","runBatch","r","resolveAll","stmt","prepare","all","selectIn","column","keys","i","chunk","slice","resolveIncremental","delta","changedBasenames","p","added","vanished","collect","reparsed","LINK_EDGES_SQL","toEdges","linkEdges","removedWithLinks","WeakMap","LINK_COLUMNS","UPSERT_LINK_SQL","links","name","schema","remove","vanishedSet","filter","existingRows","store","docs","byPath","d","extracted","addedSet","reparsedExisting","staleRows","targets","some","t","appendable","upsertRows","afterReconcile","removeHadLinks","large","dstChanged","linksChanged"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,SAASC,UAAU,QAAQ,qBAAqB;AAEhD,SAASC,WAAW,QAAQ,cAAc;AAG1C,0JAA0J;AAC1J,mJAAmJ;AAEnJ,wFAAwF;AACxF,gEAAgE;AAChE,SAASC,mBAAmBC,KAAa;IACvC,MAAMC,cAAcD,MAAME,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC5C,IAAIF,YAAYG,UAAU,CAAC,MAAM,OAAOH,YAAYI,MAAM,GAAG,IAAIJ,cAAc;IAC/E,MAAMK,SAASL,YAAYC,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC7C,OAAOG,UAAU;AACnB;AAEA,4FAA4F;AAC5F,0EAA0E;AAC1E,SAASC,QAAQC,IAAY,EAAEC,OAAe,EAAEC,OAA4C,EAAEC,IAA8B;IAC1H,MAAMC,OAAO,YAAYC,IAAI,CAACJ,WAAWX,YAAYW,WAAWA;IAChE,MAAMK,OAAO,IAAIC;IACjB,MAAMC,UAAqD,EAAE;IAC7D,MAAMC,MAAM,CAACX,QAAgBY;QAC3B,MAAMC,MAAM,GAAGb,OAAO,EAAE,EAAEY,QAAQ,MAAM,KAAK;QAC7C,IAAIJ,KAAKM,GAAG,CAACD,MAAM;QACnBL,KAAKG,GAAG,CAACE;QACTH,QAAQK,IAAI,CAAC;YAAEf;YAAQY;QAAM;IAC/B;IACA,uFAAuF;IACvF,6DAA6D;IAC7D,KAAK,MAAMI,KAAKV,KAAKW,QAAQ,CAAC,kBAAmB;YAEjBD;QAD9B,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;QACtC,IAAIhB,QAAQW,IAAIX,QAAQM,IAAI,CAAC,EAACU,WAAAA,EAAEE,KAAK,cAAPF,sBAAAA,WAAW,KAAK,EAAE,KAAK;IACvD;IACA,2FAA2F;IAC3F,8FAA8F;IAC9F,KAAK,MAAMA,KAAKV,KAAKW,QAAQ,CAAC,6EAA8E;QAC1G,MAAME,OAAOH,CAAC,CAAC,EAAE,CAACnB,IAAI;QACtB,qFAAqF;QACrF,oFAAoF;QACpF,IAAI,CAACsB,QAAQA,KAAKC,QAAQ,CAAC,QAAQ;QACnC,IAAI,oCAAoCb,IAAI,CAACY,OAAO;QACpD,MAAMnB,SAASmB,KAAKrB,UAAU,CAAC,OAAOqB,OAAOA,KAAKvB,KAAK,CAAC,IAAI,CAAC,EAAE;QAC/D,IAAII,QAAQW,IAAIX,QAAQgB,CAAC,CAAC,EAAE,KAAK;IACnC;IACA,KAAK,MAAMK,SAASC,OAAOC,MAAM,CAAClB,iBAAAA,kBAAAA,OAAQ,CAAC,GAAI;QAC7C,KAAK,MAAMmB,QAAQC,MAAMC,OAAO,CAACL,SAASA,QAAQ;YAACA;SAAM,CAAE;YACzD,IAAI,OAAOG,SAAS,UAAU;YAC9B,MAAMR,IAAI,iBAAiBW,IAAI,CAACH,KAAK3B,IAAI;YACzC,IAAI,CAACmB,GAAG;YACR,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;YACtC,IAAIhB,QAAQW,IAAIX,QAAQ;QAC1B;IACF;IACA,OAAOU;AACT;AAEA,SAASkB,YAAY5B,MAAc;IACjC,OAAOA,OAAO6B,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;AACrD;AAEA,SAASC,QAAQC,IAAY;IAC3B,OAAOzC,MAAM0C,QAAQ,CAACD,MAAMF,OAAO,CAAC,UAAU,IAAII,WAAW;AAC/D;AAEA,8FAA8F;AAC9F,kGAAkG;AAClG,SAASC,cAAcC,GAAW,EAAEnC,MAAc,EAAEoC,OAAoB,EAAEC,MAA6B;IACrG,4FAA4F;IAC5F,IAAIrC,OAAOF,UAAU,CAAC,MAAM,OAAOqC;IACnC,MAAMG,QAAQV,YAAY5B;IAC1B,MAAMuC,UAAUjD,MAAMkD,SAAS,CAAClD,MAAMmD,IAAI,CAACnD,MAAMoD,OAAO,CAACP,MAAMG;IAC/D,KAAK,MAAMK,aAAa;QAACL;QAAO,GAAGA,MAAM,GAAG,CAAC;QAAEC;QAAS,GAAGA,QAAQ,GAAG,CAAC;KAAC,CAAE;QACxE,IAAIH,QAAQtB,GAAG,CAAC6B,YAAY,OAAOA;IACrC;IACA,MAAMC,aAAaP,OAAOQ,GAAG,CAACf,QAAQQ;IACtC,IAAI,CAACM,YAAY,OAAO;IACxB,0FAA0F;IAC1F,+DAA+D;IAC/D,OAAOA,WAAWxB,QAAQ,CAACe,OAAOA,MAAMS,UAAU,CAAC,EAAE;AACvD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,SAASE,YAAYC,KAAiB;IACpC,MAAMV,SAAS,IAAIW;IACnB,MAAMC,QAAQF,MAAMG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEvD,MAAM,GAAGwD,EAAExD,MAAM,IAAKuD,CAAAA,IAAIC,IAAI,CAAC,IAAID,IAAIC,IAAI,IAAI,CAAA;IAC1G,KAAK,MAAMxB,QAAQkB,MAAO;QACxB,MAAMpC,MAAMiB,QAAQC;QACpB,MAAMyB,OAAOnB,OAAOQ,GAAG,CAAChC;QACxB,IAAI2C,MAAMA,KAAKzC,IAAI,CAACgB;aACfM,OAAOoB,GAAG,CAAC5C,KAAK;YAACkB;SAAK;IAC7B;IACA,OAAOM;AACT;AASA,4FAA4F;AAC5F,uCAAuC;AACvC,SAASqB,YAAYC,IAAe,EAAEvB,OAAoB,EAAEC,MAA6B;IACvF,MAAMuB,UAAqB,EAAE;IAC7B,KAAK,MAAMC,OAAOF,KAAM;QACtB,MAAMG,MAAM5B,cAAc2B,IAAI1B,GAAG,EAAE0B,IAAI7D,MAAM,EAAEoC,SAASC;QACxD,IAAIyB,QAAQD,IAAIC,GAAG,EAAEF,QAAQ7C,IAAI,CAAC;YAAE,GAAG8C,GAAG;YAAEC;QAAI;IAClD;IACA,OAAOF;AACT;AAEA,eAAeG,cAAcC,EAAc,EAAEL,IAAe;IAC1D,IAAIA,KAAK5D,MAAM,KAAK,GAAG;IACvB,MAAMiE,GAAGC,QAAQ,CACf,yDACAN,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAEJ,GAAG;YAAEI,EAAE/B,GAAG;YAAE+B,EAAElE,MAAM;SAAC;AAE5C;AAEA,yFAAyF;AACzF,+EAA+E;AAC/E,eAAemE,WAAWH,EAAc,EAAEjB,KAAiB;IACzD,MAAMX,UAAU,IAAI3B,IAAIsC,MAAMG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAClD,MAAMf,SAASS,YAAYC;IAC3B,MAAMqB,OAAO,MAAMJ,GAAGK,OAAO,CAAC;IAC9B,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAC5B,MAAMV,UAAUF,YAAYC,MAAMvB,SAASC;IAC3C,MAAM0B,cAAcC,IAAIJ;IACxB,OAAOA,QAAQ7D,MAAM,GAAG;AAC1B;AAEA,2FAA2F;AAC3F,2FAA2F;AAC3F,eAAewE,SAASP,EAAc,EAAEQ,MAAc,EAAEC,IAAc;IACpE,MAAMd,OAAkB,EAAE;IAC1B,IAAK,IAAIe,IAAI,GAAGA,IAAID,KAAK1E,MAAM,EAAE2E,KAAK,IAAK;QACzC,MAAMC,QAAQF,KAAKG,KAAK,CAACF,GAAGA,IAAI;QAChC,MAAMN,OAAO,MAAMJ,GAAGK,OAAO,CAAC,CAAC,gDAAgD,EAAEG,OAAO,KAAK,EAAEG,MAAMzB,GAAG,CAAC,IAAM,KAAKT,IAAI,CAAC,MAAM,CAAC,CAAC;QACjIkB,KAAK5C,IAAI,IAAM,MAAMqD,KAAKE,GAAG,IAAIK;IACnC;IACA,OAAOhB;AACT;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,eAAekB,mBAAmBb,EAAc,EAAEc,KAAqB;IACrE,MAAM1C,UAAU,IAAI3B,IAAIqE,MAAM/B,KAAK,CAACG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACxD,MAAMf,SAASS,YAAYgC,MAAM/B,KAAK;IAEtC,MAAMgC,mBAAmB,IAAItE;IAC7B,KAAK,MAAMuE,KAAKF,MAAMG,KAAK,CAAEF,iBAAiBpE,GAAG,CAACmB,QAAQkD;IAC1D,KAAK,MAAMA,KAAKF,MAAMI,QAAQ,CAAEH,iBAAiBpE,GAAG,CAACmB,QAAQkD;IAE7D,+EAA+E;IAC/E,MAAMpC,aAAa,IAAII;IACvB,MAAMmC,UAAU,CAACxB;QACf,KAAK,MAAME,OAAOF,KAAMf,WAAWa,GAAG,CAAC,GAAGI,IAAI1B,GAAG,CAAC,EAAE,EAAE0B,IAAI7D,MAAM,CAAC,EAAE,EAAE6D,IAAIjD,KAAK,EAAE,EAAEiD;IACpF;IACAsB,QAAQ,MAAMZ,SAASP,IAAI,OAAOc,MAAMM,QAAQ;IAChDD,QAAQ,MAAMZ,SAASP,IAAI,eAAe;WAAIe;KAAiB;IAC/DI,QAAQ,MAAMZ,SAASP,IAAI,OAAOc,MAAMI,QAAQ;IAEhD,MAAMtB,UAAUF,YAAY;WAAId,WAAWrB,MAAM;KAAG,EAAEa,SAASC;IAC/D,MAAM0B,cAAcC,IAAIJ;IACxB,OAAOA,QAAQ7D,MAAM,GAAG;AAC1B;AAEA,8FAA8F;AAC9F,wEAAwE;AACxE,OAAO,MAAMsF,iBAAiB,CAAC;;;4GAG6E,CAAC,CAAC;AAE9G,OAAO,SAASC,QAAQ3B,IAAyC;IAC/D,OAAOA,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAE/B,GAAG;YAAE+B,EAAEJ,GAAG;SAAC;AACvC;AAEA,qEAAqE;AACrE,OAAO,eAAeyB,UAAUvB,EAAc;IAC5C,MAAMI,OAAO,MAAMJ,GAAGK,OAAO,CAACgB;IAC9B,OAAOC,QAAS,MAAMlB,KAAKE,GAAG;AAChC;AAEA,8FAA8F;AAC9F,qGAAqG;AACrG,MAAMkB,mBAAmB,IAAIC;AAE7B,mGAAmG;AACnG,sCAAsC;AACtC,MAAMC,eAAe;IAAC;IAAO;IAAU;IAAe;CAAQ;AAC9D,MAAMC,kBAAkB;AAExB,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAO9B,EAAE;QACb,MAAMA,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;IAChB;IACA1B;IACA,4FAA4F;IAC5F,4FAA4F;IAC5F,MAAM8F,QAAO/B,EAAE,EAAEf,KAAK,EAAE6B,KAAK;QAC3B,MAAMkB,cAAc,IAAIvF,IAAIqE,MAAMI,QAAQ;QAC1C,MAAMA,WAAWjC,MAAMgD,MAAM,CAAC,CAACjB,IAAMgB,YAAYlF,GAAG,CAACkE;QACrD,IAAIE,SAASnF,MAAM,KAAK,GAAG;QAC3B,MAAMmG,eAAe,MAAM3B,SAASP,IAAI,OAAOkB;QAC/C,MAAMlB,GAAGC,QAAQ,CACf,mCACAiB,SAAShC,GAAG,CAAC,CAAC8B,IAAM;gBAACA;aAAE;QAEzB,IAAIkB,aAAanG,MAAM,GAAG,GAAGyF,iBAAiB/B,GAAG,CAACqB,OAAO;IAC3D;IACA,MAAMqB,OAAMnC,EAAE,EAAEoC,IAAoB,EAAEtB,KAAK;QACzC,MAAMuB,SAAS,IAAIrD,IAAIoD,KAAKlD,GAAG,CAAC,CAACoD,IAAM;gBAACA,EAAEvE,IAAI;gBAAEuE,EAAEC,SAAS;aAA8C;QACzG,MAAMC,WAAW,IAAI/F,IAAIqE,MAAMG,KAAK;QACpC,MAAMwB,mBAAmBL,KAAKlD,GAAG,CAAC,CAACoD,IAAMA,EAAEvE,IAAI,EAAEkE,MAAM,CAAC,CAACjB,IAAM,CAACwB,SAAS1F,GAAG,CAACkE;QAE7E,uFAAuF;QACvF,8FAA8F;QAC9F,MAAMkB,eAAe,MAAM3B,SAASP,IAAI,OAAOyC;QAC/C,MAAMC,YAAYR,aAAaD,MAAM,CAAC,CAACpC;gBACrBwC;YAAhB,MAAMM,WAAUN,cAAAA,OAAOxD,GAAG,CAACgB,IAAI1B,GAAG,eAAlBkE,yBAAAA,cAAuB,EAAE;YACzC,OAAO,CAACM,QAAQC,IAAI,CAAC,CAACC,IAAMA,EAAE7G,MAAM,KAAK6D,IAAI7D,MAAM,IAAI,AAAC6G,CAAAA,EAAEjG,KAAK,GAAG,IAAI,CAAA,MAAOiD,IAAIjD,KAAK;QACxF;QACA,IAAI8F,UAAU3G,MAAM,GAAG,GAAG;YACxB,MAAMiE,GAAGC,QAAQ,CACf,gEACAyC,UAAUxD,GAAG,CAAC,CAACgB,IAAM;oBAACA,EAAE/B,GAAG;oBAAE+B,EAAElE,MAAM;oBAAEkE,EAAEtD,KAAK;iBAAC;YAEjD4E,iBAAiB/B,GAAG,CAACqB,OAAO;QAC9B;QAEA,0FAA0F;QAC1F,mFAAmF;QACnF,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMgC,aAA0B,EAAE;QAClC,MAAMC,aAA0B,EAAE;QAClC,KAAK,MAAM,CAAChF,MAAM4E,QAAQ,IAAIN,OAAQ;YACpC,KAAK,MAAM,EAAErG,MAAM,EAAEY,KAAK,EAAE,IAAI+F,QAAS;gBACvC,MAAM9C,MAAM;oBAAC9B;oBAAM/B;oBAAQA,OAAOF,UAAU,CAAC,OAAO,OAAOgC,QAAQF,YAAY5B;oBAAUY,QAAQ,IAAI;iBAAE;gBACvG,IAAI4F,SAAS1F,GAAG,CAACiB,OAAO+E,WAAW/F,IAAI,CAAC8C;qBACnCkD,WAAWhG,IAAI,CAAC8C;YACvB;QACF;QACA,MAAMtE,WAAWyE,IAAI,SAAS0B,cAAcC,iBAAiBmB;QAC7D,IAAIC,WAAWhH,MAAM,GAAG,GAAG,MAAMiE,GAAGC,QAAQ,CAAC0B,iBAAiBoB;IAChE;IACA,MAAMC,gBAAehD,EAAE,EAAEc,KAAK;YAGLU;QAFvB,yFAAyF;QACzF,0BAA0B;QAC1B,MAAMyB,kBAAiBzB,wBAAAA,iBAAiB3C,GAAG,CAACiC,oBAArBU,mCAAAA,wBAA+B;QAEtD,8FAA8F;QAC9F,kGAAkG;QAClG,MAAM0B,QAAQpC,MAAM/B,KAAK,CAAChD,MAAM,KAAK,KAAK+E,MAAMG,KAAK,CAAClF,MAAM,KAAK+E,MAAM/B,KAAK,CAAChD,MAAM;QAEnF,MAAMoH,aAAaD,QAAQ,MAAM/C,WAAWH,IAAIc,MAAM/B,KAAK,IAAI,MAAM8B,mBAAmBb,IAAIc;QAC5FA,MAAMsC,YAAY,GAAGD,cAAcF;IACrC;AACF,EAAE"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { FileStat } from '../scan/index.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Connection } from '../store/types.ts';\nimport { maskRegions } from './fences.ts';\nimport type { ExtractedDoc, Feature, ReconcileDelta } from './types.ts';\n\n// links(src, target, target_base, dst, embed): target as written, target_base its indexed baseKey, dst the resolved path or NULL (a queryable dead link).\n// embed distinguishes `![[x]]`/`` from `[[x]]`/`[](x.md)` -- Obsidian's grain: a target both linked and embedded in one note is two rows.\n\n// Alias stripped after |; a leading # keeps the anchor as the target (self-link needs a\n// heading name), else the anchor splits off. Null = not a link.\nfunction parseWikilinkInner(inner: string): string | null {\n const beforeAlias = inner.split('|')[0].trim();\n if (beforeAlias.startsWith('#')) return beforeAlias.length > 1 ? beforeAlias : null;\n const target = beforeAlias.split('#')[0].trim();\n return target || null;\n}\n\n// A frontmatter value counts only when it is exactly a wikilink (\"[[X]]\") -- mid-string and\n// ![[...]] forms don't count there, per Obsidian's rule (probe-verified).\nfunction extract(_raw: string, rawBody: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): Array<{ target: string; embed: boolean }> {\n const body = /\\[\\[|\\]\\(/.test(rawBody) ? maskRegions(rawBody) : rawBody;\n const seen = new Set<string>();\n const results: Array<{ target: string; embed: boolean }> = [];\n const add = (target: string, embed: boolean) => {\n const key = `${target}\\0${embed ? '1' : '0'}`;\n if (seen.has(key)) return;\n seen.add(key);\n results.push({ target, embed });\n };\n // To the first ]], as Obsidian parses it, so a heading or alias holding a lone ] still\n // matches; anchor and alias split off in parseWikilinkInner.\n for (const m of body.matchAll(/\\[\\[(.*?)\\]\\]/g)) {\n const target = parseWikilinkInner(m[1]);\n if (target) add(target, body[(m.index ?? 0) - 1] === '!');\n }\n // Any internal destination, not only .md-suffixed: \\](Zektor) resolves like [[Zektor]] and\n // \\](#anchor) is a self-edge. External URLs and titled links (dest stops at whitespace) skip.\n for (const m of body.matchAll(/(!)?\\[[^\\]]*\\]\\(((?:[^()\\s]|\\([^()\\s]*\\))+)(?:\\s+(?:\"[^)]*\"|'[^)]*'))?\\)/g)) {\n const dest = m[2].trim();\n // External: a URI scheme prefix (mailto:, tel:, data:, https:...), protocol-relative\n // //host, a www. shorthand, or a scheme anywhere (the malformed double-paren case).\n if (!dest || dest.includes('://')) continue;\n if (/^([a-z][a-z0-9+.-]*:|\\/\\/|www\\.)/i.test(dest)) continue;\n const target = dest.startsWith('#') ? dest : dest.split('#')[0];\n if (target) add(target, m[1] === '!');\n }\n for (const value of Object.values(data ?? {})) {\n for (const item of Array.isArray(value) ? value : [value]) {\n if (typeof item !== 'string') continue;\n const m = /^\\[\\[(.*)\\]\\]$/.exec(item.trim());\n if (!m) continue;\n const target = parseWikilinkInner(m[1]);\n if (target) add(target, false);\n }\n }\n return results;\n}\n\nfunction cleanTarget(target: string): string {\n return target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n}\n\nfunction baseKey(path: string): string {\n return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking note's\n// directory, then basename match (shortest path wins ties, verified against a cold-loaded vault).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n // [[#Heading]]: a same-note anchor, always the note itself -- never depends on other files.\n if (target.startsWith('#')) return src;\n const clean = cleanTarget(target);\n const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n if (pathSet.has(candidate)) return candidate;\n }\n const candidates = byBase.get(baseKey(clean));\n if (!candidates) return null;\n // The linking note itself wins a basename collision (Obsidian resolves to self before the\n // shortest-path rule -- cold-load verified on the hub corpus).\n return candidates.includes(src) ? src : candidates[0];\n}\n\n// Shortest path wins a basename collision; equal lengths fall back to lexicographic, our own\n// deterministic tiebreak for a case Obsidian itself leaves registration-order-dependent.\nfunction buildByBase(files: FileStat[]): Map<string, string[]> {\n const byBase = new Map<string, string[]>();\n const paths = files.map((f) => f.relPath).sort((a, b) => a.length - b.length || (a < b ? -1 : a > b ? 1 : 0));\n for (const path of paths) {\n const key = baseKey(path);\n const list = byBase.get(key);\n if (list) list.push(path);\n else byBase.set(key, [path]);\n }\n return byBase;\n}\n\ninterface LinkRow {\n src: string;\n target: string;\n dst: string | null;\n embed: number;\n}\n\n// resolveTarget applied to `rows`, returning only the rows whose dst changed, so the caller\n// can write them in one runBatch call.\nfunction resolveRows(rows: LinkRow[], pathSet: Set<string>, byBase: Map<string, string[]>): LinkRow[] {\n const changed: LinkRow[] = [];\n for (const row of rows) {\n const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n if (dst !== row.dst) changed.push({ ...row, dst });\n }\n return changed;\n}\n\nasync function writeResolved(db: Connection, rows: LinkRow[]): Promise<void> {\n if (rows.length === 0) return;\n await db.runBatch(\n 'UPDATE links SET dst = ? WHERE src = ? AND target = ?',\n rows.map((r) => [r.dst, r.src, r.target])\n );\n}\n\n// Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable limit\n// (SQLITE_MAX_VARIABLE_NUMBER, 32766); harmless headroom on DuckDB, which has no such cap.\nasync function selectIn(db: Connection, column: string, keys: string[]): Promise<LinkRow[]> {\n const rows: LinkRow[] = [];\n for (let i = 0; i < keys.length; i += 500) {\n const chunk = keys.slice(i, i + 500);\n const stmt = await db.prepare(`SELECT src, target, dst, embed FROM links WHERE ${column} IN (${chunk.map(() => '?').join(', ')})`);\n rows.push(...((await stmt.all(...chunk)) as unknown as LinkRow[]));\n }\n return rows;\n}\n\n// Re-resolves only what this reconcile could have touched: re-stored sources, links whose\n// target basename matches an added/vanished file, and links whose dst vanished -- all indexed.\nasync function resolveIncremental(db: Connection, delta: ReconcileDelta): Promise<boolean> {\n const pathSet = new Set(delta.files.map((f) => f.relPath));\n const byBase = buildByBase(delta.files);\n\n const changedBasenames = new Set<string>();\n for (const p of delta.added) changedBasenames.add(baseKey(p));\n for (const p of delta.vanished) changedBasenames.add(baseKey(p));\n\n // \\0 can't appear in a path or target, so the key never collides; a space can.\n const candidates = new Map<string, LinkRow>();\n const collect = (rows: LinkRow[]) => {\n for (const row of rows) candidates.set(`${row.src}\\0${row.target}\\0${row.embed}`, row);\n };\n collect(await selectIn(db, 'src', delta.reparsed));\n collect(await selectIn(db, 'target_base', [...changedBasenames]));\n collect(await selectIn(db, 'dst', delta.vanished));\n\n const changed = resolveRows([...candidates.values()], pathSet, byBase);\n await writeResolved(db, changed);\n return changed.length > 0;\n}\n\n// One edge per (src, target); an embed row is excluded only when the same pair also exists as\n// a link. Self-edges are dropped so PageRank mass is not self-recycled.\nexport const LINK_EDGES_SQL = `SELECT src, dst FROM links WHERE dst IS NOT NULL AND embed = 0 AND src != dst\n UNION ALL\n SELECT src, dst FROM links l WHERE dst IS NOT NULL AND embed = 1 AND src != dst\n AND NOT EXISTS (SELECT 1 FROM links l0 WHERE l0.src = l.src AND l0.target = l.target AND l0.embed = 0)`;\n\nexport function toEdges(rows: Array<{ src: string; dst: string }>): [string, string][] {\n return rows.map((r) => [r.src, r.dst]);\n}\n\n// Resolved edges, for rank (inside reconcile's afterReconcile pass).\nexport async function linkEdges(db: Connection): Promise<[string, string][]> {\n const stmt = await db.prepare(LINK_EDGES_SQL);\n return toEdges((await stmt.all()) as Array<{ src: string; dst: string }>);\n}\n\n// Records whether remove()/store() deleted any link row, since afterReconcile runs after both\n// and dstChanged alone misses the reparsed-to-zero-links case. Keyed on delta so state dies with it.\nconst removedWithLinks = new WeakMap<ReconcileDelta, boolean>();\n\n// Cold build only: whether store() resolved at least one row to a non-null dst, so afterReconcile\n// can report linksChanged without re-reading the table it just wrote.\nconst coldResolved = new WeakMap<ReconcileDelta, boolean>();\n\n// dst stays NULL here on an incremental store (resolution runs afterward); a cold build resolves\n// it inline in store() instead, through the *_RESOLVED column list and SQL below.\nconst LINK_COLUMNS = ['src', 'target', 'target_base', 'embed'];\nconst UPSERT_LINK_SQL = 'INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base';\nconst LINK_COLUMNS_RESOLVED = ['src', 'target', 'target_base', 'dst', 'embed'];\nconst UPSERT_LINK_SQL_RESOLVED = 'INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, ?, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base';\n\nexport const links: Feature = {\n name: 'links',\n async schema(db) {\n await db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, embed INTEGER, PRIMARY KEY (src, target, embed))');\n await db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n await db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');\n },\n extract,\n // Reparsed docs are diffed in store() instead of wiped here: delete-and-reinsert resets dst\n // to NULL, making every touch-only reparse look like an edge change and recompute PageRank.\n async remove(db, paths, delta) {\n const vanishedSet = new Set(delta.vanished);\n const vanished = paths.filter((p) => vanishedSet.has(p));\n if (vanished.length === 0) return;\n const existingRows = await selectIn(db, 'src', vanished);\n await db.runBatch(\n 'DELETE FROM links WHERE src = ?',\n vanished.map((p) => [p])\n );\n if (existingRows.length > 0) removedWithLinks.set(delta, true);\n },\n async store(db, docs: ExtractedDoc[], delta) {\n const byPath = new Map(docs.map((d) => [d.path, d.extracted as Array<{ target: string; embed: boolean }>]));\n const addedSet = new Set(delta.added);\n const reparsedExisting = docs.map((d) => d.path).filter((p) => !addedSet.has(p));\n\n // Stale rows (target/embed pairs missing from the new parse) are diffed in JS from one\n // bulk SELECT, so this pass costs one query plus one batched DELETE regardless of file count.\n const existingRows = await selectIn(db, 'src', reparsedExisting);\n const staleRows = existingRows.filter((row) => {\n const targets = byPath.get(row.src) ?? [];\n return !targets.some((t) => t.target === row.target && (t.embed ? 1 : 0) === row.embed);\n });\n if (staleRows.length > 0) {\n await db.runBatch(\n 'DELETE FROM links WHERE src = ? AND target = ? AND embed = ?',\n staleRows.map((r) => [r.src, r.target, r.embed])\n );\n removedWithLinks.set(delta, true);\n }\n\n // Cold build: every path is in `added`, so pathSet/byBase cover the same complete tree\n // afterReconcile's large branch would otherwise re-derive; resolve dst here instead.\n const cold = delta.files.length === 0 || delta.added.length === delta.files.length;\n let resolveNew: ((src: string, target: string) => string | null) | undefined;\n if (cold) {\n const pathSet = new Set(delta.files.map((f) => f.relPath));\n const byBase = buildByBase(delta.files);\n resolveNew = (src, target) => resolveTarget(src, target, pathSet, byBase);\n }\n let resolvedLink = false;\n\n // Upsert preserves dst on surviving rows; a cold build resolves new rows, else they start NULL.\n // A same-note anchor gets no target_base, so SQL's `IN` excludes it from resolveIncremental's lookup.\n // Split, unlike the other features: this hook has no remove() pass behind it (see above), so\n // only a path in `added` is guaranteed to have no row yet. The rest genuinely conflict and the\n // upsert is what preserves their `dst`.\n const appendable: unknown[][] = [];\n const upsertRows: unknown[][] = [];\n for (const [path, targets] of byPath) {\n for (const { target, embed } of targets) {\n const targetBase = target.startsWith('#') ? null : baseKey(cleanTarget(target));\n if (!addedSet.has(path)) {\n upsertRows.push([path, target, targetBase, embed ? 1 : 0]);\n continue;\n }\n if (!resolveNew) {\n appendable.push([path, target, targetBase, embed ? 1 : 0]);\n continue;\n }\n const dst = resolveNew(path, target);\n if (dst !== null) resolvedLink = true;\n appendable.push([path, target, targetBase, dst, embed ? 1 : 0]);\n }\n }\n await appendRows(db, 'links', resolveNew ? LINK_COLUMNS_RESOLVED : LINK_COLUMNS, resolveNew ? UPSERT_LINK_SQL_RESOLVED : UPSERT_LINK_SQL, appendable);\n if (upsertRows.length > 0) await db.runBatch(UPSERT_LINK_SQL, upsertRows);\n if (cold) coldResolved.set(delta, resolvedLink);\n },\n async afterReconcile(db, delta) {\n // Any deleted rows mean the edge set shrank -- whether the file vanished or was reparsed\n // down to fewer/no links.\n const removeHadLinks = removedWithLinks.get(delta) ?? false;\n\n // A cold build is the only case a full pass wins. Measured 2026-09-01 on 6,566 notes, resolve\n // time alone: full is flat near 90ms at any churn, incremental 7ms at 10 files and 30ms at 2,000.\n const large = delta.files.length === 0 || delta.added.length === delta.files.length;\n // store() already resolved dst inline on this cold build (same test), so nothing is left to update.\n if (large) {\n delta.linksChanged = (coldResolved.get(delta) ?? false) || removeHadLinks;\n return;\n }\n\n const dstChanged = await resolveIncremental(db, delta);\n delta.linksChanged = dstChanged || removeHadLinks;\n },\n};\n"],"names":["posix","appendRows","maskRegions","parseWikilinkInner","inner","beforeAlias","split","trim","startsWith","length","target","extract","_raw","rawBody","_search","data","body","test","seen","Set","results","add","embed","key","has","push","m","matchAll","index","dest","includes","value","Object","values","item","Array","isArray","exec","cleanTarget","replace","baseKey","path","basename","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","candidates","get","buildByBase","files","Map","paths","map","f","relPath","sort","a","b","list","set","resolveRows","rows","changed","row","dst","writeResolved","db","runBatch","r","selectIn","column","keys","i","chunk","slice","stmt","prepare","all","resolveIncremental","delta","changedBasenames","p","added","vanished","collect","reparsed","LINK_EDGES_SQL","toEdges","linkEdges","removedWithLinks","WeakMap","coldResolved","LINK_COLUMNS","UPSERT_LINK_SQL","LINK_COLUMNS_RESOLVED","UPSERT_LINK_SQL_RESOLVED","links","name","schema","remove","vanishedSet","filter","existingRows","store","docs","byPath","d","extracted","addedSet","reparsedExisting","staleRows","targets","some","t","cold","resolveNew","resolvedLink","appendable","upsertRows","targetBase","afterReconcile","removeHadLinks","large","linksChanged","dstChanged"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,SAASC,UAAU,QAAQ,qBAAqB;AAEhD,SAASC,WAAW,QAAQ,cAAc;AAG1C,0JAA0J;AAC1J,mJAAmJ;AAEnJ,wFAAwF;AACxF,gEAAgE;AAChE,SAASC,mBAAmBC,KAAa;IACvC,MAAMC,cAAcD,MAAME,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC5C,IAAIF,YAAYG,UAAU,CAAC,MAAM,OAAOH,YAAYI,MAAM,GAAG,IAAIJ,cAAc;IAC/E,MAAMK,SAASL,YAAYC,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC7C,OAAOG,UAAU;AACnB;AAEA,4FAA4F;AAC5F,0EAA0E;AAC1E,SAASC,QAAQC,IAAY,EAAEC,OAAe,EAAEC,OAA4C,EAAEC,IAA8B;IAC1H,MAAMC,OAAO,YAAYC,IAAI,CAACJ,WAAWX,YAAYW,WAAWA;IAChE,MAAMK,OAAO,IAAIC;IACjB,MAAMC,UAAqD,EAAE;IAC7D,MAAMC,MAAM,CAACX,QAAgBY;QAC3B,MAAMC,MAAM,GAAGb,OAAO,EAAE,EAAEY,QAAQ,MAAM,KAAK;QAC7C,IAAIJ,KAAKM,GAAG,CAACD,MAAM;QACnBL,KAAKG,GAAG,CAACE;QACTH,QAAQK,IAAI,CAAC;YAAEf;YAAQY;QAAM;IAC/B;IACA,uFAAuF;IACvF,6DAA6D;IAC7D,KAAK,MAAMI,KAAKV,KAAKW,QAAQ,CAAC,kBAAmB;YAEjBD;QAD9B,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;QACtC,IAAIhB,QAAQW,IAAIX,QAAQM,IAAI,CAAC,EAACU,WAAAA,EAAEE,KAAK,cAAPF,sBAAAA,WAAW,KAAK,EAAE,KAAK;IACvD;IACA,2FAA2F;IAC3F,8FAA8F;IAC9F,KAAK,MAAMA,KAAKV,KAAKW,QAAQ,CAAC,6EAA8E;QAC1G,MAAME,OAAOH,CAAC,CAAC,EAAE,CAACnB,IAAI;QACtB,qFAAqF;QACrF,oFAAoF;QACpF,IAAI,CAACsB,QAAQA,KAAKC,QAAQ,CAAC,QAAQ;QACnC,IAAI,oCAAoCb,IAAI,CAACY,OAAO;QACpD,MAAMnB,SAASmB,KAAKrB,UAAU,CAAC,OAAOqB,OAAOA,KAAKvB,KAAK,CAAC,IAAI,CAAC,EAAE;QAC/D,IAAII,QAAQW,IAAIX,QAAQgB,CAAC,CAAC,EAAE,KAAK;IACnC;IACA,KAAK,MAAMK,SAASC,OAAOC,MAAM,CAAClB,iBAAAA,kBAAAA,OAAQ,CAAC,GAAI;QAC7C,KAAK,MAAMmB,QAAQC,MAAMC,OAAO,CAACL,SAASA,QAAQ;YAACA;SAAM,CAAE;YACzD,IAAI,OAAOG,SAAS,UAAU;YAC9B,MAAMR,IAAI,iBAAiBW,IAAI,CAACH,KAAK3B,IAAI;YACzC,IAAI,CAACmB,GAAG;YACR,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;YACtC,IAAIhB,QAAQW,IAAIX,QAAQ;QAC1B;IACF;IACA,OAAOU;AACT;AAEA,SAASkB,YAAY5B,MAAc;IACjC,OAAOA,OAAO6B,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;AACrD;AAEA,SAASC,QAAQC,IAAY;IAC3B,OAAOzC,MAAM0C,QAAQ,CAACD,MAAMF,OAAO,CAAC,UAAU,IAAII,WAAW;AAC/D;AAEA,8FAA8F;AAC9F,kGAAkG;AAClG,SAASC,cAAcC,GAAW,EAAEnC,MAAc,EAAEoC,OAAoB,EAAEC,MAA6B;IACrG,4FAA4F;IAC5F,IAAIrC,OAAOF,UAAU,CAAC,MAAM,OAAOqC;IACnC,MAAMG,QAAQV,YAAY5B;IAC1B,MAAMuC,UAAUjD,MAAMkD,SAAS,CAAClD,MAAMmD,IAAI,CAACnD,MAAMoD,OAAO,CAACP,MAAMG;IAC/D,KAAK,MAAMK,aAAa;QAACL;QAAO,GAAGA,MAAM,GAAG,CAAC;QAAEC;QAAS,GAAGA,QAAQ,GAAG,CAAC;KAAC,CAAE;QACxE,IAAIH,QAAQtB,GAAG,CAAC6B,YAAY,OAAOA;IACrC;IACA,MAAMC,aAAaP,OAAOQ,GAAG,CAACf,QAAQQ;IACtC,IAAI,CAACM,YAAY,OAAO;IACxB,0FAA0F;IAC1F,+DAA+D;IAC/D,OAAOA,WAAWxB,QAAQ,CAACe,OAAOA,MAAMS,UAAU,CAAC,EAAE;AACvD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,SAASE,YAAYC,KAAiB;IACpC,MAAMV,SAAS,IAAIW;IACnB,MAAMC,QAAQF,MAAMG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEvD,MAAM,GAAGwD,EAAExD,MAAM,IAAKuD,CAAAA,IAAIC,IAAI,CAAC,IAAID,IAAIC,IAAI,IAAI,CAAA;IAC1G,KAAK,MAAMxB,QAAQkB,MAAO;QACxB,MAAMpC,MAAMiB,QAAQC;QACpB,MAAMyB,OAAOnB,OAAOQ,GAAG,CAAChC;QACxB,IAAI2C,MAAMA,KAAKzC,IAAI,CAACgB;aACfM,OAAOoB,GAAG,CAAC5C,KAAK;YAACkB;SAAK;IAC7B;IACA,OAAOM;AACT;AASA,4FAA4F;AAC5F,uCAAuC;AACvC,SAASqB,YAAYC,IAAe,EAAEvB,OAAoB,EAAEC,MAA6B;IACvF,MAAMuB,UAAqB,EAAE;IAC7B,KAAK,MAAMC,OAAOF,KAAM;QACtB,MAAMG,MAAM5B,cAAc2B,IAAI1B,GAAG,EAAE0B,IAAI7D,MAAM,EAAEoC,SAASC;QACxD,IAAIyB,QAAQD,IAAIC,GAAG,EAAEF,QAAQ7C,IAAI,CAAC;YAAE,GAAG8C,GAAG;YAAEC;QAAI;IAClD;IACA,OAAOF;AACT;AAEA,eAAeG,cAAcC,EAAc,EAAEL,IAAe;IAC1D,IAAIA,KAAK5D,MAAM,KAAK,GAAG;IACvB,MAAMiE,GAAGC,QAAQ,CACf,yDACAN,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAEJ,GAAG;YAAEI,EAAE/B,GAAG;YAAE+B,EAAElE,MAAM;SAAC;AAE5C;AAEA,2FAA2F;AAC3F,2FAA2F;AAC3F,eAAemE,SAASH,EAAc,EAAEI,MAAc,EAAEC,IAAc;IACpE,MAAMV,OAAkB,EAAE;IAC1B,IAAK,IAAIW,IAAI,GAAGA,IAAID,KAAKtE,MAAM,EAAEuE,KAAK,IAAK;QACzC,MAAMC,QAAQF,KAAKG,KAAK,CAACF,GAAGA,IAAI;QAChC,MAAMG,OAAO,MAAMT,GAAGU,OAAO,CAAC,CAAC,gDAAgD,EAAEN,OAAO,KAAK,EAAEG,MAAMrB,GAAG,CAAC,IAAM,KAAKT,IAAI,CAAC,MAAM,CAAC,CAAC;QACjIkB,KAAK5C,IAAI,IAAM,MAAM0D,KAAKE,GAAG,IAAIJ;IACnC;IACA,OAAOZ;AACT;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,eAAeiB,mBAAmBZ,EAAc,EAAEa,KAAqB;IACrE,MAAMzC,UAAU,IAAI3B,IAAIoE,MAAM9B,KAAK,CAACG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACxD,MAAMf,SAASS,YAAY+B,MAAM9B,KAAK;IAEtC,MAAM+B,mBAAmB,IAAIrE;IAC7B,KAAK,MAAMsE,KAAKF,MAAMG,KAAK,CAAEF,iBAAiBnE,GAAG,CAACmB,QAAQiD;IAC1D,KAAK,MAAMA,KAAKF,MAAMI,QAAQ,CAAEH,iBAAiBnE,GAAG,CAACmB,QAAQiD;IAE7D,+EAA+E;IAC/E,MAAMnC,aAAa,IAAII;IACvB,MAAMkC,UAAU,CAACvB;QACf,KAAK,MAAME,OAAOF,KAAMf,WAAWa,GAAG,CAAC,GAAGI,IAAI1B,GAAG,CAAC,EAAE,EAAE0B,IAAI7D,MAAM,CAAC,EAAE,EAAE6D,IAAIjD,KAAK,EAAE,EAAEiD;IACpF;IACAqB,QAAQ,MAAMf,SAASH,IAAI,OAAOa,MAAMM,QAAQ;IAChDD,QAAQ,MAAMf,SAASH,IAAI,eAAe;WAAIc;KAAiB;IAC/DI,QAAQ,MAAMf,SAASH,IAAI,OAAOa,MAAMI,QAAQ;IAEhD,MAAMrB,UAAUF,YAAY;WAAId,WAAWrB,MAAM;KAAG,EAAEa,SAASC;IAC/D,MAAM0B,cAAcC,IAAIJ;IACxB,OAAOA,QAAQ7D,MAAM,GAAG;AAC1B;AAEA,8FAA8F;AAC9F,wEAAwE;AACxE,OAAO,MAAMqF,iBAAiB,CAAC;;;4GAG6E,CAAC,CAAC;AAE9G,OAAO,SAASC,QAAQ1B,IAAyC;IAC/D,OAAOA,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAE/B,GAAG;YAAE+B,EAAEJ,GAAG;SAAC;AACvC;AAEA,qEAAqE;AACrE,OAAO,eAAewB,UAAUtB,EAAc;IAC5C,MAAMS,OAAO,MAAMT,GAAGU,OAAO,CAACU;IAC9B,OAAOC,QAAS,MAAMZ,KAAKE,GAAG;AAChC;AAEA,8FAA8F;AAC9F,qGAAqG;AACrG,MAAMY,mBAAmB,IAAIC;AAE7B,kGAAkG;AAClG,sEAAsE;AACtE,MAAMC,eAAe,IAAID;AAEzB,iGAAiG;AACjG,kFAAkF;AAClF,MAAME,eAAe;IAAC;IAAO;IAAU;IAAe;CAAQ;AAC9D,MAAMC,kBAAkB;AACxB,MAAMC,wBAAwB;IAAC;IAAO;IAAU;IAAe;IAAO;CAAQ;AAC9E,MAAMC,2BAA2B;AAEjC,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOhC,EAAE;QACb,MAAMA,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;IAChB;IACA1B;IACA,4FAA4F;IAC5F,4FAA4F;IAC5F,MAAMgG,QAAOjC,EAAE,EAAEf,KAAK,EAAE4B,KAAK;QAC3B,MAAMqB,cAAc,IAAIzF,IAAIoE,MAAMI,QAAQ;QAC1C,MAAMA,WAAWhC,MAAMkD,MAAM,CAAC,CAACpB,IAAMmB,YAAYpF,GAAG,CAACiE;QACrD,IAAIE,SAASlF,MAAM,KAAK,GAAG;QAC3B,MAAMqG,eAAe,MAAMjC,SAASH,IAAI,OAAOiB;QAC/C,MAAMjB,GAAGC,QAAQ,CACf,mCACAgB,SAAS/B,GAAG,CAAC,CAAC6B,IAAM;gBAACA;aAAE;QAEzB,IAAIqB,aAAarG,MAAM,GAAG,GAAGwF,iBAAiB9B,GAAG,CAACoB,OAAO;IAC3D;IACA,MAAMwB,OAAMrC,EAAE,EAAEsC,IAAoB,EAAEzB,KAAK;QACzC,MAAM0B,SAAS,IAAIvD,IAAIsD,KAAKpD,GAAG,CAAC,CAACsD,IAAM;gBAACA,EAAEzE,IAAI;gBAAEyE,EAAEC,SAAS;aAA8C;QACzG,MAAMC,WAAW,IAAIjG,IAAIoE,MAAMG,KAAK;QACpC,MAAM2B,mBAAmBL,KAAKpD,GAAG,CAAC,CAACsD,IAAMA,EAAEzE,IAAI,EAAEoE,MAAM,CAAC,CAACpB,IAAM,CAAC2B,SAAS5F,GAAG,CAACiE;QAE7E,uFAAuF;QACvF,8FAA8F;QAC9F,MAAMqB,eAAe,MAAMjC,SAASH,IAAI,OAAO2C;QAC/C,MAAMC,YAAYR,aAAaD,MAAM,CAAC,CAACtC;gBACrB0C;YAAhB,MAAMM,WAAUN,cAAAA,OAAO1D,GAAG,CAACgB,IAAI1B,GAAG,eAAlBoE,yBAAAA,cAAuB,EAAE;YACzC,OAAO,CAACM,QAAQC,IAAI,CAAC,CAACC,IAAMA,EAAE/G,MAAM,KAAK6D,IAAI7D,MAAM,IAAI,AAAC+G,CAAAA,EAAEnG,KAAK,GAAG,IAAI,CAAA,MAAOiD,IAAIjD,KAAK;QACxF;QACA,IAAIgG,UAAU7G,MAAM,GAAG,GAAG;YACxB,MAAMiE,GAAGC,QAAQ,CACf,gEACA2C,UAAU1D,GAAG,CAAC,CAACgB,IAAM;oBAACA,EAAE/B,GAAG;oBAAE+B,EAAElE,MAAM;oBAAEkE,EAAEtD,KAAK;iBAAC;YAEjD2E,iBAAiB9B,GAAG,CAACoB,OAAO;QAC9B;QAEA,uFAAuF;QACvF,qFAAqF;QACrF,MAAMmC,OAAOnC,MAAM9B,KAAK,CAAChD,MAAM,KAAK,KAAK8E,MAAMG,KAAK,CAACjF,MAAM,KAAK8E,MAAM9B,KAAK,CAAChD,MAAM;QAClF,IAAIkH;QACJ,IAAID,MAAM;YACR,MAAM5E,UAAU,IAAI3B,IAAIoE,MAAM9B,KAAK,CAACG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;YACxD,MAAMf,SAASS,YAAY+B,MAAM9B,KAAK;YACtCkE,aAAa,CAAC9E,KAAKnC,SAAWkC,cAAcC,KAAKnC,QAAQoC,SAASC;QACpE;QACA,IAAI6E,eAAe;QAEnB,gGAAgG;QAChG,sGAAsG;QACtG,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMC,aAA0B,EAAE;QAClC,MAAMC,aAA0B,EAAE;QAClC,KAAK,MAAM,CAACrF,MAAM8E,QAAQ,IAAIN,OAAQ;YACpC,KAAK,MAAM,EAAEvG,MAAM,EAAEY,KAAK,EAAE,IAAIiG,QAAS;gBACvC,MAAMQ,aAAarH,OAAOF,UAAU,CAAC,OAAO,OAAOgC,QAAQF,YAAY5B;gBACvE,IAAI,CAAC0G,SAAS5F,GAAG,CAACiB,OAAO;oBACvBqF,WAAWrG,IAAI,CAAC;wBAACgB;wBAAM/B;wBAAQqH;wBAAYzG,QAAQ,IAAI;qBAAE;oBACzD;gBACF;gBACA,IAAI,CAACqG,YAAY;oBACfE,WAAWpG,IAAI,CAAC;wBAACgB;wBAAM/B;wBAAQqH;wBAAYzG,QAAQ,IAAI;qBAAE;oBACzD;gBACF;gBACA,MAAMkD,MAAMmD,WAAWlF,MAAM/B;gBAC7B,IAAI8D,QAAQ,MAAMoD,eAAe;gBACjCC,WAAWpG,IAAI,CAAC;oBAACgB;oBAAM/B;oBAAQqH;oBAAYvD;oBAAKlD,QAAQ,IAAI;iBAAE;YAChE;QACF;QACA,MAAMrB,WAAWyE,IAAI,SAASiD,aAAarB,wBAAwBF,cAAcuB,aAAapB,2BAA2BF,iBAAiBwB;QAC1I,IAAIC,WAAWrH,MAAM,GAAG,GAAG,MAAMiE,GAAGC,QAAQ,CAAC0B,iBAAiByB;QAC9D,IAAIJ,MAAMvB,aAAahC,GAAG,CAACoB,OAAOqC;IACpC;IACA,MAAMI,gBAAetD,EAAE,EAAEa,KAAK;YAGLU;QAFvB,yFAAyF;QACzF,0BAA0B;QAC1B,MAAMgC,kBAAiBhC,wBAAAA,iBAAiB1C,GAAG,CAACgC,oBAArBU,mCAAAA,wBAA+B;QAEtD,8FAA8F;QAC9F,kGAAkG;QAClG,MAAMiC,QAAQ3C,MAAM9B,KAAK,CAAChD,MAAM,KAAK,KAAK8E,MAAMG,KAAK,CAACjF,MAAM,KAAK8E,MAAM9B,KAAK,CAAChD,MAAM;QACnF,oGAAoG;QACpG,IAAIyH,OAAO;gBACa/B;YAAtBZ,MAAM4C,YAAY,GAAG,EAAChC,oBAAAA,aAAa5C,GAAG,CAACgC,oBAAjBY,+BAAAA,oBAA2B,UAAU8B;YAC3D;QACF;QAEA,MAAMG,aAAa,MAAM9C,mBAAmBZ,IAAIa;QAChDA,MAAM4C,YAAY,GAAGC,cAAcH;IACrC;AACF,EAAE"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { estimateTokens, parse } from '../chunk/index.js';
|
|
2
2
|
import { countLines } from '../scan/frontmatter.js';
|
|
3
3
|
import { appendRows } from '../store/shared.js';
|
|
4
|
-
// Heading blocks parse() already found (
|
|
4
|
+
// Heading blocks parse() already found (markdown-it/CommonMark fences), offset back onto the raw file;
|
|
5
5
|
// a section runs to just before the next heading, or EOF.
|
|
6
6
|
function sectionsFromBlocks(blocks, raw, body) {
|
|
7
7
|
const rawLines = raw.split('\n');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n level: number;\n heading: string;\n startLine: number;\n endLine: number;\n tokens: number;\n}\n\n// Heading blocks parse() already found (
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n level: number;\n heading: string;\n startLine: number;\n endLine: number;\n tokens: number;\n}\n\n// Heading blocks parse() already found (markdown-it/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n const rawLines = raw.split('\\n');\n const offset = rawLines.length - countLines(body);\n const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n for (let s = 0; s < found.length; s++) {\n if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n found[s].tokens = Math.ceil(estimateTokens(text));\n }\n return found;\n}\n\nconst SECTION_COLUMNS = ['path', 'idx', 'level', 'heading', 'start_line', 'end_line', 'tokens'];\n\nexport const sections: Feature = {\n name: 'sections',\n async schema(db) {\n await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n },\n extract(raw, body, _search, _data, _cfg, blocks) {\n return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n },\n async remove(db, paths) {\n if (paths.length === 0) return;\n await db.runBatch(\n 'DELETE FROM sections WHERE \"path\" = ?',\n paths.map((p) => [p])\n );\n },\n async store(db, docs) {\n const rows: unknown[][] = [];\n for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n // write's lock, so a path called \"added\" here can already have this row from a concurrent\n // reconcile that committed first -- same file, same parse, same row. A store with an append\n // path has no second writer, and remove() cleared every touched path above, so it skips the guard.\n await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n },\n};\n"],"names":["estimateTokens","parse","countLines","appendRows","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","SECTION_COLUMNS","sections","name","schema","db","exec","extract","_search","_data","_cfg","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push"],"mappings":"AACA,SAASA,cAAc,EAAEC,KAAK,QAAQ,oBAAoB;AAC1D,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAchD,uGAAuG;AACvG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,MAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,MAAMC,SAASF,SAASG,MAAM,GAAGT,WAAWK;IAC5C,MAAMK,QAAmBP,OAAOQ,MAAM,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,WAAWC,GAAG,CAAC,CAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGZ;YAAQa,SAASf,SAASG,MAAM;YAAEa,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMD,MAAM,EAAEc,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMD,MAAM,EAAEC,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,MAAMF,OAAOZ,SAASkB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAAC7B,eAAeoB;IAC7C;IACA,OAAOR;AACT;AAEA,MAAMkB,kBAAkB;IAAC;IAAQ;IAAO;IAAS;IAAW;IAAc;IAAY;CAAS;AAE/F,OAAO,MAAMC,WAAoB;IAC/BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,4KAA4K,CAAC;IAC9L;IACAC,SAAQ9B,GAAG,EAAEC,IAAI,EAAE8B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAElC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUJ,MAAMM,OAAOD,KAAKC;IACxD;IACA,MAAMiC,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAM9B,MAAM,KAAK,GAAG;QACxB,MAAMuB,GAAGQ,QAAQ,CACf,yCACAD,MAAMzB,GAAG,CAAC,CAAC2B,IAAM;gBAACA;aAAE;IAExB;IACA,MAAMC,OAAMV,EAAE,EAAEW,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM,AAACG,UAAwBC,OAAO,CAAC,CAACxB,GAAGyB,MAAQJ,KAAKK,IAAI,CAAC;gBAACJ;gBAAMG;gBAAKzB,EAAER,KAAK;gBAAEQ,EAAEN,OAAO;gBAAEM,EAAEH,SAAS;gBAAEG,EAAEF,OAAO;gBAAEE,EAAED,MAAM;aAAC;QAChK,wFAAwF;QACxF,0FAA0F;QAC1F,4FAA4F;QAC5F,mGAAmG;QACnG,MAAMrB,WAAW+B,IAAI,YAAYJ,iBAAiB,qJAAqJgB;IACzM;AACF,EAAE"}
|
package/dist/esm/scan/pool.js
CHANGED
|
@@ -40,7 +40,7 @@ export class ParsePool {
|
|
|
40
40
|
return this.pool;
|
|
41
41
|
}
|
|
42
42
|
// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
|
|
43
|
-
// extracted text and feature values, never the
|
|
43
|
+
// extracted text and feature values, never the token tree.
|
|
44
44
|
async run(files, features, cfg, onParsed, maxWorkers) {
|
|
45
45
|
// cfg and the feature selection are constant for the dispatch, so they cross once per worker
|
|
46
46
|
// as workerData. Features carry closures and cannot cross at all; only their names do.
|