sensemaking 0.17.0 → 0.18.0

package/dist/esm/commands/peek.js

@@ -17,54 +17,62 @@ export function resolveNote(paths, arg) {
 const PEEK_LIST_LIMIT = 20;
 // peek: everything about one note except its prose -- frontmatter, outline with line
 // ranges + token estimates (so the follow-up Read is a range, not the file), links both ways.
-export function peek(db, cfg, pathArg, overrides = {}) {
-    var _row__parse_error, _row__size;
-    var _db_prepare_get;
-    const paths = db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path);
-    const path = resolveNote(paths, pathArg);
-    const row = db.prepare('SELECT * FROM frontmatter WHERE "path" = ?').get(path);
-    const parseError = (_row__parse_error = row._parse_error) !== null && _row__parse_error !== void 0 ? _row__parse_error : null;
-    const frontmatter = {};
-    for (const [key, value] of Object.entries(row)){
-        if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;
-    }
-    const sectionsTotal = featureEnabled(cfg, 'sections') ? db.prepare('SELECT COUNT(*) AS n FROM sections WHERE "path" = ?').get(path).n : 0;
-    const sections = featureEnabled(cfg, 'sections') ? db.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE "path" = ? ORDER BY idx LIMIT ?').all(path, PEEK_LIST_LIMIT) : [];
-    let outbound = [];
-    let backlinks = [];
-    let unresolved = [];
-    let backlinksTotal = 0;
-    const _allowed = scopedPaths(db, cfg, overrides);
-    if (featureEnabled(cfg, 'links')) {
-        const out = db.prepare('SELECT target, dst FROM links WHERE src = ? AND (dst IS NULL OR dst != src) ORDER BY target').all(path);
-        outbound = [
-            ...new Set(out.filter((l)=>l.dst !== null).map((l)=>l.dst))
-        ];
-        unresolved = out.filter((l)=>l.dst === null).map((l)=>l.target);
-        backlinksTotal = db.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ? AND src != dst').get(path).n;
-        backlinks = db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst ORDER BY src LIMIT ?').all(path, PEEK_LIST_LIMIT).map((r)=>r.src);
-    }
-    // content.text is the stripped body already computed at reconcile time (no extra file read),
-    // floored by the byte/4 estimate so stripped-away syntax and frontmatter can't vanish from the price.
-    const body = (_db_prepare_get = db.prepare('SELECT text FROM content WHERE "path" = ?').get(path)) === null || _db_prepare_get === void 0 ? void 0 : _db_prepare_get.text;
-    const byteTokens = Math.ceil(((_row__size = row._size) !== null && _row__size !== void 0 ? _row__size : 0) / 4);
-    const tokens = body !== undefined ? Math.max(Math.ceil(estimateTokens(body)), byteTokens) : byteTokens;
-    return {
-        path,
-        tokens,
-        frontmatter,
-        parseError,
-        sections,
-        outbound: outbound.slice(0, PEEK_LIST_LIMIT),
-        backlinks,
-        unresolved: unresolved.slice(0, PEEK_LIST_LIMIT),
-        sectionsTotal,
-        outboundTotal: outbound.length,
-        backlinksTotal,
-        unresolvedTotal: unresolved.length,
-        off: [
-            'sections',
-            'links'
-        ].filter((name)=>!featureEnabled(cfg, name))
-    };
+export async function peek(store, cfg, pathArg, overrides = {}) {
+    return store.transaction(async ()=>{
+        var _row__parse_error, _row__size;
+        var _this;
+        const pathsStmt = await store.prepare('SELECT "path" FROM frontmatter');
+        const paths = (await pathsStmt.all()).map((r)=>r.path);
+        const path = resolveNote(paths, pathArg);
+        const rowStmt = await store.prepare('SELECT * FROM frontmatter WHERE "path" = ?');
+        const row = await rowStmt.get(path);
+        const parseError = (_row__parse_error = row._parse_error) !== null && _row__parse_error !== void 0 ? _row__parse_error : null;
+        const frontmatter = {};
+        for (const [key, value] of Object.entries(row)){
+            if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;
+        }
+        let sectionsTotal = 0;
+        let sections = [];
+        if (featureEnabled(cfg, 'sections')) {
+            sectionsTotal = (await (await store.prepare('SELECT COUNT(*) AS n FROM sections WHERE "path" = ?')).get(path)).n;
+            sections = await (await store.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE "path" = ? ORDER BY idx LIMIT ?')).all(path, PEEK_LIST_LIMIT);
+        }
+        let outbound = [];
+        let backlinks = [];
+        let unresolved = [];
+        let backlinksTotal = 0;
+        await scopedPaths(store, cfg, overrides); // validates --preset/--where/--include/--exclude even though peek's own output isn't scope-filtered
+        if (featureEnabled(cfg, 'links')) {
+            const out = await (await store.prepare('SELECT target, dst FROM links WHERE src = ? AND (dst IS NULL OR dst != src) ORDER BY target')).all(path);
+            outbound = [
+                ...new Set(out.filter((l)=>l.dst !== null).map((l)=>l.dst))
+            ];
+            unresolved = out.filter((l)=>l.dst === null).map((l)=>l.target);
+            backlinksTotal = (await (await store.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ? AND src != dst')).get(path)).n;
+            backlinks = (await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst ORDER BY src LIMIT ?')).all(path, PEEK_LIST_LIMIT)).map((r)=>r.src);
+        }
+        // content.text is the stripped body already computed at reconcile time (no extra file read),
+        // floored by the byte/4 estimate so stripped-away syntax and frontmatter can't vanish from the price.
+        const body = (_this = await (await store.prepare('SELECT text FROM content WHERE "path" = ?')).get(path)) === null || _this === void 0 ? void 0 : _this.text;
+        const byteTokens = Math.ceil(((_row__size = row._size) !== null && _row__size !== void 0 ? _row__size : 0) / 4);
+        const tokens = body !== undefined ? Math.max(Math.ceil(estimateTokens(body)), byteTokens) : byteTokens;
+        return {
+            path,
+            tokens,
+            frontmatter,
+            parseError,
+            sections,
+            outbound: outbound.slice(0, PEEK_LIST_LIMIT),
+            backlinks,
+            unresolved: unresolved.slice(0, PEEK_LIST_LIMIT),
+            sectionsTotal,
+            outboundTotal: outbound.length,
+            backlinksTotal,
+            unresolvedTotal: unresolved.length,
+            off: [
+                'sections',
+                'links'
+            ].filter((name)=>!featureEnabled(cfg, name))
+        };
+    });
 }

package/dist/esm/commands/peek.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/peek.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport { estimateTokens } from '../chunk/index.ts';\nimport type { FeatureName, ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { featureEnabled } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { INTERNAL_COLUMNS, scopedPaths } from './scope.ts';\n\n// Note resolution shared by peek and path: an exact path, or a unique basename (case\n// insensitive, .md stripped).\nexport function resolveNote(paths: string[], arg: string): string {\n  const exact = paths.find((p) => p === arg);\n  if (exact) return exact;\n  const base = posix.basename(arg).replace(/\\.md$/i, '').toLowerCase();\n  const matches = paths.filter((p) => posix.basename(p).replace(/\\.md$/i, '').toLowerCase() === base);\n  if (matches.length === 1) return matches[0];\n  if (matches.length > 1) throw new SenseError('NOTE_AMBIGUOUS', `\"${arg}\" is ambiguous: ${matches.join(', ')}`);\n  throw new SenseError('NOTE_NOT_FOUND', `no note matches \"${arg}\"`);\n}\n\nexport interface Peek {\n  path: string;\n  tokens: number;\n  frontmatter: Row;\n  // Set when the note's frontmatter was refused, so an empty frontmatter block reads as \"did\n  // not parse\" rather than \"has none\". Same reason `_parse_error` sits in the row.\n  parseError: string | null;\n  sections: Row[];\n  outbound: string[];\n  backlinks: string[];\n  unresolved: string[];\n  // Totals before truncation: a hub can have thousands of backlinks (or a note thousands of\n  // headings), and peek's whole point is bounded output. Query the sections/links tables\n  // directly for the full list.\n  sectionsTotal: number;\n  outboundTotal: number;\n  backlinksTotal: number;\n  unresolvedTotal: number;\n  // Bounded k-hop expansion beyond the immediate ring already shown by outbound/backlinks\n  // (depth starts at 2).\n  off: FeatureName[]; // disabled features whose blocks are omitted (not empty)\n}\n\nconst PEEK_LIST_LIMIT = 20;\n\n// peek: everything about one note except its prose -- frontmatter, outline with line\n// ranges + token estimates (so the follow-up Read is a range, not the file), links both ways.\nexport function peek(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides = {}): Peek {\n  const paths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const path = resolveNote(paths, pathArg);\n\n  const row = db.prepare('SELECT * FROM frontmatter WHERE \"path\" = ?').get(path) as Row;\n  const parseError = (row._parse_error as string | null) ?? null;\n  const frontmatter: Row = {};\n  for (const [key, value] of Object.entries(row)) {\n    if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;\n  }\n\n  const sectionsTotal = featureEnabled(cfg, 'sections') ? (db.prepare('SELECT COUNT(*) AS n FROM sections WHERE \"path\" = ?').get(path) as { n: number }).n : 0;\n  const sections = featureEnabled(cfg, 'sections') ? (db.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE \"path\" = ? ORDER BY idx LIMIT ?').all(path, PEEK_LIST_LIMIT) as Row[]) : [];\n\n  let outbound: string[] = [];\n  let backlinks: string[] = [];\n  let unresolved: string[] = [];\n  let backlinksTotal = 0;\n  const _allowed = scopedPaths(db, cfg, overrides);\n  if (featureEnabled(cfg, 'links')) {\n    const out = db.prepare('SELECT target, dst FROM links WHERE src = ? AND (dst IS NULL OR dst != src) ORDER BY target').all(path) as Array<{ target: string; dst: string | null }>;\n    outbound = [...new Set(out.filter((l) => l.dst !== null).map((l) => l.dst as string))];\n    unresolved = out.filter((l) => l.dst === null).map((l) => l.target);\n    backlinksTotal = (db.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ? AND src != dst').get(path) as { n: number }).n;\n    backlinks = (db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst ORDER BY src LIMIT ?').all(path, PEEK_LIST_LIMIT) as Array<{ src: string }>).map((r) => r.src);\n  }\n\n  // content.text is the stripped body already computed at reconcile time (no extra file read),\n  // floored by the byte/4 estimate so stripped-away syntax and frontmatter can't vanish from the price.\n  const body = (db.prepare('SELECT text FROM content WHERE \"path\" = ?').get(path) as { text: string } | undefined)?.text;\n  const byteTokens = Math.ceil(((row._size as number) ?? 0) / 4);\n  const tokens = body !== undefined ? Math.max(Math.ceil(estimateTokens(body)), byteTokens) : byteTokens;\n\n  return {\n    path,\n    tokens,\n    frontmatter,\n    parseError,\n    sections,\n    outbound: outbound.slice(0, PEEK_LIST_LIMIT),\n    backlinks,\n    unresolved: unresolved.slice(0, PEEK_LIST_LIMIT),\n    sectionsTotal,\n    outboundTotal: outbound.length,\n    backlinksTotal,\n    unresolvedTotal: unresolved.length,\n    off: (['sections', 'links'] as FeatureName[]).filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n"],"names":["posix","estimateTokens","featureEnabled","SenseError","INTERNAL_COLUMNS","scopedPaths","resolveNote","paths","arg","exact","find","p","base","basename","replace","toLowerCase","matches","filter","length","join","PEEK_LIST_LIMIT","peek","db","cfg","pathArg","overrides","row","prepare","all","map","r","path","get","parseError","_parse_error","frontmatter","key","value","Object","entries","has","sectionsTotal","n","sections","outbound","backlinks","unresolved","backlinksTotal","_allowed","out","Set","l","dst","target","src","body","text","byteTokens","Math","ceil","_size","tokens","undefined","max","slice","outboundTotal","unresolvedTotal","off","name"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,SAASC,cAAc,QAAQ,oBAAoB;AAEnD,SAASC,cAAc,QAAQ,qBAAqB;AACpD,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,gBAAgB,EAAEC,WAAW,QAAQ,aAAa;AAE3D,qFAAqF;AACrF,8BAA8B;AAC9B,OAAO,SAASC,YAAYC,KAAe,EAAEC,GAAW;IACtD,MAAMC,QAAQF,MAAMG,IAAI,CAAC,CAACC,IAAMA,MAAMH;IACtC,IAAIC,OAAO,OAAOA;IAClB,MAAMG,OAAOZ,MAAMa,QAAQ,CAACL,KAAKM,OAAO,CAAC,UAAU,IAAIC,WAAW;IAClE,MAAMC,UAAUT,MAAMU,MAAM,CAAC,CAACN,IAAMX,MAAMa,QAAQ,CAACF,GAAGG,OAAO,CAAC,UAAU,IAAIC,WAAW,OAAOH;IAC9F,IAAII,QAAQE,MAAM,KAAK,GAAG,OAAOF,OAAO,CAAC,EAAE;IAC3C,IAAIA,QAAQE,MAAM,GAAG,GAAG,MAAM,IAAIf,WAAW,kBAAkB,CAAC,CAAC,EAAEK,IAAI,gBAAgB,EAAEQ,QAAQG,IAAI,CAAC,OAAO;IAC7G,MAAM,IAAIhB,WAAW,kBAAkB,CAAC,iBAAiB,EAAEK,IAAI,CAAC,CAAC;AACnE;AAyBA,MAAMY,kBAAkB;AAExB,qFAAqF;AACrF,8FAA8F;AAC9F,OAAO,SAASC,KAAKC,EAAgB,EAAEC,GAAmB,EAAEC,OAAe,EAAEC,YAA6B,CAAC,CAAC;QAKtFC,mBAyBWA;QADjBJ;IA5Bd,MAAMf,QAAQ,AAACe,GAAGK,OAAO,CAAC,kCAAkCC,GAAG,GAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAC/G,MAAMA,OAAOzB,YAAYC,OAAOiB;IAEhC,MAAME,MAAMJ,GAAGK,OAAO,CAAC,8CAA8CK,GAAG,CAACD;IACzE,MAAME,cAAcP,oBAAAA,IAAIQ,YAAY,cAAhBR,+BAAAA,oBAAsC;IAC1D,MAAMS,cAAmB,CAAC;IAC1B,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAOC,OAAO,CAACb,KAAM;QAC9C,IAAI,CAACtB,iBAAiBoC,GAAG,CAACJ,QAAQC,UAAU,MAAMF,WAAW,CAACC,IAAI,GAAGC;IACvE;IAEA,MAAMI,gBAAgBvC,eAAeqB,KAAK,cAAc,AAACD,GAAGK,OAAO,CAAC,uDAAuDK,GAAG,CAACD,MAAwBW,CAAC,GAAG;IAC3J,MAAMC,WAAWzC,eAAeqB,KAAK,cAAeD,GAAGK,OAAO,CAAC,2GAA2GC,GAAG,CAACG,MAAMX,mBAA6B,EAAE;IAEnN,IAAIwB,WAAqB,EAAE;IAC3B,IAAIC,YAAsB,EAAE;IAC5B,IAAIC,aAAuB,EAAE;IAC7B,IAAIC,iBAAiB;IACrB,MAAMC,WAAW3C,YAAYiB,IAAIC,KAAKE;IACtC,IAAIvB,eAAeqB,KAAK,UAAU;QAChC,MAAM0B,MAAM3B,GAAGK,OAAO,CAAC,+FAA+FC,GAAG,CAACG;QAC1Ha,WAAW;eAAI,IAAIM,IAAID,IAAIhC,MAAM,CAAC,CAACkC,IAAMA,EAAEC,GAAG,KAAK,MAAMvB,GAAG,CAAC,CAACsB,IAAMA,EAAEC,GAAG;SAAa;QACtFN,aAAaG,IAAIhC,MAAM,CAAC,CAACkC,IAAMA,EAAEC,GAAG,KAAK,MAAMvB,GAAG,CAAC,CAACsB,IAAMA,EAAEE,MAAM;QAClEN,iBAAiB,AAACzB,GAAGK,OAAO,CAAC,2EAA2EK,GAAG,CAACD,MAAwBW,CAAC;QACrIG,YAAY,AAACvB,GAAGK,OAAO,CAAC,oFAAoFC,GAAG,CAACG,MAAMX,iBAA4CS,GAAG,CAAC,CAACC,IAAMA,EAAEwB,GAAG;IACpL;IAEA,6FAA6F;IAC7F,sGAAsG;IACtG,MAAMC,QAAQjC,kBAAAA,GAAGK,OAAO,CAAC,6CAA6CK,GAAG,CAACD,mBAA5DT,sCAAD,AAACA,gBAAoGkC,IAAI;IACtH,MAAMC,aAAaC,KAAKC,IAAI,CAAC,EAAEjC,aAAAA,IAAIkC,KAAK,cAATlC,wBAAAA,aAAwB,KAAK;IAC5D,MAAMmC,SAASN,SAASO,YAAYJ,KAAKK,GAAG,CAACL,KAAKC,IAAI,CAAC1D,eAAesD,QAAQE,cAAcA;IAE5F,OAAO;QACL1B;QACA8B;QACA1B;QACAF;QACAU;QACAC,UAAUA,SAASoB,KAAK,CAAC,GAAG5C;QAC5ByB;QACAC,YAAYA,WAAWkB,KAAK,CAAC,GAAG5C;QAChCqB;QACAwB,eAAerB,SAAS1B,MAAM;QAC9B6B;QACAmB,iBAAiBpB,WAAW5B,MAAM;QAClCiD,KAAK,AAAC;YAAC;YAAY;SAAQ,CAAmBlD,MAAM,CAAC,CAACmD,OAAS,CAAClE,eAAeqB,KAAK6C;IACtF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/peek.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport { estimateTokens } from '../chunk/index.ts';\nimport type { FeatureName, ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { featureEnabled } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport type { Store } from '../store/types.ts';\nimport { INTERNAL_COLUMNS, scopedPaths } from './scope.ts';\n\n// Note resolution shared by peek and path: an exact path, or a unique basename (case\n// insensitive, .md stripped).\nexport function resolveNote(paths: string[], arg: string): string {\n  const exact = paths.find((p) => p === arg);\n  if (exact) return exact;\n  const base = posix.basename(arg).replace(/\\.md$/i, '').toLowerCase();\n  const matches = paths.filter((p) => posix.basename(p).replace(/\\.md$/i, '').toLowerCase() === base);\n  if (matches.length === 1) return matches[0];\n  if (matches.length > 1) throw new SenseError('NOTE_AMBIGUOUS', `\"${arg}\" is ambiguous: ${matches.join(', ')}`);\n  throw new SenseError('NOTE_NOT_FOUND', `no note matches \"${arg}\"`);\n}\n\nexport interface Peek {\n  path: string;\n  tokens: number;\n  frontmatter: Row;\n  // Set when the note's frontmatter was refused, so an empty frontmatter block reads as \"did\n  // not parse\" rather than \"has none\". Same reason `_parse_error` sits in the row.\n  parseError: string | null;\n  sections: Row[];\n  outbound: string[];\n  backlinks: string[];\n  unresolved: string[];\n  // Totals before truncation: a hub can have thousands of backlinks (or a note thousands of\n  // headings), and peek's whole point is bounded output. Query the sections/links tables\n  // directly for the full list.\n  sectionsTotal: number;\n  outboundTotal: number;\n  backlinksTotal: number;\n  unresolvedTotal: number;\n  // Bounded k-hop expansion beyond the immediate ring already shown by outbound/backlinks\n  // (depth starts at 2).\n  off: FeatureName[]; // disabled features whose blocks are omitted (not empty)\n}\n\nconst PEEK_LIST_LIMIT = 20;\n\n// peek: everything about one note except its prose -- frontmatter, outline with line\n// ranges + token estimates (so the follow-up Read is a range, not the file), links both ways.\nexport async function peek(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides = {}): Promise<Peek> {\n  return store.transaction(async () => {\n    const pathsStmt = await store.prepare('SELECT \"path\" FROM frontmatter');\n    const paths = ((await pathsStmt.all()) as Array<{ path: string }>).map((r) => r.path);\n    const path = resolveNote(paths, pathArg);\n\n    const rowStmt = await store.prepare('SELECT * FROM frontmatter WHERE \"path\" = ?');\n    const row = (await rowStmt.get(path)) as Row;\n    const parseError = (row._parse_error as string | null) ?? null;\n    const frontmatter: Row = {};\n    for (const [key, value] of Object.entries(row)) {\n      if (!INTERNAL_COLUMNS.has(key) && value !== null) frontmatter[key] = value;\n    }\n\n    let sectionsTotal = 0;\n    let sections: Row[] = [];\n    if (featureEnabled(cfg, 'sections')) {\n      sectionsTotal = ((await (await store.prepare('SELECT COUNT(*) AS n FROM sections WHERE \"path\" = ?')).get(path)) as { n: number }).n;\n      sections = (await (await store.prepare('SELECT level, heading, start_line, end_line, tokens FROM sections WHERE \"path\" = ? ORDER BY idx LIMIT ?')).all(path, PEEK_LIST_LIMIT)) as Row[];\n    }\n\n    let outbound: string[] = [];\n    let backlinks: string[] = [];\n    let unresolved: string[] = [];\n    let backlinksTotal = 0;\n    await scopedPaths(store, cfg, overrides); // validates --preset/--where/--include/--exclude even though peek's own output isn't scope-filtered\n    if (featureEnabled(cfg, 'links')) {\n      const out = (await (await store.prepare('SELECT target, dst FROM links WHERE src = ? AND (dst IS NULL OR dst != src) ORDER BY target')).all(path)) as Array<{ target: string; dst: string | null }>;\n      outbound = [...new Set(out.filter((l) => l.dst !== null).map((l) => l.dst as string))];\n      unresolved = out.filter((l) => l.dst === null).map((l) => l.target);\n      backlinksTotal = ((await (await store.prepare('SELECT COUNT(DISTINCT src) AS n FROM links WHERE dst = ? AND src != dst')).get(path)) as { n: number }).n;\n      backlinks = ((await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst ORDER BY src LIMIT ?')).all(path, PEEK_LIST_LIMIT)) as Array<{ src: string }>).map((r) => r.src);\n    }\n\n    // content.text is the stripped body already computed at reconcile time (no extra file read),\n    // floored by the byte/4 estimate so stripped-away syntax and frontmatter can't vanish from the price.\n    const body = ((await (await store.prepare('SELECT text FROM content WHERE \"path\" = ?')).get(path)) as { text: string } | undefined)?.text;\n    const byteTokens = Math.ceil(((row._size as number) ?? 0) / 4);\n    const tokens = body !== undefined ? Math.max(Math.ceil(estimateTokens(body)), byteTokens) : byteTokens;\n\n    return {\n      path,\n      tokens,\n      frontmatter,\n      parseError,\n      sections,\n      outbound: outbound.slice(0, PEEK_LIST_LIMIT),\n      backlinks,\n      unresolved: unresolved.slice(0, PEEK_LIST_LIMIT),\n      sectionsTotal,\n      outboundTotal: outbound.length,\n      backlinksTotal,\n      unresolvedTotal: unresolved.length,\n      off: (['sections', 'links'] as FeatureName[]).filter((name) => !featureEnabled(cfg, name)),\n    };\n  });\n}\n"],"names":["posix","estimateTokens","featureEnabled","SenseError","INTERNAL_COLUMNS","scopedPaths","resolveNote","paths","arg","exact","find","p","base","basename","replace","toLowerCase","matches","filter","length","join","PEEK_LIST_LIMIT","peek","store","cfg","pathArg","overrides","transaction","row","pathsStmt","prepare","all","map","r","path","rowStmt","get","parseError","_parse_error","frontmatter","key","value","Object","entries","has","sectionsTotal","sections","n","outbound","backlinks","unresolved","backlinksTotal","out","Set","l","dst","target","src","body","text","byteTokens","Math","ceil","_size","tokens","undefined","max","slice","outboundTotal","unresolvedTotal","off","name"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AACpC,SAASC,cAAc,QAAQ,oBAAoB;AAEnD,SAASC,cAAc,QAAQ,qBAAqB;AACpD,SAASC,UAAU,QAAQ,eAAe;AAG1C,SAASC,gBAAgB,EAAEC,WAAW,QAAQ,aAAa;AAE3D,qFAAqF;AACrF,8BAA8B;AAC9B,OAAO,SAASC,YAAYC,KAAe,EAAEC,GAAW;IACtD,MAAMC,QAAQF,MAAMG,IAAI,CAAC,CAACC,IAAMA,MAAMH;IACtC,IAAIC,OAAO,OAAOA;IAClB,MAAMG,OAAOZ,MAAMa,QAAQ,CAACL,KAAKM,OAAO,CAAC,UAAU,IAAIC,WAAW;IAClE,MAAMC,UAAUT,MAAMU,MAAM,CAAC,CAACN,IAAMX,MAAMa,QAAQ,CAACF,GAAGG,OAAO,CAAC,UAAU,IAAIC,WAAW,OAAOH;IAC9F,IAAII,QAAQE,MAAM,KAAK,GAAG,OAAOF,OAAO,CAAC,EAAE;IAC3C,IAAIA,QAAQE,MAAM,GAAG,GAAG,MAAM,IAAIf,WAAW,kBAAkB,CAAC,CAAC,EAAEK,IAAI,gBAAgB,EAAEQ,QAAQG,IAAI,CAAC,OAAO;IAC7G,MAAM,IAAIhB,WAAW,kBAAkB,CAAC,iBAAiB,EAAEK,IAAI,CAAC,CAAC;AACnE;AAyBA,MAAMY,kBAAkB;AAExB,qFAAqF;AACrF,8FAA8F;AAC9F,OAAO,eAAeC,KAAKC,KAAY,EAAEC,GAAmB,EAAEC,OAAe,EAAEC,YAA6B,CAAC,CAAC;IAC5G,OAAOH,MAAMI,WAAW,CAAC;YAOHC,mBA6BWA;YADhB;QAlCf,MAAMC,YAAY,MAAMN,MAAMO,OAAO,CAAC;QACtC,MAAMtB,QAAQ,AAAE,CAAA,MAAMqB,UAAUE,GAAG,EAAC,EAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;QACpF,MAAMA,OAAO3B,YAAYC,OAAOiB;QAEhC,MAAMU,UAAU,MAAMZ,MAAMO,OAAO,CAAC;QACpC,MAAMF,MAAO,MAAMO,QAAQC,GAAG,CAACF;QAC/B,MAAMG,cAAcT,oBAAAA,IAAIU,YAAY,cAAhBV,+BAAAA,oBAAsC;QAC1D,MAAMW,cAAmB,CAAC;QAC1B,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAOC,OAAO,CAACf,KAAM;YAC9C,IAAI,CAACvB,iBAAiBuC,GAAG,CAACJ,QAAQC,UAAU,MAAMF,WAAW,CAACC,IAAI,GAAGC;QACvE;QAEA,IAAII,gBAAgB;QACpB,IAAIC,WAAkB,EAAE;QACxB,IAAI3C,eAAeqB,KAAK,aAAa;YACnCqB,gBAAgB,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMtB,MAAMO,OAAO,CAAC,sDAAqD,EAAGM,GAAG,CAACF,KAAI,EAAqBa,CAAC;YACnID,WAAY,MAAM,AAAC,CAAA,MAAMvB,MAAMO,OAAO,CAAC,0GAAyG,EAAGC,GAAG,CAACG,MAAMb;QAC/J;QAEA,IAAI2B,WAAqB,EAAE;QAC3B,IAAIC,YAAsB,EAAE;QAC5B,IAAIC,aAAuB,EAAE;QAC7B,IAAIC,iBAAiB;QACrB,MAAM7C,YAAYiB,OAAOC,KAAKE,YAAY,oGAAoG;QAC9I,IAAIvB,eAAeqB,KAAK,UAAU;YAChC,MAAM4B,MAAO,MAAM,AAAC,CAAA,MAAM7B,MAAMO,OAAO,CAAC,8FAA6F,EAAGC,GAAG,CAACG;YAC5Ic,WAAW;mBAAI,IAAIK,IAAID,IAAIlC,MAAM,CAAC,CAACoC,IAAMA,EAAEC,GAAG,KAAK,MAAMvB,GAAG,CAAC,CAACsB,IAAMA,EAAEC,GAAG;aAAa;YACtFL,aAAaE,IAAIlC,MAAM,CAAC,CAACoC,IAAMA,EAAEC,GAAG,KAAK,MAAMvB,GAAG,CAAC,CAACsB,IAAMA,EAAEE,MAAM;YAClEL,iBAAiB,AAAE,CAAA,MAAM,AAAC,CAAA,MAAM5B,MAAMO,OAAO,CAAC,0EAAyE,EAAGM,GAAG,CAACF,KAAI,EAAqBa,CAAC;YACxJE,YAAY,AAAE,CAAA,MAAM,AAAC,CAAA,MAAM1B,MAAMO,OAAO,CAAC,mFAAkF,EAAGC,GAAG,CAACG,MAAMb,gBAAe,EAA8BW,GAAG,CAAC,CAACC,IAAMA,EAAEwB,GAAG;QACvM;QAEA,6FAA6F;QAC7F,sGAAsG;QACtG,MAAMC,QAAS,QAAA,MAAM,AAAC,CAAA,MAAMnC,MAAMO,OAAO,CAAC,4CAA2C,EAAGM,GAAG,CAACF,mBAA7E,4BAAF,AAAE,MAAsHyB,IAAI;QACzI,MAAMC,aAAaC,KAAKC,IAAI,CAAC,EAAElC,aAAAA,IAAImC,KAAK,cAATnC,wBAAAA,aAAwB,KAAK;QAC5D,MAAMoC,SAASN,SAASO,YAAYJ,KAAKK,GAAG,CAACL,KAAKC,IAAI,CAAC5D,eAAewD,QAAQE,cAAcA;QAE5F,OAAO;YACL1B;YACA8B;YACAzB;YACAF;YACAS;YACAE,UAAUA,SAASmB,KAAK,CAAC,GAAG9C;YAC5B4B;YACAC,YAAYA,WAAWiB,KAAK,CAAC,GAAG9C;YAChCwB;YACAuB,eAAepB,SAAS7B,MAAM;YAC9BgC;YACAkB,iBAAiBnB,WAAW/B,MAAM;YAClCmD,KAAK,AAAC;gBAAC;gBAAY;aAAQ,CAAmBpD,MAAM,CAAC,CAACqD,OAAS,CAACpE,eAAeqB,KAAK+C;QACtF;IACF;AACF"}

package/dist/esm/commands/related.d.ts

@@ -1,6 +1,6 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig, SearchOverrides } from '../config/index.js';
-export declare function relatedNotes(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{
+import type { Store } from '../store/types.js';
+export declare function relatedNotes(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{
     path: string;
     similarity: number;
 }>>;

package/dist/esm/commands/related.js

@@ -6,11 +6,11 @@ import { resolveNote } from './peek.js';
 import { scopedPaths, scopeHasEmbeddings } from './scope.js';
 // Notes most similar by cosine, excluding self and everything already linked either way.
 // Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.
-export async function relatedNotes(db, cfg, pathArg, overrides, k) {
-    const paths = db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path);
+export async function relatedNotes(store, cfg, pathArg, overrides, k) {
+    const paths = (await (await store.prepare('SELECT "path" FROM frontmatter')).all()).map((r)=>r.path);
     const path = resolveNote(paths, pathArg);
-    const outbound = db.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src').all(path).map((r)=>r.dst);
-    const backlinks = db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst').all(path).map((r)=>r.src);
+    const outbound = (await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)).map((r)=>r.dst);
+    const backlinks = (await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)).map((r)=>r.src);
     const exclude = new Set([
         path,
         ...outbound,
@@ -35,15 +35,15 @@ export async function relatedNotes(db, cfg, pathArg, overrides, k) {
     if (localModelMissing(e)) {
         throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path "${e.model}" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);
     }
-    const allowed = scopedPaths(db, cfg, overrides);
+    const allowed = await scopedPaths(store, cfg, overrides);
     // Top up pending rows before the seed check, or a fresh index reports every note as
     // having no indexed text until some search has run.
-    await embedPending(db, cfg, cfg.baseDir);
-    if (!hasEmbedding(db, path)) {
+    await embedPending(store, cfg, cfg.baseDir);
+    if (!await hasEmbedding(store, path)) {
         throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);
     }
-    if (!scopeHasEmbeddings(db, cfg, allowed)) return [];
-    return similarNotes(db, cfg, path, {
+    if (!await scopeHasEmbeddings(store, cfg, allowed)) return [];
+    return similarNotes(store, cfg, path, {
         exclude,
         allowed,
         k

package/dist/esm/commands/related.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedConfig, embedEnabled, resolveSearch } from '../config/index.ts';\nimport { embedPending, hasEmbedding, similarNotes } from '../embed/query.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings } from './scope.ts';\n\n// Notes most similar by cosine, excluding self and everything already linked either way.\n// Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.\nexport async function relatedNotes(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  const paths = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  const path = resolveNote(paths, pathArg);\n\n  const outbound = (db.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src').all(path) as Array<{ dst: string }>).map((r) => r.dst);\n  const backlinks = (db.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst').all(path) as Array<{ src: string }>).map((r) => r.src);\n  const exclude = new Set([path, ...outbound, ...backlinks]);\n\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause. An empty table then means one thing: nothing near in meaning that this\n  // note does not already link to, which is a real answer.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');\n  }\n  // search gates on the same signal (see wantsVectors above); reading it here too keeps a\n  // preset's declared signals meaning one thing. Without this, an overlapping vectors-on\n  // preset's vectors would answer for a scope that declined them.\n  if (effective.signals.vectors === undefined) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n  }\n  // A downloadable HF id proceeds -- embedPending's getProvider call fetches it lazily on\n  // consent. Only a local path with missing files can never fetch itself.\n  const e = embedConfig(cfg); // embedEnabled(cfg) above guarantees this is set\n  if (localModelMissing(e)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);\n  }\n  const allowed = scopedPaths(db, cfg, overrides);\n  // Top up pending rows before the seed check, or a fresh index reports every note as\n  // having no indexed text until some search has run.\n  await embedPending(db, cfg, cfg.baseDir);\n  if (!hasEmbedding(db, path)) {\n    throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n  }\n  if (!scopeHasEmbeddings(db, cfg, allowed)) return [];\n  return similarNotes(db, cfg, path, { exclude, allowed, k });\n}\n"],"names":["embedConfig","embedEnabled","resolveSearch","embedPending","hasEmbedding","similarNotes","localModelMissing","MODEL_FILENAMES","SenseError","resolveNote","scopedPaths","scopeHasEmbeddings","relatedNotes","db","cfg","pathArg","overrides","k","paths","prepare","all","map","r","path","outbound","dst","backlinks","src","exclude","Set","effective","signals","vectors","undefined","presetName","e","model","allowed","baseDir"],"mappings":"AAEA,SAASA,WAAW,EAAEC,YAAY,EAAEC,aAAa,QAAQ,qBAAqB;AAC9E,SAASC,YAAY,EAAEC,YAAY,EAAEC,YAAY,QAAQ,oBAAoB;AAC7E,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,WAAW,QAAQ,YAAY;AACxC,SAASC,WAAW,EAAEC,kBAAkB,QAAQ,aAAa;AAE7D,yFAAyF;AACzF,sFAAsF;AACtF,OAAO,eAAeC,aAAaC,EAAgB,EAAEC,GAAmB,EAAEC,OAAe,EAAEC,SAA0B,EAAEC,CAAS;IAC9H,MAAMC,QAAQ,AAACL,GAAGM,OAAO,CAAC,kCAAkCC,GAAG,GAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAC/G,MAAMA,OAAOd,YAAYS,OAAOH;IAEhC,MAAMS,WAAW,AAACX,GAAGM,OAAO,CAAC,mFAAmFC,GAAG,CAACG,MAAiCF,GAAG,CAAC,CAACC,IAAMA,EAAEG,GAAG;IACrK,MAAMC,YAAY,AAACb,GAAGM,OAAO,CAAC,+DAA+DC,GAAG,CAACG,MAAiCF,GAAG,CAAC,CAACC,IAAMA,EAAEK,GAAG;IAClJ,MAAMC,UAAU,IAAIC,IAAI;QAACN;WAASC;WAAaE;KAAU;IAEzD,yFAAyF;IACzF,2FAA2F;IAC3F,yDAAyD;IACzD,MAAMI,YAAY5B,cAAcY,KAAKE;IACrC,IAAI,CAACf,aAAaa,MAAM;QACtB,MAAM,IAAIN,WAAW,kBAAkB;IACzC;IACA,wFAAwF;IACxF,uFAAuF;IACvF,gEAAgE;IAChE,IAAIsB,UAAUC,OAAO,CAACC,OAAO,KAAKC,WAAW;QAC3C,MAAM,IAAIzB,WAAW,uBAAuB,CAAC,QAAQ,EAAEsB,UAAUI,UAAU,CAAC,uKAAuK,CAAC;IACtP;IACA,wFAAwF;IACxF,wEAAwE;IACxE,MAAMC,IAAInC,YAAYc,MAAM,iDAAiD;IAC7E,IAAIR,kBAAkB6B,IAAI;QACxB,MAAM,IAAI3B,WAAW,uBAAuB,CAAC,2FAA2F,EAAE2B,EAAEC,KAAK,CAAC,aAAa,EAAE7B,gBAAgB,sGAAsG,CAAC;IAC1R;IACA,MAAM8B,UAAU3B,YAAYG,IAAIC,KAAKE;IACrC,oFAAoF;IACpF,oDAAoD;IACpD,MAAMb,aAAaU,IAAIC,KAAKA,IAAIwB,OAAO;IACvC,IAAI,CAAClC,aAAaS,IAAIU,OAAO;QAC3B,MAAM,IAAIf,WAAW,qBAAqB,GAAGe,KAAK,4GAA4G,CAAC;IACjK;IACA,IAAI,CAACZ,mBAAmBE,IAAIC,KAAKuB,UAAU,OAAO,EAAE;IACpD,OAAOhC,aAAaQ,IAAIC,KAAKS,MAAM;QAAEK;QAASS;QAASpB;IAAE;AAC3D"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedConfig, embedEnabled, resolveSearch } from '../config/index.ts';\nimport { embedPending, hasEmbedding, similarNotes } from '../embed/query.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Store } from '../store/types.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings } from './scope.ts';\n\n// Notes most similar by cosine, excluding self and everything already linked either way.\n// Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.\nexport async function relatedNotes(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  const paths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n  const path = resolveNote(paths, pathArg);\n\n  const outbound = ((await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)) as Array<{ dst: string }>).map((r) => r.dst);\n  const backlinks = ((await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)) as Array<{ src: string }>).map((r) => r.src);\n  const exclude = new Set([path, ...outbound, ...backlinks]);\n\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause. An empty table then means one thing: nothing near in meaning that this\n  // note does not already link to, which is a real answer.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');\n  }\n  // search gates on the same signal (see wantsVectors above); reading it here too keeps a\n  // preset's declared signals meaning one thing. Without this, an overlapping vectors-on\n  // preset's vectors would answer for a scope that declined them.\n  if (effective.signals.vectors === undefined) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n  }\n  // A downloadable HF id proceeds -- embedPending's getProvider call fetches it lazily on\n  // consent. Only a local path with missing files can never fetch itself.\n  const e = embedConfig(cfg); // embedEnabled(cfg) above guarantees this is set\n  if (localModelMissing(e)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);\n  }\n  const allowed = await scopedPaths(store, cfg, overrides);\n  // Top up pending rows before the seed check, or a fresh index reports every note as\n  // having no indexed text until some search has run.\n  await embedPending(store, cfg, cfg.baseDir);\n  if (!(await hasEmbedding(store, path))) {\n    throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n  }\n  if (!(await scopeHasEmbeddings(store, cfg, allowed))) return [];\n  return similarNotes(store, cfg, path, { exclude, allowed, k });\n}\n"],"names":["embedConfig","embedEnabled","resolveSearch","embedPending","hasEmbedding","similarNotes","localModelMissing","MODEL_FILENAMES","SenseError","resolveNote","scopedPaths","scopeHasEmbeddings","relatedNotes","store","cfg","pathArg","overrides","k","paths","prepare","all","map","r","path","outbound","dst","backlinks","src","exclude","Set","effective","signals","vectors","undefined","presetName","e","model","allowed","baseDir"],"mappings":"AACA,SAASA,WAAW,EAAEC,YAAY,EAAEC,aAAa,QAAQ,qBAAqB;AAC9E,SAASC,YAAY,EAAEC,YAAY,EAAEC,YAAY,QAAQ,oBAAoB;AAC7E,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,YAAY;AACxC,SAASC,WAAW,EAAEC,kBAAkB,QAAQ,aAAa;AAE7D,yFAAyF;AACzF,sFAAsF;AACtF,OAAO,eAAeC,aAAaC,KAAY,EAAEC,GAAmB,EAAEC,OAAe,EAAEC,SAA0B,EAAEC,CAAS;IAC1H,MAAMC,QAAQ,AAAE,CAAA,MAAM,AAAC,CAAA,MAAML,MAAMM,OAAO,CAAC,iCAAgC,EAAGC,GAAG,EAAC,EAA+BC,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IAClI,MAAMA,OAAOd,YAAYS,OAAOH;IAEhC,MAAMS,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMX,MAAMM,OAAO,CAAC,kFAAiF,EAAGC,GAAG,CAACG,KAAI,EAA8BF,GAAG,CAAC,CAACC,IAAMA,EAAEG,GAAG;IACxL,MAAMC,YAAY,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMb,MAAMM,OAAO,CAAC,8DAA6D,EAAGC,GAAG,CAACG,KAAI,EAA8BF,GAAG,CAAC,CAACC,IAAMA,EAAEK,GAAG;IACrK,MAAMC,UAAU,IAAIC,IAAI;QAACN;WAASC;WAAaE;KAAU;IAEzD,yFAAyF;IACzF,2FAA2F;IAC3F,yDAAyD;IACzD,MAAMI,YAAY5B,cAAcY,KAAKE;IACrC,IAAI,CAACf,aAAaa,MAAM;QACtB,MAAM,IAAIN,WAAW,kBAAkB;IACzC;IACA,wFAAwF;IACxF,uFAAuF;IACvF,gEAAgE;IAChE,IAAIsB,UAAUC,OAAO,CAACC,OAAO,KAAKC,WAAW;QAC3C,MAAM,IAAIzB,WAAW,uBAAuB,CAAC,QAAQ,EAAEsB,UAAUI,UAAU,CAAC,uKAAuK,CAAC;IACtP;IACA,wFAAwF;IACxF,wEAAwE;IACxE,MAAMC,IAAInC,YAAYc,MAAM,iDAAiD;IAC7E,IAAIR,kBAAkB6B,IAAI;QACxB,MAAM,IAAI3B,WAAW,uBAAuB,CAAC,2FAA2F,EAAE2B,EAAEC,KAAK,CAAC,aAAa,EAAE7B,gBAAgB,sGAAsG,CAAC;IAC1R;IACA,MAAM8B,UAAU,MAAM3B,YAAYG,OAAOC,KAAKE;IAC9C,oFAAoF;IACpF,oDAAoD;IACpD,MAAMb,aAAaU,OAAOC,KAAKA,IAAIwB,OAAO;IAC1C,IAAI,CAAE,MAAMlC,aAAaS,OAAOU,OAAQ;QACtC,MAAM,IAAIf,WAAW,qBAAqB,GAAGe,KAAK,4GAA4G,CAAC;IACjK;IACA,IAAI,CAAE,MAAMZ,mBAAmBE,OAAOC,KAAKuB,UAAW,OAAO,EAAE;IAC/D,OAAOhC,aAAaQ,OAAOC,KAAKS,MAAM;QAAEK;QAASS;QAASpB;IAAE;AAC9D"}

package/dist/esm/commands/scope.d.ts

@@ -1,10 +1,10 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig, SearchOverrides } from '../config/index.js';
+import type { Store } from '../store/types.js';
 export declare const INTERNAL_COLUMNS: Set<string>;
 export declare function inScope(path: string, include: string[], exclude?: string[]): boolean;
-export declare function scopeHasEmbeddings(db: DatabaseSync, cfg: ResolvedConfig, scopedPaths: Set<string>): boolean;
-export declare function rawScope(db: DatabaseSync, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Set<string>;
-export declare function narrowByWhere(db: DatabaseSync, paths: Set<string>, where: string | undefined): Set<string>;
-export declare function scopedPaths(db: DatabaseSync, cfg: ResolvedConfig, overrides: SearchOverrides): Set<string>;
-export declare function materializeScope(db: DatabaseSync, table: string, paths: Set<string>): void;
-export declare function setupMapScope(db: DatabaseSync, paths: Set<string>): void;
+export declare function scopeHasEmbeddings(store: Store, cfg: ResolvedConfig, scopedPaths: Set<string>): Promise<boolean>;
+export declare function rawScope(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Promise<Set<string>>;
+export declare function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>>;
+export declare function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>>;
+export declare function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void>;
+export declare function setupMapScope(store: Store, paths: Set<string>): Promise<void>;

package/dist/esm/commands/scope.js

@@ -17,29 +17,38 @@ export function inScope(path, include, exclude) {
     if (exclude === null || exclude === void 0 ? void 0 : exclude.some((g)=>matchesGlob(path, g))) return false;
     return true;
 }
-export function scopeHasEmbeddings(db, cfg, scopedPaths) {
+export async function scopeHasEmbeddings(store, cfg, scopedPaths) {
     if (!anyPresetEmbeds(cfg)) return false; // the embeddings table doesn't exist at all in this case
-    const rows = db.prepare('SELECT DISTINCT "path" FROM embeddings').all();
+    const stmt = await store.prepare('SELECT DISTINCT "path" FROM embeddings');
+    const rows = await stmt.all();
     return rows.some((r)=>scopedPaths.has(r.path));
 }
 // Scope only (no --where): preset_files for a named preset, JS glob matching for an ad hoc
 // include/exclude override. Shared by scopedPaths() and search(), which also needs the
 // pre-where set to size the candidate-pool filter.
-export function rawScope(db, cfg, overrides, allPaths) {
+export async function rawScope(store, cfg, overrides, allPaths) {
     const effective = resolveSearch(cfg, overrides);
     const { include, exclude } = effective;
     const adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;
-    if (!adHocScope) return new Set(db.prepare('SELECT "path" FROM preset_files WHERE preset = ?').all(effective.presetName).map((r)=>r.path));
-    const paths = allPaths !== null && allPaths !== void 0 ? allPaths : db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path);
+    if (!adHocScope) {
+        const stmt = await store.prepare('SELECT "path" FROM preset_files WHERE preset = ?');
+        return new Set((await stmt.all(effective.presetName)).map((r)=>r.path));
+    }
+    const paths = allPaths !== null && allPaths !== void 0 ? allPaths : await allFrontmatterPaths(store);
     return new Set(paths.filter((p)=>inScope(p, include, exclude)));
 }
-export function narrowByWhere(db, paths, where) {
+async function allFrontmatterPaths(store) {
+    const stmt = await store.prepare('SELECT "path" FROM frontmatter');
+    return (await stmt.all()).map((r)=>r.path);
+}
+export async function narrowByWhere(store, paths, where) {
     if (!where) return paths;
     let whereRows;
     try {
-        whereRows = db.prepare(`SELECT "path" FROM frontmatter f WHERE (${where})`).all();
+        const stmt = await store.prepare(`SELECT "path" FROM frontmatter f WHERE (${where})`);
+        whereRows = await stmt.all();
     } catch (err) {
-        throw columnHint(db, err);
+        throw columnHint(await store.docs.columns(), err);
     }
     const wherePaths = new Set(whereRows.map((r)=>r.path));
     return new Set([
@@ -48,19 +57,21 @@ export function narrowByWhere(db, paths, where) {
 }
 // The scope resolver for non-search commands (path, peek, map): same coverage rule search()
 // applies, then narrowed by the resolved `where`.
-export function scopedPaths(db, cfg, overrides) {
+export async function scopedPaths(store, cfg, overrides) {
     const effective = resolveSearch(cfg, overrides);
-    return narrowByWhere(db, rawScope(db, cfg, overrides), effective.where);
+    return narrowByWhere(store, await rawScope(store, cfg, overrides), effective.where);
 }
 // Materializes a path set into a named temp table (same shape as traverse.ts's allowed_nodes)
 // so a query can join/filter against it cheaply instead of binding a parameter per path.
-export function materializeScope(db, table, paths) {
-    db.exec(`DROP TABLE IF EXISTS ${table}`);
-    db.exec(`CREATE TEMP TABLE ${table} ("path" TEXT PRIMARY KEY)`);
-    db.prepare(`INSERT INTO ${table} SELECT DISTINCT value FROM json_each(?1)`).run(JSON.stringify([
+export async function materializeScope(store, table, paths) {
+    await store.exec(`DROP TABLE IF EXISTS ${table}`);
+    await store.exec(`CREATE TEMP TABLE ${table} ("path" TEXT PRIMARY KEY)`);
+    await store.runBatch(`INSERT INTO ${table} ("path") VALUES (?)`, [
         ...paths
-    ]));
+    ].map((p)=>[
+            p
+        ]));
 }
-export function setupMapScope(db, paths) {
-    materializeScope(db, '_map_scope', paths);
+export async function setupMapScope(store, paths) {
+    await materializeScope(store, '_map_scope', paths);
 }

package/dist/esm/commands/scope.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/scope.ts"],"sourcesContent":["import { matchesGlob } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { anyPresetEmbeds, resolveSearch } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\n\nexport const INTERNAL_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error']);\n\n// node:path's matchesGlob is experimental (stable behind an unstable-API flag) as of the\n// engines floor (Node >=22.20); scope filtering only ever needs single-pattern matching, so\n// it's used here in JS rather than running a directory walk in the query path.\nexport function inScope(path: string, include: string[], exclude?: string[]): boolean {\n  if (!include.some((g) => matchesGlob(path, g))) return false;\n  if (exclude?.some((g) => matchesGlob(path, g))) return false;\n  return true;\n}\n\nexport function scopeHasEmbeddings(db: DatabaseSync, cfg: ResolvedConfig, scopedPaths: Set<string>): boolean {\n  if (!anyPresetEmbeds(cfg)) return false; // the embeddings table doesn't exist at all in this case\n  const rows = db.prepare('SELECT DISTINCT \"path\" FROM embeddings').all() as Array<{ path: string }>;\n  return rows.some((r) => scopedPaths.has(r.path));\n}\n\n// Scope only (no --where): preset_files for a named preset, JS glob matching for an ad hoc\n// include/exclude override. Shared by scopedPaths() and search(), which also needs the\n// pre-where set to size the candidate-pool filter.\nexport function rawScope(db: DatabaseSync, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Set<string> {\n  const effective = resolveSearch(cfg, overrides);\n  const { include, exclude } = effective;\n  const adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;\n  if (!adHocScope) return new Set((db.prepare('SELECT \"path\" FROM preset_files WHERE preset = ?').all(effective.presetName) as Array<{ path: string }>).map((r) => r.path));\n  const paths = allPaths ?? (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n  return new Set(paths.filter((p) => inScope(p, include, exclude)));\n}\n\nexport function narrowByWhere(db: DatabaseSync, paths: Set<string>, where: string | undefined): Set<string> {\n  if (!where) return paths;\n  let whereRows: Array<{ path: string }>;\n  try {\n    whereRows = db.prepare(`SELECT \"path\" FROM frontmatter f WHERE (${where})`).all() as Array<{ path: string }>;\n  } catch (err) {\n    throw columnHint(db, err as Error);\n  }\n  const wherePaths = new Set(whereRows.map((r) => r.path));\n  return new Set([...paths].filter((p) => wherePaths.has(p)));\n}\n\n// The scope resolver for non-search commands (path, peek, map): same coverage rule search()\n// applies, then narrowed by the resolved `where`.\nexport function scopedPaths(db: DatabaseSync, cfg: ResolvedConfig, overrides: SearchOverrides): Set<string> {\n  const effective = resolveSearch(cfg, overrides);\n  return narrowByWhere(db, rawScope(db, cfg, overrides), effective.where);\n}\n\n// Materializes a path set into a named temp table (same shape as traverse.ts's allowed_nodes)\n// so a query can join/filter against it cheaply instead of binding a parameter per path.\nexport function materializeScope(db: DatabaseSync, table: string, paths: Set<string>): void {\n  db.exec(`DROP TABLE IF EXISTS ${table}`);\n  db.exec(`CREATE TEMP TABLE ${table} (\"path\" TEXT PRIMARY KEY)`);\n  db.prepare(`INSERT INTO ${table} SELECT DISTINCT value FROM json_each(?1)`).run(JSON.stringify([...paths]));\n}\n\nexport function setupMapScope(db: DatabaseSync, paths: Set<string>): void {\n  materializeScope(db, '_map_scope', paths);\n}\n"],"names":["matchesGlob","anyPresetEmbeds","resolveSearch","columnHint","INTERNAL_COLUMNS","Set","inScope","path","include","exclude","some","g","scopeHasEmbeddings","db","cfg","scopedPaths","rows","prepare","all","r","has","rawScope","overrides","allPaths","effective","adHocScope","undefined","noExclude","presetName","map","paths","filter","p","narrowByWhere","where","whereRows","err","wherePaths","materializeScope","table","exec","run","JSON","stringify","setupMapScope"],"mappings":"AAAA,SAASA,WAAW,QAAQ,YAAY;AAGxC,SAASC,eAAe,EAAEC,aAAa,QAAQ,qBAAqB;AACpE,SAASC,UAAU,QAAQ,2BAA2B;AAEtD,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;CAAe,EAAE;AAExG,yFAAyF;AACzF,4FAA4F;AAC5F,+EAA+E;AAC/E,OAAO,SAASC,QAAQC,IAAY,EAAEC,OAAiB,EAAEC,OAAkB;IACzE,IAAI,CAACD,QAAQE,IAAI,CAAC,CAACC,IAAMX,YAAYO,MAAMI,KAAK,OAAO;IACvD,IAAIF,oBAAAA,8BAAAA,QAASC,IAAI,CAAC,CAACC,IAAMX,YAAYO,MAAMI,KAAK,OAAO;IACvD,OAAO;AACT;AAEA,OAAO,SAASC,mBAAmBC,EAAgB,EAAEC,GAAmB,EAAEC,WAAwB;IAChG,IAAI,CAACd,gBAAgBa,MAAM,OAAO,OAAO,yDAAyD;IAClG,MAAME,OAAOH,GAAGI,OAAO,CAAC,0CAA0CC,GAAG;IACrE,OAAOF,KAAKN,IAAI,CAAC,CAACS,IAAMJ,YAAYK,GAAG,CAACD,EAAEZ,IAAI;AAChD;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,mDAAmD;AACnD,OAAO,SAASc,SAASR,EAAgB,EAAEC,GAAmB,EAAEQ,SAA0B,EAAEC,QAAmB;IAC7G,MAAMC,YAAYtB,cAAcY,KAAKQ;IACrC,MAAM,EAAEd,OAAO,EAAEC,OAAO,EAAE,GAAGe;IAC7B,MAAMC,aAAaH,UAAUd,OAAO,KAAKkB,aAAaJ,UAAUb,OAAO,KAAKiB,aAAaJ,UAAUK,SAAS,KAAK;IACjH,IAAI,CAACF,YAAY,OAAO,IAAIpB,IAAI,AAACQ,GAAGI,OAAO,CAAC,oDAAoDC,GAAG,CAACM,UAAUI,UAAU,EAA8BC,GAAG,CAAC,CAACV,IAAMA,EAAEZ,IAAI;IACvK,MAAMuB,QAAQP,qBAAAA,sBAAAA,WAAY,AAACV,GAAGI,OAAO,CAAC,kCAAkCC,GAAG,GAA+BW,GAAG,CAAC,CAACV,IAAMA,EAAEZ,IAAI;IAC3H,OAAO,IAAIF,IAAIyB,MAAMC,MAAM,CAAC,CAACC,IAAM1B,QAAQ0B,GAAGxB,SAASC;AACzD;AAEA,OAAO,SAASwB,cAAcpB,EAAgB,EAAEiB,KAAkB,EAAEI,KAAyB;IAC3F,IAAI,CAACA,OAAO,OAAOJ;IACnB,IAAIK;IACJ,IAAI;QACFA,YAAYtB,GAAGI,OAAO,CAAC,CAAC,wCAAwC,EAAEiB,MAAM,CAAC,CAAC,EAAEhB,GAAG;IACjF,EAAE,OAAOkB,KAAK;QACZ,MAAMjC,WAAWU,IAAIuB;IACvB;IACA,MAAMC,aAAa,IAAIhC,IAAI8B,UAAUN,GAAG,CAAC,CAACV,IAAMA,EAAEZ,IAAI;IACtD,OAAO,IAAIF,IAAI;WAAIyB;KAAM,CAACC,MAAM,CAAC,CAACC,IAAMK,WAAWjB,GAAG,CAACY;AACzD;AAEA,4FAA4F;AAC5F,kDAAkD;AAClD,OAAO,SAASjB,YAAYF,EAAgB,EAAEC,GAAmB,EAAEQ,SAA0B;IAC3F,MAAME,YAAYtB,cAAcY,KAAKQ;IACrC,OAAOW,cAAcpB,IAAIQ,SAASR,IAAIC,KAAKQ,YAAYE,UAAUU,KAAK;AACxE;AAEA,8FAA8F;AAC9F,yFAAyF;AACzF,OAAO,SAASI,iBAAiBzB,EAAgB,EAAE0B,KAAa,EAAET,KAAkB;IAClFjB,GAAG2B,IAAI,CAAC,CAAC,qBAAqB,EAAED,OAAO;IACvC1B,GAAG2B,IAAI,CAAC,CAAC,kBAAkB,EAAED,MAAM,0BAA0B,CAAC;IAC9D1B,GAAGI,OAAO,CAAC,CAAC,YAAY,EAAEsB,MAAM,yCAAyC,CAAC,EAAEE,GAAG,CAACC,KAAKC,SAAS,CAAC;WAAIb;KAAM;AAC3G;AAEA,OAAO,SAASc,cAAc/B,EAAgB,EAAEiB,KAAkB;IAChEQ,iBAAiBzB,IAAI,cAAciB;AACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/scope.ts"],"sourcesContent":["import { matchesGlob } from 'node:path';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { anyPresetEmbeds, resolveSearch } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\nimport type { Store } from '../store/types.ts';\n\nexport const INTERNAL_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error']);\n\n// node:path's matchesGlob is experimental (stable behind an unstable-API flag) as of the\n// engines floor (Node >=22.20); scope filtering only ever needs single-pattern matching, so\n// it's used here in JS rather than running a directory walk in the query path.\nexport function inScope(path: string, include: string[], exclude?: string[]): boolean {\n  if (!include.some((g) => matchesGlob(path, g))) return false;\n  if (exclude?.some((g) => matchesGlob(path, g))) return false;\n  return true;\n}\n\nexport async function scopeHasEmbeddings(store: Store, cfg: ResolvedConfig, scopedPaths: Set<string>): Promise<boolean> {\n  if (!anyPresetEmbeds(cfg)) return false; // the embeddings table doesn't exist at all in this case\n  const stmt = await store.prepare('SELECT DISTINCT \"path\" FROM embeddings');\n  const rows = (await stmt.all()) as Array<{ path: string }>;\n  return rows.some((r) => scopedPaths.has(r.path));\n}\n\n// Scope only (no --where): preset_files for a named preset, JS glob matching for an ad hoc\n// include/exclude override. Shared by scopedPaths() and search(), which also needs the\n// pre-where set to size the candidate-pool filter.\nexport async function rawScope(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  const { include, exclude } = effective;\n  const adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;\n  if (!adHocScope) {\n    const stmt = await store.prepare('SELECT \"path\" FROM preset_files WHERE preset = ?');\n    return new Set(((await stmt.all(effective.presetName)) as Array<{ path: string }>).map((r) => r.path));\n  }\n  const paths = allPaths ?? (await allFrontmatterPaths(store));\n  return new Set(paths.filter((p) => inScope(p, include, exclude)));\n}\n\nasync function allFrontmatterPaths(store: Store): Promise<string[]> {\n  const stmt = await store.prepare('SELECT \"path\" FROM frontmatter');\n  return ((await stmt.all()) as Array<{ path: string }>).map((r) => r.path);\n}\n\nexport async function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>> {\n  if (!where) return paths;\n  let whereRows: Array<{ path: string }>;\n  try {\n    const stmt = await store.prepare(`SELECT \"path\" FROM frontmatter f WHERE (${where})`);\n    whereRows = (await stmt.all()) as Array<{ path: string }>;\n  } catch (err) {\n    throw columnHint(await store.docs.columns(), err as Error);\n  }\n  const wherePaths = new Set(whereRows.map((r) => r.path));\n  return new Set([...paths].filter((p) => wherePaths.has(p)));\n}\n\n// The scope resolver for non-search commands (path, peek, map): same coverage rule search()\n// applies, then narrowed by the resolved `where`.\nexport async function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  return narrowByWhere(store, await rawScope(store, cfg, overrides), effective.where);\n}\n\n// Materializes a path set into a named temp table (same shape as traverse.ts's allowed_nodes)\n// so a query can join/filter against it cheaply instead of binding a parameter per path.\nexport async function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void> {\n  await store.exec(`DROP TABLE IF EXISTS ${table}`);\n  await store.exec(`CREATE TEMP TABLE ${table} (\"path\" TEXT PRIMARY KEY)`);\n  await store.runBatch(\n    `INSERT INTO ${table} (\"path\") VALUES (?)`,\n    [...paths].map((p) => [p])\n  );\n}\n\nexport async function setupMapScope(store: Store, paths: Set<string>): Promise<void> {\n  await materializeScope(store, '_map_scope', paths);\n}\n"],"names":["matchesGlob","anyPresetEmbeds","resolveSearch","columnHint","INTERNAL_COLUMNS","Set","inScope","path","include","exclude","some","g","scopeHasEmbeddings","store","cfg","scopedPaths","stmt","prepare","rows","all","r","has","rawScope","overrides","allPaths","effective","adHocScope","undefined","noExclude","presetName","map","paths","allFrontmatterPaths","filter","p","narrowByWhere","where","whereRows","err","docs","columns","wherePaths","materializeScope","table","exec","runBatch","setupMapScope"],"mappings":"AAAA,SAASA,WAAW,QAAQ,YAAY;AAExC,SAASC,eAAe,EAAEC,aAAa,QAAQ,qBAAqB;AACpE,SAASC,UAAU,QAAQ,2BAA2B;AAGtD,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;CAAe,EAAE;AAExG,yFAAyF;AACzF,4FAA4F;AAC5F,+EAA+E;AAC/E,OAAO,SAASC,QAAQC,IAAY,EAAEC,OAAiB,EAAEC,OAAkB;IACzE,IAAI,CAACD,QAAQE,IAAI,CAAC,CAACC,IAAMX,YAAYO,MAAMI,KAAK,OAAO;IACvD,IAAIF,oBAAAA,8BAAAA,QAASC,IAAI,CAAC,CAACC,IAAMX,YAAYO,MAAMI,KAAK,OAAO;IACvD,OAAO;AACT;AAEA,OAAO,eAAeC,mBAAmBC,KAAY,EAAEC,GAAmB,EAAEC,WAAwB;IAClG,IAAI,CAACd,gBAAgBa,MAAM,OAAO,OAAO,yDAAyD;IAClG,MAAME,OAAO,MAAMH,MAAMI,OAAO,CAAC;IACjC,MAAMC,OAAQ,MAAMF,KAAKG,GAAG;IAC5B,OAAOD,KAAKR,IAAI,CAAC,CAACU,IAAML,YAAYM,GAAG,CAACD,EAAEb,IAAI;AAChD;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,mDAAmD;AACnD,OAAO,eAAee,SAAST,KAAY,EAAEC,GAAmB,EAAES,SAA0B,EAAEC,QAAmB;IAC/G,MAAMC,YAAYvB,cAAcY,KAAKS;IACrC,MAAM,EAAEf,OAAO,EAAEC,OAAO,EAAE,GAAGgB;IAC7B,MAAMC,aAAaH,UAAUf,OAAO,KAAKmB,aAAaJ,UAAUd,OAAO,KAAKkB,aAAaJ,UAAUK,SAAS,KAAK;IACjH,IAAI,CAACF,YAAY;QACf,MAAMV,OAAO,MAAMH,MAAMI,OAAO,CAAC;QACjC,OAAO,IAAIZ,IAAI,AAAE,CAAA,MAAMW,KAAKG,GAAG,CAACM,UAAUI,UAAU,CAAA,EAA+BC,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;IACtG;IACA,MAAMwB,QAAQP,qBAAAA,sBAAAA,WAAa,MAAMQ,oBAAoBnB;IACrD,OAAO,IAAIR,IAAI0B,MAAME,MAAM,CAAC,CAACC,IAAM5B,QAAQ4B,GAAG1B,SAASC;AACzD;AAEA,eAAeuB,oBAAoBnB,KAAY;IAC7C,MAAMG,OAAO,MAAMH,MAAMI,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKG,GAAG,EAAC,EAA+BW,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;AAC1E;AAEA,OAAO,eAAe4B,cAActB,KAAY,EAAEkB,KAAkB,EAAEK,KAAyB;IAC7F,IAAI,CAACA,OAAO,OAAOL;IACnB,IAAIM;IACJ,IAAI;QACF,MAAMrB,OAAO,MAAMH,MAAMI,OAAO,CAAC,CAAC,wCAAwC,EAAEmB,MAAM,CAAC,CAAC;QACpFC,YAAa,MAAMrB,KAAKG,GAAG;IAC7B,EAAE,OAAOmB,KAAK;QACZ,MAAMnC,WAAW,MAAMU,MAAM0B,IAAI,CAACC,OAAO,IAAIF;IAC/C;IACA,MAAMG,aAAa,IAAIpC,IAAIgC,UAAUP,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;IACtD,OAAO,IAAIF,IAAI;WAAI0B;KAAM,CAACE,MAAM,CAAC,CAACC,IAAMO,WAAWpB,GAAG,CAACa;AACzD;AAEA,4FAA4F;AAC5F,kDAAkD;AAClD,OAAO,eAAenB,YAAYF,KAAY,EAAEC,GAAmB,EAAES,SAA0B;IAC7F,MAAME,YAAYvB,cAAcY,KAAKS;IACrC,OAAOY,cAActB,OAAO,MAAMS,SAAST,OAAOC,KAAKS,YAAYE,UAAUW,KAAK;AACpF;AAEA,8FAA8F;AAC9F,yFAAyF;AACzF,OAAO,eAAeM,iBAAiB7B,KAAY,EAAE8B,KAAa,EAAEZ,KAAkB;IACpF,MAAMlB,MAAM+B,IAAI,CAAC,CAAC,qBAAqB,EAAED,OAAO;IAChD,MAAM9B,MAAM+B,IAAI,CAAC,CAAC,kBAAkB,EAAED,MAAM,0BAA0B,CAAC;IACvE,MAAM9B,MAAMgC,QAAQ,CAClB,CAAC,YAAY,EAAEF,MAAM,oBAAoB,CAAC,EAC1C;WAAIZ;KAAM,CAACD,GAAG,CAAC,CAACI,IAAM;YAACA;SAAE;AAE7B;AAEA,OAAO,eAAeY,cAAcjC,KAAY,EAAEkB,KAAkB;IAClE,MAAMW,iBAAiB7B,OAAO,cAAckB;AAC9C"}

package/dist/esm/commands/search.d.ts

@@ -1,6 +1,6 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig } from '../config/index.js';
 import type { Row } from '../output/output.js';
+import type { Store } from '../store/types.js';
 export interface SearchOptions {
     k?: number;
     where?: string;
@@ -9,4 +9,4 @@ export interface SearchOptions {
     exclude?: string[];
     noExclude?: boolean;
 }
-export declare function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<Row[]>;
+export declare function search(store: Store, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<Row[]>;

package/dist/esm/commands/search.js

@@ -1,10 +1,9 @@
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { contentTokenize, embedConfig, featureEnabled, resolveSearch } from '../config/index.js';
+import { embedConfig, featureEnabled, resolveSearch } from '../config/index.js';
 import { localModelMissing, MODEL_FILENAMES } from '../embed/store.js';
 import { SenseError } from '../errors.js';
 import { searchError } from '../output/search-error.js';
-import { segmentMatch } from '../text/segment.js';
 import { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.js';
 import { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.js';
 // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
@@ -97,25 +96,25 @@ function lineNumberAt(text, offset) {
     for(let i = 0; i < offset; i++)if (text.charCodeAt(i) === 10) line++;
     return line;
 }
-function lineRangeFor(db, path, line) {
-    const row = db.prepare('SELECT start_line, end_line FROM sections WHERE "path" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1').get(path, line, line);
+async function lineRangeFor(store, path, line) {
+    const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE "path" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');
+    const row = await stmt.get(path, line, line);
     return row ? `L${row.start_line}-${row.end_line}` : null;
 }
 // The declared or defaulted signals compose via RRF; `via` names which ones produced each row.
 // `opts` arrives already resolved (config.ts:resolveSearch).
-export async function search(db, cfg, terms, opts = {}) {
-    var _hits_get, _chunkLines_get, _chunkSimilarity_get;
+export async function search(store, cfg, terms, opts = {}) {
     const effective = resolveSearch(cfg, opts);
     const { k, signals } = effective;
-    const allPaths = db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path);
-    const scopePaths = rawScope(db, cfg, opts, allPaths);
+    const allPaths = (await (await store.prepare('SELECT "path" FROM frontmatter')).all()).map((r)=>r.path);
+    const scopePaths = await rawScope(store, cfg, opts, allPaths);
     const scopeActive = scopePaths.size < allPaths.length;
     // The set every candidate pool must be filtered to before truncation: scope narrowed by
     // --where, the same composition scopedPaths() gives the other commands. Runs the same where
     // fragment matchSql does, so a bad column gets the same attributed error either way.
     let allowedPaths;
     try {
-        allowedPaths = narrowByWhere(db, scopePaths, effective.where);
+        allowedPaths = await narrowByWhere(store, scopePaths, effective.where);
     } catch (err) {
         throw searchError(err, terms, effective.where);
     }
@@ -129,17 +128,9 @@ export async function search(db, cfg, terms, opts = {}) {
             throw new SenseError('EMBED_MODEL_MISSING', `preset "${effective.presetName}" searches with vectors, but the local model path "${e.model}" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop "vectors" from that preset's signals to search without them`);
         }
     }
-    const semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, allowedPaths);
-    // Terms pass to MATCH as written, with one transform: a run of text whose language marks no
-    // word boundaries becomes a quoted grapheme phrase (and a title:/summary:/text: qualifier
-    // ahead of it retargets its _seg column), matching how it was indexed. Text that already
-    // carries boundaries comes through untouched, so this is a no-op for the languages that
-    // never needed it. Gated on the same predicate that decided whether the sidecars were
-    // populated (contentTokenize(cfg) === undefined), so index and query can't disagree. Invalid
-    // syntax still errors rather than being rewritten.
-    // --where
-    // applies inside the candidate query (a post-filter would drop matches ranked past the pool)
-    // and again on the final select, for link-derived rows.
+    const semanticEnabled = wantsVectors && await scopeHasEmbeddings(store, cfg, allowedPaths);
+    // --where applies inside the candidate query (a post-filter would drop matches ranked past
+    // the pool) and again on the final select, for link-derived rows.
     const scope = effective.where;
     const whereJoin = scope ? `JOIN frontmatter f ON f."path" = content.path` : '';
     const whereCond = scope ? `AND (${scope})` : '';
@@ -147,16 +138,16 @@ export async function search(db, cfg, terms, opts = {}) {
     // after -- otherwise scoped notes ranking below the global top-`fetch` never reach the
     // filter. A join against a temp table, not a bound parameter list: real scopes run to
     // thousands of paths, past SQLITE_MAX_VARIABLE_NUMBER on older builds.
-    if (scopeActive) materializeScope(db, '_search_scope', scopePaths);
+    if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);
     const scopeCond = scopeActive ? `AND content.path IN (SELECT "path" FROM _search_scope)` : '';
-    const query = contentTokenize(cfg) === undefined ? segmentMatch(terms) : terms;
     const candidates = new Map();
     let matchRows = [];
-    // A query that rewrites to nothing (e.g. only unspaced-script punctuation) has no
-    // searchable words: zero lexical rows, and the other signals still compose.
-    if (signals.words !== undefined && query.trim() !== '') {
+    // A query that is only whitespace has no searchable words: zero lexical rows, and the other
+    // signals still compose. Any store-specific rewrite of `terms` (sqlite's grapheme phrase for
+    // unspaced scripts, duckdb's word/substring split) happens inside store.lexical.query itself.
+    if (signals.words !== undefined && terms.trim() !== '') {
         try {
-            matchRows = wordsCandidates(db, candidates, query, whereJoin, whereCond, scopeCond, fetch, signals.words);
+            matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);
         } catch (err) {
             throw searchError(err, terms, scope);
         }
@@ -168,16 +159,31 @@ export async function search(db, cfg, terms, opts = {}) {
     // hit === null here means the bound suppressed snippet(), not "no match" -- distinguish
     // from via='link' rows (never in matchRows, so absent from this set) below.
     const oversized = new Set(matchRows.filter((r)=>r.hit === null).map((r)=>r.path));
-    if (signals.links !== undefined) linksCandidates(db, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);
+    if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);
     let chunkLines = new Map();
     let chunkSimilarity = new Map();
     if (semanticEnabled) {
-        ({ chunkLines, chunkSimilarity } = await vectorsCandidates(db, cfg, candidates, terms, fetch, allowedPaths, signals.vectors));
+        ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors));
+    }
+    await store.exec('DROP TABLE IF EXISTS _search');
+    // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the
+    // rounded score/similarity are printed at full precision in rows.
+    await store.exec('CREATE TEMP TABLE _search ("path" TEXT PRIMARY KEY, score DOUBLE, via TEXT, hit TEXT, lines TEXT, similarity DOUBLE)');
+    if (candidates.size > 0) {
+        await store.runBatch('INSERT INTO _search ("path", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)', [
+            ...candidates
+        ].map(([path, c])=>{
+            var _hits_get, _chunkLines_get, _chunkSimilarity_get;
+            return [
+                path,
+                c.score,
+                c.via,
+                (_hits_get = hits.get(path)) !== null && _hits_get !== void 0 ? _hits_get : null,
+                (_chunkLines_get = chunkLines.get(path)) !== null && _chunkLines_get !== void 0 ? _chunkLines_get : null,
+                (_chunkSimilarity_get = chunkSimilarity.get(path)) !== null && _chunkSimilarity_get !== void 0 ? _chunkSimilarity_get : null
+            ];
+        }));
     }
-    db.exec('DROP TABLE IF EXISTS _search');
-    db.exec('CREATE TEMP TABLE _search ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');
-    const insert = db.prepare('INSERT INTO _search ("path", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');
-    for (const [path, c] of candidates)insert.run(path, c.score, c.via, (_hits_get = hits.get(path)) !== null && _hits_get !== void 0 ? _hits_get : null, (_chunkLines_get = chunkLines.get(path)) !== null && _chunkLines_get !== void 0 ? _chunkLines_get : null, (_chunkSimilarity_get = chunkSimilarity.get(path)) !== null && _chunkSimilarity_get !== void 0 ? _chunkSimilarity_get : null);
     // All three candidate paths (match, link, vector) already filtered to scope+where before
     // reaching _search; this reapplies --where anyway since the join to frontmatter is already
     // needed for the path column.
@@ -185,9 +191,10 @@ export async function search(db, cfg, terms, opts = {}) {
     // lines is always present now: semantic rows carry their chunk's range, oversized-doc
     // lexical rows gain one below, everything else stays null. similarity stays semantic-only.
     const similarityCol = semanticEnabled ? ', _search.similarity' : '';
-    const rows = db.prepare(`SELECT f."path" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}
+    const selectStmt = await store.prepare(`SELECT f."path" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}
        FROM _search JOIN frontmatter f ON f."path" = _search."path" JOIN content ON content.path = _search."path"
-       ${where} ORDER BY _search.score DESC LIMIT ?`).all(k);
+       ${where} ORDER BY _search.score DESC LIMIT ?`);
+    const rows = await selectStmt.all(k);
     if (oversized.size > 0) {
         const bareTerms = extractBareTerms(terms);
         for (const row of rows){
@@ -200,7 +207,7 @@ export async function search(db, cfg, terms, opts = {}) {
             }
             const { excerpt, offset } = computeExcerpt(text, bareTerms);
             row.hit = excerpt;
-            if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? lineRangeFor(db, row.path, lineNumberAt(text, offset)) : null;
+            if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path, lineNumberAt(text, offset)) : null;
         }
     }
     return rows;