sensemaking 0.24.7 → 0.25.0

package/dist/cjs/graph/traverse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/graph/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\nimport type { Store } from '../store/types.ts';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\ninterface FindPathOptions {\n  directed?: boolean;\n  maxDepth?: number;\n  allowed?: Set<string>;\n}\n\nasync function resetVisited(store: Store): Promise<void> {\n  await store.exec('DROP TABLE IF EXISTS temp.visited');\n  await store.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');\n}\n\n// Materializes the caller-resolved scope Set into a temp table so it joins like any other\n// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.\nasync function setupAllowed(store: Store, allowed: Set<string>, extra?: string): Promise<void> {\n  const all = new Set(allowed);\n  if (extra !== undefined) all.add(extra);\n  await store.exec('DROP TABLE IF EXISTS temp.allowed_nodes');\n  await store.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');\n  await store.runBatch(\n    'INSERT INTO allowed_nodes (path) VALUES (?)',\n    [...all].map((p) => [p])\n  );\n}\n\n// Only resolved links (dst IS NOT NULL) are followed, in either direction.\nfunction candidateSelect(direction: Direction): string {\n  const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  if (direction === 'forward') return forward;\n  if (direction === 'reverse') return reverse;\n  return `${forward} UNION ALL ${reverse}`;\n}\n\n// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against\n// everything already visited, optionally filtered to the allowed set, inserted at the new depth (?2); RETURNING gives what this ring newly reached.\nfunction ringSql(direction: Direction, hasAllowed: boolean): string {\n  return `\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (${candidateSelect(direction)}) cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}\nGROUP BY cand.path\nORDER BY cand.path\nRETURNING path`;\n}\n\nasync function reconstructPath(store: Store, to: string): Promise<string[]> {\n  const lookup = await store.prepare('SELECT pred FROM visited WHERE path = ?');\n  const path: string[] = [];\n  let cur: string | null = to;\n  while (cur !== null) {\n    path.push(cur);\n    const row = (await lookup.get(cur)) as { pred: string | null } | undefined;\n    cur = row ? row.pred : null;\n  }\n  return path.reverse();\n}\n\n// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching\n// personalizedRank; null when `to` is unreached within `maxDepth`.\nexport async function findPath(store: Store, from: string, to: string, opts: FindPathOptions = {}): Promise<string[] | null> {\n  if (from === to) return [from];\n\n  const direction: Direction = opts.directed ? 'forward' : 'both';\n  const maxDepth = opts.maxDepth ?? Infinity;\n  if (maxDepth <= 0) return null;\n\n  await resetVisited(store);\n  const insertStart = await store.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)');\n  await insertStart.run(from);\n  if (opts.allowed) await setupAllowed(store, opts.allowed, to);\n\n  const stmt = await store.prepare(ringSql(direction, !!opts.allowed));\n  for (let ring = 1; ring <= maxDepth; ring++) {\n    const rows = (await stmt.all(ring - 1, ring)) as Array<{ path: string }>;\n    if (rows.length === 0) break;\n    if (rows.some((row) => row.path === to)) return reconstructPath(store, to);\n  }\n  return null;\n}\n"],"names":["findPath","resetVisited","store","exec","setupAllowed","allowed","extra","all","Set","undefined","add","runBatch","map","p","candidateSelect","direction","forward","reverse","ringSql","hasAllowed","reconstructPath","to","lookup","path","cur","row","prepare","push","get","pred","from","opts","maxDepth","insertStart","stmt","ring","rows","directed","Infinity","run","length","some"],"mappings":"AAAA,2FAA2F;AAC3F,oEAAoE;;;;;+BAkE9CA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAvDtB,SAAeC,aAAaC,KAAY;;;;;oBACtC;;wBAAMA,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMC,IAAI,CAAC;;;oBAAjB;;;;;;IACF;;AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,SAAeC,aAAaF,KAAY,EAAEG,OAAoB,EAAEC,KAAc;;YACtEC;;;;oBAAAA,MAAM,IAAIC,IAAIH;oBACpB,IAAIC,UAAUG,WAAWF,IAAIG,GAAG,CAACJ;oBACjC;;wBAAMJ,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMS,QAAQ,CAClB,+CACA,AAAC,qBAAGJ,KAAKK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFzB;;;;;;IAIF;;AAEA,2EAA2E;AAC3E,SAASC,gBAAgBC,SAAoB;IAC3C,IAAMC,UAAU;IAChB,IAAMC,UAAU;IAChB,IAAIF,cAAc,WAAW,OAAOC;IACpC,IAAID,cAAc,WAAW,OAAOE;IACpC,OAAO,AAAC,GAAuBA,OAArBD,SAAQ,eAAqB,OAARC;AACjC;AAEA,0FAA0F;AAC1F,oJAAoJ;AACpJ,SAASC,QAAQH,SAAoB,EAAEI,UAAmB;IACxD,OAAO,AAAC,0FAKRA,OAFML,gBAAgBC,YAAW,mFAEuD,OAAxFI,aAAa,wEAAwE,IAAG;AAI1F;AAEA,SAAeC,gBAAgBlB,KAAY,EAAEmB,EAAU;;YAC/CC,QACAC,MACFC,KAGIC;;;;oBALO;;wBAAMvB,MAAMwB,OAAO,CAAC;;;oBAA7BJ,SAAS;oBACTC;oBACFC,MAAqBH;;;yBAClBG,CAAAA,QAAQ,IAAG;;;;oBAChBD,KAAKI,IAAI,CAACH;oBACG;;wBAAMF,OAAOM,GAAG,CAACJ;;;oBAAxBC,MAAO;oBACbD,MAAMC,MAAMA,IAAII,IAAI,GAAG;;;;;;oBAEzB;;wBAAON,KAAKN,OAAO;;;;IACrB;;AAIO,SAAejB;wCAASE,KAAY,EAAE4B,IAAY,EAAET,EAAU;YAAEU,MAIpDA,gBADXhB,WACAiB,UAIAC,aAIAC,MACGC,MACDC;;;;;oBAd6DL,OAAAA,oEAAwB,CAAC;oBAC9F,IAAID,SAAST,IAAI;;;4BAAQS;;;oBAEnBf,YAAuBgB,KAAKM,QAAQ,GAAG,YAAY;oBACnDL,YAAWD,iBAAAA,KAAKC,QAAQ,cAAbD,4BAAAA,iBAAiBO;oBAClC,IAAIN,YAAY,GAAG;;wBAAO;;oBAE1B;;wBAAM/B,aAAaC;;;oBAAnB;oBACoB;;wBAAMA,MAAMwB,OAAO,CAAC;;;oBAAlCO,cAAc;oBACpB;;wBAAMA,YAAYM,GAAG,CAACT;;;oBAAtB;yBACIC,KAAK1B,OAAO,EAAZ0B;;;;oBAAc;;wBAAM3B,aAAaF,OAAO6B,KAAK1B,OAAO,EAAEgB;;;oBAAxC;;;oBAEL;;wBAAMnB,MAAMwB,OAAO,CAACR,QAAQH,WAAW,CAAC,CAACgB,KAAK1B,OAAO;;;oBAA5D6B,OAAO;oBACJC,OAAO;;;yBAAGA,CAAAA,QAAQH,QAAO;;;;oBAClB;;wBAAME,KAAK3B,GAAG,CAAC4B,OAAO,GAAGA;;;oBAAjCC,OAAQ;oBACd,IAAIA,KAAKI,MAAM,KAAK,GAAG;;;;oBACvB,IAAIJ,KAAKK,IAAI,CAAC,SAAChB;+BAAQA,IAAIF,IAAI,KAAKF;wBAAK;;wBAAOD,gBAAgBlB,OAAOmB;;;;oBAHpCc;;;;;;oBAKrC;;wBAAO;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/graph/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\n\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\ninterface FindPathOptions {\n  directed?: boolean;\n  maxDepth?: number;\n  allowed?: Set<string>;\n}\n\nasync function resetVisited(store: Store): Promise<void> {\n  await store.exec('DROP TABLE IF EXISTS temp.visited');\n  await store.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');\n}\n\n// Materializes the caller-resolved scope Set into a temp table so it joins like any other\n// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.\nasync function setupAllowed(store: Store, allowed: Set<string>, extra?: string): Promise<void> {\n  const all = new Set(allowed);\n  if (extra !== undefined) all.add(extra);\n  await store.exec('DROP TABLE IF EXISTS temp.allowed_nodes');\n  await store.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');\n  await store.runBatch(\n    'INSERT INTO allowed_nodes (path) VALUES (?)',\n    [...all].map((p) => [p])\n  );\n}\n\n// Only resolved links (dst IS NOT NULL) are followed, in either direction.\nfunction candidateSelect(direction: Direction): string {\n  const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  if (direction === 'forward') return forward;\n  if (direction === 'reverse') return reverse;\n  return `${forward} UNION ALL ${reverse}`;\n}\n\n// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against\n// everything already visited, optionally filtered to the allowed set, inserted at the new depth (?2); RETURNING gives what this ring newly reached.\nfunction ringSql(direction: Direction, hasAllowed: boolean): string {\n  return `\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (${candidateSelect(direction)}) cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}\nGROUP BY cand.path\nORDER BY cand.path\nRETURNING path`;\n}\n\nasync function reconstructPath(store: Store, to: string): Promise<string[]> {\n  const lookup = await store.prepare('SELECT pred FROM visited WHERE path = ?');\n  const path: string[] = [];\n  let cur: string | null = to;\n  while (cur !== null) {\n    path.push(cur);\n    const row = (await lookup.get(cur)) as { pred: string | null } | undefined;\n    cur = row ? row.pred : null;\n  }\n  return path.reverse();\n}\n\n// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching\n// personalizedRank; null when `to` is unreached within `maxDepth`.\nexport function findPath(store: Store, from: string, to: string, opts: FindPathOptions = {}): Promise<string[] | null> {\n  return serialQuery(store, () => findIndexedPath(store, from, to, opts));\n}\n\nasync function findIndexedPath(store: Store, from: string, to: string, opts: FindPathOptions): Promise<string[] | null> {\n  if (from === to) return [from];\n\n  const direction: Direction = opts.directed ? 'forward' : 'both';\n  const maxDepth = opts.maxDepth ?? Infinity;\n  if (maxDepth <= 0) return null;\n\n  await resetVisited(store);\n  const insertStart = await store.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)');\n  await insertStart.run(from);\n  if (opts.allowed) await setupAllowed(store, opts.allowed, to);\n\n  const stmt = await store.prepare(ringSql(direction, !!opts.allowed));\n  for (let ring = 1; ring <= maxDepth; ring++) {\n    const rows = (await stmt.all(ring - 1, ring)) as Array<{ path: string }>;\n    if (rows.length === 0) break;\n    if (rows.some((row) => row.path === to)) return reconstructPath(store, to);\n  }\n  return null;\n}\n"],"names":["findPath","resetVisited","store","exec","setupAllowed","allowed","extra","all","Set","undefined","add","runBatch","map","p","candidateSelect","direction","forward","reverse","ringSql","hasAllowed","reconstructPath","to","lookup","path","cur","row","prepare","push","get","pred","from","opts","serialQuery","findIndexedPath","maxDepth","insertStart","stmt","ring","rows","directed","Infinity","run","length","some"],"mappings":"AAAA,2FAA2F;AAC3F,oEAAoE;;;;;+BAoEpDA;;;eAAAA;;;6BAlEY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAW5B,SAAeC,aAAaC,KAAY;;;;;oBACtC;;wBAAMA,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMC,IAAI,CAAC;;;oBAAjB;;;;;;IACF;;AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,SAAeC,aAAaF,KAAY,EAAEG,OAAoB,EAAEC,KAAc;;YACtEC;;;;oBAAAA,MAAM,IAAIC,IAAIH;oBACpB,IAAIC,UAAUG,WAAWF,IAAIG,GAAG,CAACJ;oBACjC;;wBAAMJ,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMS,QAAQ,CAClB,+CACA,AAAC,qBAAGJ,KAAKK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFzB;;;;;;IAIF;;AAEA,2EAA2E;AAC3E,SAASC,gBAAgBC,SAAoB;IAC3C,IAAMC,UAAU;IAChB,IAAMC,UAAU;IAChB,IAAIF,cAAc,WAAW,OAAOC;IACpC,IAAID,cAAc,WAAW,OAAOE;IACpC,OAAO,AAAC,GAAuBA,OAArBD,SAAQ,eAAqB,OAARC;AACjC;AAEA,0FAA0F;AAC1F,oJAAoJ;AACpJ,SAASC,QAAQH,SAAoB,EAAEI,UAAmB;IACxD,OAAO,AAAC,0FAKRA,OAFML,gBAAgBC,YAAW,mFAEuD,OAAxFI,aAAa,wEAAwE,IAAG;AAI1F;AAEA,SAAeC,gBAAgBlB,KAAY,EAAEmB,EAAU;;YAC/CC,QACAC,MACFC,KAGIC;;;;oBALO;;wBAAMvB,MAAMwB,OAAO,CAAC;;;oBAA7BJ,SAAS;oBACTC;oBACFC,MAAqBH;;;yBAClBG,CAAAA,QAAQ,IAAG;;;;oBAChBD,KAAKI,IAAI,CAACH;oBACG;;wBAAMF,OAAOM,GAAG,CAACJ;;;oBAAxBC,MAAO;oBACbD,MAAMC,MAAMA,IAAII,IAAI,GAAG;;;;;;oBAEzB;;wBAAON,KAAKN,OAAO;;;;IACrB;;AAIO,SAASjB,SAASE,KAAY,EAAE4B,IAAY,EAAET,EAAU;QAAEU,OAAAA,iEAAwB,CAAC;IACxF,OAAOC,IAAAA,0BAAW,EAAC9B,OAAO;eAAM+B,gBAAgB/B,OAAO4B,MAAMT,IAAIU;;AACnE;AAEA,SAAeE,gBAAgB/B,KAAY,EAAE4B,IAAY,EAAET,EAAU,EAAEU,IAAqB;;YAIzEA,gBADXhB,WACAmB,UAIAC,aAIAC,MACGC,MACDC;;;;oBAbR,IAAIR,SAAST,IAAI;;;4BAAQS;;;oBAEnBf,YAAuBgB,KAAKQ,QAAQ,GAAG,YAAY;oBACnDL,YAAWH,iBAAAA,KAAKG,QAAQ,cAAbH,4BAAAA,iBAAiBS;oBAClC,IAAIN,YAAY,GAAG;;wBAAO;;oBAE1B;;wBAAMjC,aAAaC;;;oBAAnB;oBACoB;;wBAAMA,MAAMwB,OAAO,CAAC;;;oBAAlCS,cAAc;oBACpB;;wBAAMA,YAAYM,GAAG,CAACX;;;oBAAtB;yBACIC,KAAK1B,OAAO,EAAZ0B;;;;oBAAc;;wBAAM3B,aAAaF,OAAO6B,KAAK1B,OAAO,EAAEgB;;;oBAAxC;;;oBAEL;;wBAAMnB,MAAMwB,OAAO,CAACR,QAAQH,WAAW,CAAC,CAACgB,KAAK1B,OAAO;;;oBAA5D+B,OAAO;oBACJC,OAAO;;;yBAAGA,CAAAA,QAAQH,QAAO;;;;oBAClB;;wBAAME,KAAK7B,GAAG,CAAC8B,OAAO,GAAGA;;;oBAAjCC,OAAQ;oBACd,IAAIA,KAAKI,MAAM,KAAK,GAAG;;;;oBACvB,IAAIJ,KAAKK,IAAI,CAAC,SAAClB;+BAAQA,IAAIF,IAAI,KAAKF;wBAAK;;wBAAOD,gBAAgBlB,OAAOmB;;;;oBAHpCgB;;;;;;oBAKrC;;wBAAO;;;;IACT"}

package/dist/cjs/index.d.cts

@@ -1,13 +1,13 @@
 export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.js';
 export { mapTree, peek, presetCoverage, search } from './commands/index.js';
-export type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.js';
+export type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
 export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
-export type { OpenResult, Stages } from './store/index.js';
-export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
+export type { BuildOptions, BuildResult, OpenOptions, OpenResult, Stages } from './store/index.js';
+export { buildIndex as build, CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/cjs/index.d.ts

@@ -1,13 +1,13 @@
 export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.js';
 export { mapTree, peek, presetCoverage, search } from './commands/index.js';
-export type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.js';
+export type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
 export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
-export type { OpenResult, Stages } from './store/index.js';
-export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
+export type { BuildOptions, BuildResult, OpenOptions, OpenResult, Stages } from './store/index.js';
+export { buildIndex as build, CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/cjs/index.js

@@ -32,6 +32,9 @@ _export(exports, {
     get SenseError () {
         return _errorsts.SenseError;
     },
+    get build () {
+        return _indexts2.buildIndex;
+    },
     get clearCache () {
         return _indexts2.clearCache;
     },

package/dist/cjs/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult, Stages } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAW9DA;eAAAA,0BAAgB;;QANhBC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIQC;eAAAA,oBAAU;;QANXC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAMXC;eAAbC,mBAAS;;QAR9BC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAEsC;uBAIvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { BuildOptions, BuildResult, OpenOptions, OpenResult, Stages } from './store/index.ts';\nexport { buildIndex as build, CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","build","buildIndex","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAWzCA;eAAAA,0BAAgB;;QANrCC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIIC;eAAdC,oBAAU;;QAA6BC;eAAAA,oBAAU;;QANhCC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAMUC;eAAbC,mBAAS;;QARnDC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAE2D;uBAI5D"}

package/dist/cjs/lib/atomic-write.js

@@ -8,17 +8,31 @@ Object.defineProperty(exports, "writeFileAtomic", {
         return writeFileAtomic;
     }
 });
+var _nodecrypto = require("node:crypto");
 var _nodefs = require("node:fs");
+function _instanceof(left, right) {
+    "@swc/helpers - instanceof";
+    if (right != null && typeof Symbol !== "undefined" && right[Symbol.hasInstance]) {
+        return !!right[Symbol.hasInstance](left);
+    } else return left instanceof right;
+}
 function writeFileAtomic(path, data) {
-    var tmp = "".concat(path, ".part");
-    (0, _nodefs.writeFileSync)(tmp, data);
+    var tmp = "".concat(path, ".").concat((0, _nodecrypto.randomUUID)(), ".part");
+    var staged = false;
     try {
+        (0, _nodefs.writeFileSync)(tmp, data, {
+            flag: 'wx'
+        });
+        staged = true;
         (0, _nodefs.renameSync)(tmp, path);
     } catch (err) {
-        try {
-            (0, _nodefs.unlinkSync)(tmp);
-        } catch (unused) {
-        // best-effort cleanup; the rename error below is what matters
+        // An exclusive-create collision belongs to another writer; never remove its staging file.
+        if (staged || !(_instanceof(err, Error) && 'code' in err && err.code === 'EEXIST')) {
+            try {
+                (0, _nodefs.unlinkSync)(tmp);
+            } catch (unused) {
+            // best-effort cleanup; the original error below is what matters
+            }
         }
         throw err;
     }

package/dist/cjs/lib/atomic-write.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/lib/atomic-write.ts"],"sourcesContent":["import { renameSync, unlinkSync, writeFileSync } from 'node:fs';\n\n// Stage-then-swap: write beside the target, then rename over it. A crash or full disk mid-write\n// leaves the target untouched instead of truncated. Matches embed/store.ts's model-download idiom.\nexport function writeFileAtomic(path: string, data: string | Buffer): void {\n  const tmp = `${path}.part`;\n  writeFileSync(tmp, data);\n  try {\n    renameSync(tmp, path);\n  } catch (err) {\n    try {\n      unlinkSync(tmp);\n    } catch {\n      // best-effort cleanup; the rename error below is what matters\n    }\n    throw err;\n  }\n}\n"],"names":["writeFileAtomic","path","data","tmp","writeFileSync","renameSync","err","unlinkSync"],"mappings":";;;;+BAIgBA;;;eAAAA;;;sBAJsC;AAI/C,SAASA,gBAAgBC,IAAY,EAAEC,IAAqB;IACjE,IAAMC,MAAM,AAAC,GAAO,OAALF,MAAK;IACpBG,IAAAA,qBAAa,EAACD,KAAKD;IACnB,IAAI;QACFG,IAAAA,kBAAU,EAACF,KAAKF;IAClB,EAAE,OAAOK,KAAK;QACZ,IAAI;YACFC,IAAAA,kBAAU,EAACJ;QACb,EAAE,eAAM;QACN,8DAA8D;QAChE;QACA,MAAMG;IACR;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/lib/atomic-write.ts"],"sourcesContent":["import { randomUUID } from 'node:crypto';\nimport { renameSync, unlinkSync, writeFileSync } from 'node:fs';\n\n// Stage-then-swap: write beside the target, then rename over it. A crash or full disk mid-write\n// leaves the target untouched instead of truncated. Matches embed/store.ts's model-download idiom.\nexport function writeFileAtomic(path: string, data: string | Buffer): void {\n  const tmp = `${path}.${randomUUID()}.part`;\n  let staged = false;\n  try {\n    writeFileSync(tmp, data, { flag: 'wx' });\n    staged = true;\n    renameSync(tmp, path);\n  } catch (err) {\n    // An exclusive-create collision belongs to another writer; never remove its staging file.\n    if (staged || !(err instanceof Error && 'code' in err && err.code === 'EEXIST')) {\n      try {\n        unlinkSync(tmp);\n      } catch {\n        // best-effort cleanup; the original error below is what matters\n      }\n    }\n    throw err;\n  }\n}\n"],"names":["writeFileAtomic","path","data","tmp","randomUUID","staged","writeFileSync","flag","renameSync","err","Error","code","unlinkSync"],"mappings":";;;;+BAKgBA;;;eAAAA;;;0BALW;sBAC2B;;;;;;;AAI/C,SAASA,gBAAgBC,IAAY,EAAEC,IAAqB;IACjE,IAAMC,MAAM,AAAC,GAAUC,OAARH,MAAK,KAAgB,OAAbG,IAAAA,sBAAU,KAAG;IACpC,IAAIC,SAAS;IACb,IAAI;QACFC,IAAAA,qBAAa,EAACH,KAAKD,MAAM;YAAEK,MAAM;QAAK;QACtCF,SAAS;QACTG,IAAAA,kBAAU,EAACL,KAAKF;IAClB,EAAE,OAAOQ,KAAK;QACZ,0FAA0F;QAC1F,IAAIJ,UAAU,CAAEI,CAAAA,AAAG,YAAHA,KAAeC,UAAS,UAAUD,OAAOA,IAAIE,IAAI,KAAK,QAAO,GAAI;YAC/E,IAAI;gBACFC,IAAAA,kBAAU,EAACT;YACb,EAAE,eAAM;YACN,gEAAgE;YAClE;QACF;QACA,MAAMM;IACR;AACF"}

package/dist/cjs/lib/serial-query.d.cts

@@ -0,0 +1 @@
+export declare function serialQuery<T>(handle: object, query: () => Promise<T>): Promise<T>;

package/dist/cjs/lib/serial-query.d.ts

@@ -0,0 +1 @@
+export declare function serialQuery<T>(handle: object, query: () => Promise<T>): Promise<T>;

package/dist/cjs/lib/serial-query.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "serialQuery", {
+    enumerable: true,
+    get: function() {
+        return serialQuery;
+    }
+});
+var pending = new WeakMap();
+function serialQuery(handle, query) {
+    var _pending_get;
+    var result = ((_pending_get = pending.get(handle)) !== null && _pending_get !== void 0 ? _pending_get : Promise.resolve()).then(query);
+    var settled = result.then(function() {}, function() {});
+    pending.set(handle, settled);
+    void settled.then(function() {
+        if (pending.get(handle) === settled) pending.delete(handle);
+    });
+    return result;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/lib/serial-query.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/lib/serial-query.ts"],"sourcesContent":["const pending = new WeakMap<object, Promise<void>>();\n\n// High-level queries share temporary tables and transaction state on a handle.\n// Raw SQL and handle closure remain caller-owned and must await these queries.\nexport function serialQuery<T>(handle: object, query: () => Promise<T>): Promise<T> {\n  const result = (pending.get(handle) ?? Promise.resolve()).then(query);\n  const settled = result.then(\n    () => {},\n    () => {}\n  );\n  pending.set(handle, settled);\n  void settled.then(() => {\n    if (pending.get(handle) === settled) pending.delete(handle);\n  });\n  return result;\n}\n"],"names":["serialQuery","pending","WeakMap","handle","query","result","get","Promise","resolve","then","settled","set","delete"],"mappings":";;;;+BAIgBA;;;eAAAA;;;AAJhB,IAAMC,UAAU,IAAIC;AAIb,SAASF,YAAeG,MAAc,EAAEC,KAAuB;QACpDH;IAAhB,IAAMI,SAAS,EAACJ,eAAAA,QAAQK,GAAG,CAACH,qBAAZF,0BAAAA,eAAuBM,QAAQC,OAAO,IAAIC,IAAI,CAACL;IAC/D,IAAMM,UAAUL,OAAOI,IAAI,CACzB,YAAO,GACP,YAAO;IAETR,QAAQU,GAAG,CAACR,QAAQO;IACpB,KAAKA,QAAQD,IAAI,CAAC;QAChB,IAAIR,QAAQK,GAAG,CAACH,YAAYO,SAAST,QAAQW,MAAM,CAACT;IACtD;IACA,OAAOE;AACT"}

package/dist/cjs/scan/index.d.cts

@@ -12,6 +12,7 @@ export interface ParsedDoc {
     presets: string[];
     data: Record<string, string | number | bigint | null>;
     parseError: string | null;
+    source: string;
     search: {
         title: string;
         summary: string;
@@ -19,6 +20,10 @@ export interface ParsedDoc {
     };
     extracted: Record<string, unknown>;
 }
+export declare function parseSource(file: FileStat, raw: string, extractors?: Feature[], cfg?: Config): {
+    doc: ParsedDoc;
+    warnings: string[];
+};
 export declare function parseFile(file: FileStat, extractors?: Feature[], cfg?: Config): {
     doc: ParsedDoc;
     warnings: string[];

package/dist/cjs/scan/index.d.ts

@@ -12,6 +12,7 @@ export interface ParsedDoc {
     presets: string[];
     data: Record<string, string | number | bigint | null>;
     parseError: string | null;
+    source: string;
     search: {
         title: string;
         summary: string;
@@ -19,6 +20,10 @@ export interface ParsedDoc {
     };
     extracted: Record<string, unknown>;
 }
+export declare function parseSource(file: FileStat, raw: string, extractors?: Feature[], cfg?: Config): {
+    doc: ParsedDoc;
+    warnings: string[];
+};
 export declare function parseFile(file: FileStat, extractors?: Feature[], cfg?: Config): {
     doc: ParsedDoc;
     warnings: string[];

package/dist/cjs/scan/index.js

@@ -24,6 +24,9 @@ _export(exports, {
     get parseFile () {
         return parseFile;
     },
+    get parseSource () {
+        return parseSource;
+    },
     get toPosixPath () {
         return _listts.toPosixPath;
     }
@@ -33,10 +36,8 @@ var _indexts = require("../chunk/index.js");
 var _stripts = require("../text/strip.js");
 var _frontmatterts = require("./frontmatter.js");
 var _listts = require("./list.js");
-function parseFile(file) {
-    var extractors = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : [], cfg = arguments.length > 2 ? arguments[2] : void 0;
-    // Preserve the authored bytes: feature extractors and section offsets use the raw document.
-    var raw = (0, _nodefs.readFileSync)(file.absPath, 'utf8');
+function parseSource(file, raw) {
+    var extractors = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : [], cfg = arguments.length > 3 ? arguments[3] : void 0;
     var warnings = [];
     var _splitFrontmatter = (0, _frontmatterts.splitFrontmatter)(raw), fm = _splitFrontmatter.fm, content = _splitFrontmatter.body;
     var _ref = fm === null ? {
@@ -91,6 +92,7 @@ function parseFile(file) {
             presets: file.presets,
             data: mapped,
             parseError: parseError,
+            source: raw,
             search: search,
             extracted: Object.fromEntries(extractors.filter(function(f) {
                 return f.extract;
@@ -105,4 +107,10 @@ function parseFile(file) {
         warnings: warnings
     };
 }
+function parseFile(file) {
+    var extractors = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : [], cfg = arguments.length > 2 ? arguments[2] : void 0;
+    // Preserve the authored bytes: feature extractors, indexed snippets, and section offsets use
+    // the same decoded string from this one read.
+    return parseSource(file, (0, _nodefs.readFileSync)(file.absPath, 'utf8'), extractors, cfg);
+}
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/scan/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors and section offsets use the raw document.\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","looksLikeDatetime","normalizeDate","parseFile","toPosixPath","file","extractors","cfg","raw","readFileSync","absPath","warnings","splitFrontmatter","fm","body","content","data","parseError","parseFrontmatter","relPath","mapped","Object","keys","key","has","push","value","mapValue","Number","isNaN","Date","parse","blocks","search","title","normalizeText","summary","text","textFromBlocks","doc","mtimeMs","ctimeMs","size","presets","extracted","fromEntries","filter","f","extract","map","name"],"mappings":";;;;;;;;;;;QAU2CA;eAAAA,+BAAgB;;QAElDC;eAAAA,iBAAS;;QAFTC;eAAAA,gCAAiB;;QAAEC;eAAAA,4BAAa;;QAoBzBC;eAAAA;;QAlBIC;eAAAA,mBAAW;;;sBAZF;uBACP;uBAGS;6BACkF;sBAO1E;AAkBhC,SAASD,UAAUE,IAAc;QAAEC,aAAAA,iEAAwB,EAAE,EAAEC;IACpE,4FAA4F;IAC5F,IAAMC,MAAMC,IAAAA,oBAAY,EAACJ,KAAKK,OAAO,EAAE;IACvC,IAAMC,WAAqB,EAAE;IAE7B,IAA8BC,oBAAAA,IAAAA,+BAAgB,EAACJ,MAAvCK,KAAsBD,kBAAtBC,IAAIC,AAAMC,UAAYH,kBAAlBE;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIC,IAAAA,+BAAgB,EAACb,KAAKc,OAAO,EAAEN,IAAIF,WAAlIK,OAAqBH,KAArBG,MAAMC,aAAeJ,KAAfI;IACd,IAAMG,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAOC,IAAI,CAACN,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMO,MAAN;YACH,IAAIxB,+BAAgB,CAACyB,GAAG,CAACD,MAAM;gBAC7BZ,SAASc,IAAI,CAAC,AAAC,YAAwDF,OAA7ClB,KAAKc,OAAO,EAAC,kCAAoC,OAAJI,KAAI;gBAC3E;YACF;YACA,IAAMG,QAAQC,IAAAA,uBAAQ,EAACX,IAAI,CAACO,IAAI;YAChC,IAAI,OAAOG,UAAU,YAAYzB,IAAAA,gCAAiB,EAACyB,UAAUE,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACL,SAAS;gBAC5Ff,SAASc,IAAI,CAAC,AAAC,YAA4BF,OAAjBlB,KAAKc,OAAO,EAAC,MAAgCO,OAA5BH,KAAI,0BAA8B,OAANG,OAAM;YAC/E;YACAN,MAAM,CAACG,IAAI,GAAGG;QAChB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,yFAAyF;IACzF,4EAA4E;IAC5E,IAAMM,SAASD,IAAAA,cAAK,EAAChB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMkB,SAAS;QAAEC,OAAOC,IAAAA,4BAAa,EAACnB,KAAKkB,KAAK;QAAGE,SAASD,IAAAA,4BAAa,EAACnB,KAAKoB,OAAO;QAAGC,MAAMC,IAAAA,uBAAc,EAACN;IAAQ;IAEtH,OAAO;QACLO,KAAK;YACHpB,SAASd,KAAKc,OAAO;YACrBqB,SAASnC,KAAKmC,OAAO;YACrBC,SAASpC,KAAKoC,OAAO;YACrBC,MAAMrC,KAAKqC,IAAI;YACfC,SAAStC,KAAKsC,OAAO;YACrB3B,MAAMI;YACNH,YAAAA;YACAgB,QAAAA;YACAW,WAAWvB,OAAOwB,WAAW,CAACvC,WAAWwC,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYvC,KAAKO,SAASkB,QAAQjB,MAAMT,KAAKyB;iBAAQ;;QAC7I;QACArB,UAAAA;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/index.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { parse } from '../chunk/index.ts';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport { textFromBlocks } from '../text/strip.ts';\nimport { looksLikeDatetime, mapValue, normalizeText, parseFrontmatter, RESERVED_COLUMNS, splitFrontmatter } from './frontmatter.ts';\nimport type { FileStat } from './list.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; store/sqlite/reconcile.ts does the SQL.\n\nexport { looksLikeDatetime, normalizeDate, RESERVED_COLUMNS } from './frontmatter.ts';\nexport type { FileStat } from './list.ts';\nexport { listFiles, toPosixPath } from './list.ts';\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  ctimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // Exact decoded source bytes read for this parse generation. Stored outside the FTS table so\n  // snippets and delayed embedding never consult a newer filesystem generation.\n  source: string;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nexport function parseSource(file: FileStat, raw: string, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // Parsed once and shared: the FTS text path (textFromBlocks, ~ strip.ts) and any feature\n  // that needs a parse tree (embed's chunker) both read this same block list.\n  const blocks = parse(content);\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: textFromBlocks(blocks) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      ctimeMs: file.ctimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      source: raw,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search, data, cfg, blocks)])),\n    },\n    warnings,\n  };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = [], cfg?: Config): { doc: ParsedDoc; warnings: string[] } {\n  // Preserve the authored bytes: feature extractors, indexed snippets, and section offsets use\n  // the same decoded string from this one read.\n  return parseSource(file, readFileSync(file.absPath, 'utf8'), extractors, cfg);\n}\n"],"names":["RESERVED_COLUMNS","listFiles","looksLikeDatetime","normalizeDate","parseFile","parseSource","toPosixPath","file","raw","extractors","cfg","warnings","splitFrontmatter","fm","body","content","data","parseError","parseFrontmatter","relPath","mapped","Object","keys","key","has","push","value","mapValue","Number","isNaN","Date","parse","blocks","search","title","normalizeText","summary","text","textFromBlocks","doc","mtimeMs","ctimeMs","size","presets","source","extracted","fromEntries","filter","f","extract","map","name","readFileSync","absPath"],"mappings":";;;;;;;;;;;QAU2CA;eAAAA,+BAAgB;;QAElDC;eAAAA,iBAAS;;QAFTC;eAAAA,gCAAiB;;QAAEC;eAAAA,4BAAa;;QAmEzBC;eAAAA;;QA5CAC;eAAAA;;QArBIC;eAAAA,mBAAW;;;sBAZF;uBACP;uBAGS;6BACkF;sBAO1E;AAqBhC,SAASD,YAAYE,IAAc,EAAEC,GAAW;QAAEC,aAAAA,iEAAwB,EAAE,EAAEC;IACnF,IAAMC,WAAqB,EAAE;IAE7B,IAA8BC,oBAAAA,IAAAA,+BAAgB,EAACJ,MAAvCK,KAAsBD,kBAAtBC,IAAIC,AAAMC,UAAYH,kBAAlBE;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAEG,MAAM,CAAC;QAA8BC,YAAY;IAAK,IAAIC,IAAAA,+BAAgB,EAACX,KAAKY,OAAO,EAAEN,IAAIF,WAAlIK,OAAqBH,KAArBG,MAAMC,aAAeJ,KAAfI;IACd,IAAMG,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAOC,IAAI,CAACN,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMO,MAAN;YACH,IAAIvB,+BAAgB,CAACwB,GAAG,CAACD,MAAM;gBAC7BZ,SAASc,IAAI,CAAC,AAAC,YAAwDF,OAA7ChB,KAAKY,OAAO,EAAC,kCAAoC,OAAJI,KAAI;gBAC3E;YACF;YACA,IAAMG,QAAQC,IAAAA,uBAAQ,EAACX,IAAI,CAACO,IAAI;YAChC,IAAI,OAAOG,UAAU,YAAYxB,IAAAA,gCAAiB,EAACwB,UAAUE,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACL,SAAS;gBAC5Ff,SAASc,IAAI,CAAC,AAAC,YAA4BF,OAAjBhB,KAAKY,OAAO,EAAC,MAAgCO,OAA5BH,KAAI,0BAA8B,OAANG,OAAM;YAC/E;YACAN,MAAM,CAACG,IAAI,GAAGG;QAChB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,yFAAyF;IACzF,4EAA4E;IAC5E,IAAMM,SAASD,IAAAA,cAAK,EAAChB;IAErB,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMkB,SAAS;QAAEC,OAAOC,IAAAA,4BAAa,EAACnB,KAAKkB,KAAK;QAAGE,SAASD,IAAAA,4BAAa,EAACnB,KAAKoB,OAAO;QAAGC,MAAMC,IAAAA,uBAAc,EAACN;IAAQ;IAEtH,OAAO;QACLO,KAAK;YACHpB,SAASZ,KAAKY,OAAO;YACrBqB,SAASjC,KAAKiC,OAAO;YACrBC,SAASlC,KAAKkC,OAAO;YACrBC,MAAMnC,KAAKmC,IAAI;YACfC,SAASpC,KAAKoC,OAAO;YACrB3B,MAAMI;YACNH,YAAAA;YACA2B,QAAQpC;YACRyB,QAAAA;YACAY,WAAWxB,OAAOyB,WAAW,CAACrC,WAAWsC,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEG,IAAI;qBAAEH,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYxC,KAAKO,SAASkB,QAAQjB,MAAMN,KAAKsB;iBAAQ;;QAC7I;QACArB,UAAAA;IACF;AACF;AAEO,SAASP,UAAUG,IAAc;QAAEE,aAAAA,iEAAwB,EAAE,EAAEC;IACpE,6FAA6F;IAC7F,8CAA8C;IAC9C,OAAOL,YAAYE,MAAM6C,IAAAA,oBAAY,EAAC7C,KAAK8C,OAAO,EAAE,SAAS5C,YAAYC;AAC3E"}

package/dist/cjs/scan/pool.d.cts

@@ -9,8 +9,13 @@ export type FileResult = {
 };
 export declare class ParsePool {
     private pool;
+    private workerData;
+    private operation;
     poolsCreated: number;
+    private enqueue;
     private ensure;
     run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]>;
+    private dispatch;
+    private destroy;
     close(): Promise<void>;
 }

package/dist/cjs/scan/pool.d.ts

@@ -9,8 +9,13 @@ export type FileResult = {
 };
 export declare class ParsePool {
     private pool;
+    private workerData;
+    private operation;
     poolsCreated: number;
+    private enqueue;
     private ensure;
     run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]>;
+    private dispatch;
+    private destroy;
     close(): Promise<void>;
 }

package/dist/cjs/scan/pool.js

@@ -9,8 +9,17 @@ Object.defineProperty(exports, "ParsePool", {
     }
 });
 var _nodemodule = require("node:module");
+var _nodeutil = require("node:util");
 var _workerfilets = require("../lib/worker-file.js");
 var _workererrorts = require("./worker-error.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -40,6 +49,17 @@ function _async_to_generator(fn) {
 function _class_call_check(instance, Constructor) {
     if (!(instance instanceof Constructor)) throw new TypeError("Cannot call a class as a function");
 }
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
 function _ts_generator(thisArg, body) {
     var f, y, t, _ = {
         label: 0,
@@ -139,82 +159,166 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
 // Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
 var _require = typeof require === 'undefined' ? (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString()) : require;
 var ParsePool = /*#__PURE__*/ function() {
     "use strict";
     function ParsePool() {
         _class_call_check(this, ParsePool);
-        // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;
-        // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.
+        this.operation = Promise.resolve();
+        // Tinypools this instance has constructed. Repeated equivalent dispatches leave this unchanged;
+        // a context change or close followed by run constructs another pool.
         this.poolsCreated = 0;
     }
     var _proto = ParsePool.prototype;
+    _proto.enqueue = function enqueue(operation) {
+        var result = this.operation.then(operation);
+        this.operation = result.then(function() {
+            return undefined;
+        }, function() {
+            return undefined;
+        });
+        return result;
+    };
     _proto.ensure = function ensure(workerData, maxWorkers) {
-        if (!this.pool) {
-            var Tinypool = _require('tinypool').Tinypool;
-            this.pool = new Tinypool({
-                filename: (0, _workerfilets.resolveWorkerFile)('parse'),
-                minThreads: maxWorkers,
-                maxThreads: maxWorkers,
-                workerData: workerData
+        return _async_to_generator(function() {
+            var Tinypool;
+            return _ts_generator(this, function(_state) {
+                switch(_state.label){
+                    case 0:
+                        if (!(this.pool && !(0, _nodeutil.isDeepStrictEqual)(this.workerData, workerData))) return [
+                            3,
+                            2
+                        ];
+                        return [
+                            4,
+                            this.destroy()
+                        ];
+                    case 1:
+                        _state.sent();
+                        _state.label = 2;
+                    case 2:
+                        if (!this.pool) {
+                            Tinypool = _require('tinypool').Tinypool;
+                            this.pool = new Tinypool({
+                                filename: (0, _workerfilets.resolveWorkerFile)('parse'),
+                                minThreads: maxWorkers,
+                                maxThreads: maxWorkers,
+                                workerData: workerData
+                            });
+                            this.workerData = workerData;
+                            this.poolsCreated++;
+                        }
+                        return [
+                            2,
+                            this.pool
+                        ];
+                }
             });
-            this.poolsCreated++;
-        }
-        return this.pool;
+        }).call(this);
     };
     // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
     // extracted text and feature values, never the token tree.
     _proto.run = function run(files, features, cfg, onParsed, maxWorkers) {
+        var _this = this;
+        // Snapshot at invocation so caller mutation cannot change queued work or its reuse baseline.
+        var workerData = {
+            cfg: structuredClone(cfg),
+            featureNames: _to_consumable_array(new Set(features.map(function(feature) {
+                return feature.name;
+            }))).sort()
+        };
+        return this.enqueue(function() {
+            return _this.dispatch(files, workerData, onParsed, maxWorkers);
+        });
+    };
+    _proto.dispatch = function dispatch(files, workerData, onParsed, maxWorkers) {
         return _async_to_generator(function() {
-            var workerData, pool, done;
+            var pool, done, settled, results, failures, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, result;
             return _ts_generator(this, function(_state) {
-                // cfg and the feature selection are constant for the dispatch, so they cross once per worker
-                // as workerData. Features carry closures and cannot cross at all; only their names do.
-                workerData = {
-                    cfg: cfg,
-                    featureNames: features.map(function(feature) {
-                        return feature.name;
-                    })
-                };
-                pool = this.ensure(workerData, maxWorkers);
-                done = 0;
-                // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order
-                // regardless of task completion order, which first-seen column order depends on downstream.
-                return [
-                    2,
-                    Promise.all(files.map(function(file) {
-                        return _async_to_generator(function() {
-                            var result;
-                            return _ts_generator(this, function(_state) {
-                                switch(_state.label){
-                                    case 0:
-                                        return [
-                                            4,
-                                            pool.run(file)
-                                        ];
-                                    case 1:
-                                        result = _state.sent();
-                                        if (!result.ok) throw (0, _workererrorts.reviveError)(result.error);
-                                        onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
-                                        return [
-                                            2,
-                                            {
-                                                doc: result.doc,
-                                                warnings: result.warnings,
-                                                parseMs: result.parseMs
-                                            }
-                                        ];
+                switch(_state.label){
+                    case 0:
+                        return [
+                            4,
+                            this.ensure(workerData, maxWorkers)
+                        ];
+                    case 1:
+                        pool = _state.sent();
+                        done = 0;
+                        return [
+                            4,
+                            Promise.allSettled(files.map(function(file) {
+                                return _async_to_generator(function() {
+                                    var result;
+                                    return _ts_generator(this, function(_state) {
+                                        switch(_state.label){
+                                            case 0:
+                                                return [
+                                                    4,
+                                                    pool.run(file)
+                                                ];
+                                            case 1:
+                                                result = _state.sent();
+                                                if (!result.ok) throw (0, _workererrorts.reviveError)(result.error);
+                                                onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
+                                                return [
+                                                    2,
+                                                    {
+                                                        doc: result.doc,
+                                                        warnings: result.warnings,
+                                                        parseMs: result.parseMs
+                                                    }
+                                                ];
+                                        }
+                                    });
+                                })();
+                            }))
+                        ];
+                    case 2:
+                        settled = _state.sent();
+                        results = [];
+                        failures = [];
+                        _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                        try {
+                            for(_iterator = settled[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                result = _step.value;
+                                if (result.status === 'fulfilled') results.push(result.value);
+                                else failures.push(result.reason);
+                            }
+                        } catch (err) {
+                            _didIteratorError = true;
+                            _iteratorError = err;
+                        } finally{
+                            try {
+                                if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                    _iterator.return();
                                 }
-                            });
-                        })();
-                    }))
-                ];
+                            } finally{
+                                if (_didIteratorError) {
+                                    throw _iteratorError;
+                                }
+                            }
+                        }
+                        if (failures.length === 1) throw failures[0];
+                        if (failures.length > 1) throw new AggregateError(failures, "".concat(failures.length, " parse tasks failed"));
+                        // allSettled preserves input order, which downstream first-seen column order depends on.
+                        return [
+                            2,
+                            results
+                        ];
+                }
             });
         }).call(this);
     };
-    // A pool never created costs nothing to destroy.
-    _proto.close = function close() {
+    _proto.destroy = function destroy() {
         return _async_to_generator(function() {
             var pool;
             return _ts_generator(this, function(_state) {
@@ -225,6 +329,7 @@ var ParsePool = /*#__PURE__*/ function() {
                         ];
                         pool = this.pool;
                         this.pool = undefined;
+                        this.workerData = undefined;
                         return [
                             4,
                             pool.destroy()
@@ -238,6 +343,13 @@ var ParsePool = /*#__PURE__*/ function() {
             });
         }).call(this);
     };
+    // A pool never created costs nothing to destroy. Queuing lets active dispatches drain first.
+    _proto.close = function close() {
+        var _this = this;
+        return this.enqueue(function() {
+            return _this.destroy();
+        });
+    };
     return ParsePool;
 }();
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }