sensemaking 0.23.1 → 0.24.1

package/dist/esm/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,6FAA6F;AAC7F,mGAAmG;AACnG,mGAAmG;AACnG,kGAAkG;AAClG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB,CAAU"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'vectors': the store's VectorStore is functionally implemented, not present-but-inert; checked\n// at open (index.ts) before a config with an \"embed\" block is allowed onto a store lacking it.\n// 'segment': the engine can run segment() (Intl.Segmenter grapheme clustering) as a SQL function;\n// turso's client cannot register it and has no rewrite for it, unlike has()/basename(), which now\n// work everywhere. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['vectors', 'segment'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\n// A ranked match, nothing more. Snippets are cut from the document above the store\n// (commands/search.ts), identically for every engine, so no store carries them.\nexport interface LexicalHit {\n  path: string;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with snippet, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,iGAAiG;AACjG,+FAA+F;AAC/F,kGAAkG;AAClG,kGAAkG;AAClG,qGAAqG;AACrG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;CAAU,CAAU"}

package/dist/esm/store/vectors.d.ts

@@ -2,6 +2,13 @@ import type { Connection } from './types.js';
 export declare const TARGET_CHUNK_CAP = 16;
 export declare function sampleEvenly<T>(rows: T[], cap?: number): T[];
 export declare function asCosine(score: number): number;
+export declare function compareVectorScores(a: {
+    path: string;
+    score: number;
+}, b: {
+    path: string;
+    score: number;
+}): number;
 export declare function pendingRows(conn: Connection): Promise<Array<{
     path: string;
     chunk: number;

package/dist/esm/store/vectors.js

@@ -7,11 +7,17 @@ export function sampleEvenly(rows, cap = TARGET_CHUNK_CAP) {
     const step = Math.max(1, Math.ceil(rows.length / cap));
     return rows.filter((_, i)=>i % step === 0);
 }
-// Dequantised int8 dot products (sqlite) land a little either side of a true cosine (an identical
-// pair can print 1.001); array_cosine_similarity (duckdb) has no such error but is rounded the same way, so both stores print the same bounded number.
+// Every store computes a real cosine; clamp float error at the ends and round for the public score.
 export function asCosine(score) {
     return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
 }
+// Rank by the computed cosine before rounding for display. Bytewise path order resolves only an
+// exact score tie, so two distinguishable similarities never cross a caller's result boundary.
+export function compareVectorScores(a, b) {
+    const byScore = b.score - a.score;
+    if (byScore !== 0) return byScore;
+    return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));
+}
 export async function pendingRows(conn) {
     const stmt = await conn.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk');
     return await stmt.all();

package/dist/esm/store/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Dequantised int8 dot products (sqlite) land a little either side of a true cosine (an identical\n// pair can print 1.001); array_cosine_similarity (duckdb) has no such error but is rounded the same way, so both stores print the same bounded number.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","pendingRows","conn","stmt","prepare","all","hasVectorRow","path","row","get","undefined"],"mappings":"AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMA,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,kGAAkG;AAClG,uJAAuJ;AACvJ,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,OAAO,eAAeG,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEK,IAAY;IAC/D,MAAMJ,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMI,MAAO,MAAML,KAAKM,GAAG,CAACF;IAC5B,OAAOC,QAAQE;AACjB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Every store computes a real cosine; clamp float error at the ends and round for the public score.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Rank by the computed cosine before rounding for display. Bytewise path order resolves only an\n// exact score tie, so two distinguishable similarities never cross a caller's result boundary.\nexport function compareVectorScores(a: { path: string; score: number }, b: { path: string; score: number }): number {\n  const byScore = b.score - a.score;\n  if (byScore !== 0) return byScore;\n  return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","asCosine","score","round","min","compareVectorScores","a","b","byScore","Buffer","compare","from","path","pendingRows","conn","stmt","prepare","all","hasVectorRow","row","get","undefined"],"mappings":"AAEA,gGAAgG;AAChG,yFAAyF;AACzF,OAAO,MAAMA,mBAAmB,GAAG;AAEnC,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASC,aAAgBC,IAAS,EAAEC,MAAcH,gBAAgB;IACvE,MAAMI,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIP,SAAS;AAC5C;AAEA,oGAAoG;AACpG,OAAO,SAASQ,SAASC,KAAa;IACpC,OAAOR,KAAKS,KAAK,CAACT,KAAKU,GAAG,CAAC,GAAGV,KAAKC,GAAG,CAAC,CAAC,GAAGO,UAAU,QAAQ;AAC/D;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,OAAO,SAASG,oBAAoBC,CAAkC,EAAEC,CAAkC;IACxG,MAAMC,UAAUD,EAAEL,KAAK,GAAGI,EAAEJ,KAAK;IACjC,IAAIM,YAAY,GAAG,OAAOA;IAC1B,OAAOC,OAAOC,OAAO,CAACD,OAAOE,IAAI,CAACL,EAAEM,IAAI,GAAGH,OAAOE,IAAI,CAACJ,EAAEK,IAAI;AAC/D;AAEA,OAAO,eAAeC,YAAYC,IAAgB;IAChD,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,OAAQ,MAAMD,KAAKE,GAAG;AACxB;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeC,aAAaJ,IAAgB,EAAEF,IAAY;IAC/D,MAAMG,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMG,MAAO,MAAMJ,KAAKK,GAAG,CAACR;IAC5B,OAAOO,QAAQE;AACjB"}

package/dist/esm/text/segment.d.ts

@@ -1,4 +1,17 @@
 export declare const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}";
 export declare function hasUnspacedRun(text: string): boolean;
+export interface SearchToken {
+    text: string;
+    start: number;
+    end: number;
+    unspaced: boolean;
+}
+export declare function searchTokens(text: string): SearchToken[];
+export declare function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean;
+export declare function unspacedRuns(text: string): Array<{
+    start: number;
+    end: number;
+}>;
 export declare function segmentField(text: string): string;
 export declare function segmentMatch(terms: string): string;
+export declare function foldForSearch(text: string): string;

package/dist/esm/text/segment.js

@@ -1,3 +1,4 @@
+import { stemmer } from 'stemmer';
 // Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in
 // seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.
 // Scripts written without word spaces: a closed set of writing systems.
@@ -12,6 +13,70 @@ const HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');
 export function hasUnspacedRun(text) {
     return HAS_RUN.test(text);
 }
+// Search tokens share the native separator contract: letters, marks, and numbers form tokens;
+// underscore, apostrophe, and hyphen separate them. Unspaced-script runs stay whole so callers
+// can apply substring semantics, while adjacent Latin text still gets its own token.
+export function searchTokens(text) {
+    const tokens = [];
+    for (const match of text.matchAll(/[\p{L}\p{N}][\p{L}\p{M}\p{N}]*/gu)){
+        const start = match.index;
+        const end = start + match[0].length;
+        let cursor = start;
+        for (const run of text.slice(start, end).matchAll(RUN)){
+            const runStart = start + run.index;
+            if (runStart > cursor) tokens.push({
+                text: text.slice(cursor, runStart),
+                start: cursor,
+                end: runStart,
+                unspaced: false
+            });
+            const runEnd = runStart + run[0].length;
+            tokens.push({
+                text: run[0],
+                start: runStart,
+                end: runEnd,
+                unspaced: true
+            });
+            cursor = runEnd;
+        }
+        if (cursor < end) tokens.push({
+            text: text.slice(cursor, end),
+            start: cursor,
+            end,
+            unspaced: false
+        });
+    }
+    return tokens;
+}
+// Phrase membership is checked after native ranking. A CJK token matches inside one authored
+// run; Latin tokens compare folded Porter stems. Each call scans one field once per candidate.
+export function matchesSearchPhrase(text, phrase) {
+    const haystack = searchTokens(text);
+    if (phrase.length === 0 || haystack.length < phrase.length) return false;
+    for(let start = 0; start <= haystack.length - phrase.length; start++){
+        let matches = true;
+        for(let i = 0; i < phrase.length; i++){
+            const query = phrase[i];
+            const authored = haystack[start + i];
+            if (query.unspaced) {
+                if (!authored.unspaced || !authored.text.includes(query.text)) matches = false;
+            } else if (authored.unspaced || stemmer(foldForSearch(authored.text)) !== stemmer(foldForSearch(query.text))) {
+                matches = false;
+            }
+            if (!matches) break;
+        }
+        if (matches) return true;
+    }
+    return false;
+}
+// Character spans of unspaced-script runs in text, the same boundary the index segments on --
+// the snippet marker uses this to keep substring semantics inside a run and word/stem matching outside it.
+export function unspacedRuns(text) {
+    return Array.from(text.matchAll(RUN), (m)=>({
+            start: m.index,
+            end: m.index + m[0].length
+        }));
+}
 // Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.
 // Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run
 // never segments at all, so no command pays it at module load.
@@ -87,3 +152,8 @@ export function segmentMatch(terms) {
     }
     return out;
 }
+// FTS tokenizers compare letters without combining marks. Keep the authored string for output and
+// offsets, but use this key when JS needs to compare a matched word to its source segment.
+export function foldForSearch(text) {
+    return text.normalize('NFD').replace(/\p{M}/gu, '').toLowerCase();
+}

package/dist/esm/text/segment.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","RUN","RegExp","HAS_RUN","hasUnspacedRun","text","test","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","run","Intl","Segmenter","undefined","granularity","Array","from","segment","s","splitOnPunctuation","groups","g","push","length","filter","segmentField","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","segmentMatch","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;AAClF,OAAO,MAAMA,mBAAmB,wGAAwG;AACxI,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,MAAM,IAAIC,OAAO,CAAC,KAAK,EAAEF,iBAAiB,WAAW,CAAC,EAAE;AAC9D,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEF,iBAAiB,CAAC,CAAC,EAAE;AAEpD,yFAAyF;AACzF,+CAA+C;AAC/C,OAAO,SAASI,eAAeC,IAAY;IACzC,OAAOF,QAAQG,IAAI,CAACD;AACtB;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAIE;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,MAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5BJ,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAIK,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOC,MAAMC,IAAI,CAACV,kBAAkBW,OAAO,CAACP,MAAM,CAACQ,IAAMA,EAAED,OAAO;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBT,GAAW;IACrC,MAAMU,SAAqB;QAAC,EAAE;KAAC;IAC/B,KAAK,MAAMC,KAAKZ,UAAUC,KAAM;QAC9B,IAAIH,YAAYF,IAAI,CAACgB,IAAID,OAAOE,IAAI,CAAC,EAAE;aAClCF,MAAM,CAACA,OAAOG,MAAM,GAAG,EAAE,CAACD,IAAI,CAACD;IACtC;IACA,OAAOD,OAAOI,MAAM,CAAC,CAACH,IAAMA,EAAEE,MAAM,GAAG;AACzC;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,OAAO,SAASE,aAAarB,IAAY;IACvC,IAAI,CAACF,QAAQG,IAAI,CAACD,OAAO,OAAO;IAChC,MAAMsB,MAAMtB,KAAKuB,OAAO,CAAC3B,KAAK,CAACU;QAC7B,MAAMkB,OAAOT,mBAAmBT,KAC7BmB,GAAG,CAAC,CAACR,IAAMA,EAAES,IAAI,CAAC,MAClBA,IAAI,CAAC,CAAC,CAAC,EAAEtB,QAAQ,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,EAAEA,QAAQ,CAAC,EAAEoB,KAAK,CAAC,EAAEpB,QAAQ,CAAC,CAAC;IAC1C;IACA,OAAOkB,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,MAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAASxB,GAAW,EAAEyB,YAAoB;IACjD,OAAOhB,mBAAmBT,KACvBmB,GAAG,CAAC,CAACR,IAAM,GAAGc,eAAed,EAAEE,MAAM,GAAG,IAAI,CAAC,CAAC,EAAEF,EAAES,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGT,CAAC,CAAC,EAAE,EAAE,EACvES,IAAI,CAAC;AACV;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,OAAO,SAASM,aAAaC,KAAa;IACxC,IAAI,CAACnC,QAAQG,IAAI,CAACgC,QAAQ,OAAOA;IACjC,IAAIX,MAAM;IACV,IAAIY,SAAS;IACb,MAAMC,SAASF,MAAMG,KAAK,CAACxC,MAAM,2CAA2C;IAC5E,IAAK,IAAIyC,IAAI,GAAGA,IAAIF,OAAOhB,MAAM,EAAEkB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAMC,MAAMH,MAAM,CAACE,EAAE,CAAE,IAAIC,OAAO,KAAKJ,SAAS,CAACA;YACtDZ,OAAOa,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVZ,OAAOa,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,MAAME,IAAIjB,IAAIkB,KAAK,CAACZ;QACpB,IAAIW,GAAG;YACLjB,MAAM,GAAGA,IAAImB,KAAK,CAAC,GAAGF,EAAEG,KAAK,IAAIH,CAAC,CAAC,EAAE,GAAGT,SAASK,MAAM,CAACE,EAAE,EAAE,GAAGE,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG;QACtF,OAAO;YACLjB,OAAOQ,SAASK,MAAM,CAACE,EAAE,EAAER;QAC7B;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\n\n// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n\nexport interface SearchToken {\n  text: string;\n  start: number;\n  end: number;\n  unspaced: boolean;\n}\n\n// Search tokens share the native separator contract: letters, marks, and numbers form tokens;\n// underscore, apostrophe, and hyphen separate them. Unspaced-script runs stay whole so callers\n// can apply substring semantics, while adjacent Latin text still gets its own token.\nexport function searchTokens(text: string): SearchToken[] {\n  const tokens: SearchToken[] = [];\n  for (const match of text.matchAll(/[\\p{L}\\p{N}][\\p{L}\\p{M}\\p{N}]*/gu)) {\n    const start = match.index;\n    const end = start + match[0].length;\n    let cursor = start;\n    for (const run of text.slice(start, end).matchAll(RUN)) {\n      const runStart = start + run.index;\n      if (runStart > cursor) tokens.push({ text: text.slice(cursor, runStart), start: cursor, end: runStart, unspaced: false });\n      const runEnd = runStart + run[0].length;\n      tokens.push({ text: run[0], start: runStart, end: runEnd, unspaced: true });\n      cursor = runEnd;\n    }\n    if (cursor < end) tokens.push({ text: text.slice(cursor, end), start: cursor, end, unspaced: false });\n  }\n  return tokens;\n}\n\n// Phrase membership is checked after native ranking. A CJK token matches inside one authored\n// run; Latin tokens compare folded Porter stems. Each call scans one field once per candidate.\nexport function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean {\n  const haystack = searchTokens(text);\n  if (phrase.length === 0 || haystack.length < phrase.length) return false;\n  for (let start = 0; start <= haystack.length - phrase.length; start++) {\n    let matches = true;\n    for (let i = 0; i < phrase.length; i++) {\n      const query = phrase[i];\n      const authored = haystack[start + i];\n      if (query.unspaced) {\n        if (!authored.unspaced || !authored.text.includes(query.text)) matches = false;\n      } else if (authored.unspaced || stemmer(foldForSearch(authored.text)) !== stemmer(foldForSearch(query.text))) {\n        matches = false;\n      }\n      if (!matches) break;\n    }\n    if (matches) return true;\n  }\n  return false;\n}\n\n// Character spans of unspaced-script runs in text, the same boundary the index segments on --\n// the snippet marker uses this to keep substring semantics inside a run and word/stem matching outside it.\nexport function unspacedRuns(text: string): Array<{ start: number; end: number }> {\n  return Array.from(text.matchAll(RUN), (m) => ({ start: m.index, end: m.index + m[0].length }));\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n\n// FTS tokenizers compare letters without combining marks. Keep the authored string for output and\n// offsets, but use this key when JS needs to compare a matched word to its source segment.\nexport function foldForSearch(text: string): string {\n  return text.normalize('NFD').replace(/\\p{M}/gu, '').toLowerCase();\n}\n"],"names":["stemmer","UNSPACED_SCRIPTS","RUN","RegExp","HAS_RUN","hasUnspacedRun","text","test","searchTokens","tokens","match","matchAll","start","index","end","length","cursor","run","slice","runStart","push","unspaced","runEnd","matchesSearchPhrase","phrase","haystack","matches","i","query","authored","includes","foldForSearch","unspacedRuns","Array","from","m","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","Intl","Segmenter","undefined","granularity","segment","s","splitOnPunctuation","groups","g","filter","segmentField","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","segmentMatch","terms","quoted","pieces","split","ch","normalize","toLowerCase"],"mappings":"AAAA,SAASA,OAAO,QAAQ,UAAU;AAElC,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;AAClF,OAAO,MAAMC,mBAAmB,wGAAwG;AACxI,2FAA2F;AAC3F,gDAAgD;AAChD,MAAMC,MAAM,IAAIC,OAAO,CAAC,KAAK,EAAEF,iBAAiB,WAAW,CAAC,EAAE;AAC9D,MAAMG,UAAU,IAAID,OAAO,CAAC,CAAC,EAAEF,iBAAiB,CAAC,CAAC,EAAE;AAEpD,yFAAyF;AACzF,+CAA+C;AAC/C,OAAO,SAASI,eAAeC,IAAY;IACzC,OAAOF,QAAQG,IAAI,CAACD;AACtB;AASA,8FAA8F;AAC9F,+FAA+F;AAC/F,qFAAqF;AACrF,OAAO,SAASE,aAAaF,IAAY;IACvC,MAAMG,SAAwB,EAAE;IAChC,KAAK,MAAMC,SAASJ,KAAKK,QAAQ,CAAC,oCAAqC;QACrE,MAAMC,QAAQF,MAAMG,KAAK;QACzB,MAAMC,MAAMF,QAAQF,KAAK,CAAC,EAAE,CAACK,MAAM;QACnC,IAAIC,SAASJ;QACb,KAAK,MAAMK,OAAOX,KAAKY,KAAK,CAACN,OAAOE,KAAKH,QAAQ,CAACT,KAAM;YACtD,MAAMiB,WAAWP,QAAQK,IAAIJ,KAAK;YAClC,IAAIM,WAAWH,QAAQP,OAAOW,IAAI,CAAC;gBAAEd,MAAMA,KAAKY,KAAK,CAACF,QAAQG;gBAAWP,OAAOI;gBAAQF,KAAKK;gBAAUE,UAAU;YAAM;YACvH,MAAMC,SAASH,WAAWF,GAAG,CAAC,EAAE,CAACF,MAAM;YACvCN,OAAOW,IAAI,CAAC;gBAAEd,MAAMW,GAAG,CAAC,EAAE;gBAAEL,OAAOO;gBAAUL,KAAKQ;gBAAQD,UAAU;YAAK;YACzEL,SAASM;QACX;QACA,IAAIN,SAASF,KAAKL,OAAOW,IAAI,CAAC;YAAEd,MAAMA,KAAKY,KAAK,CAACF,QAAQF;YAAMF,OAAOI;YAAQF;YAAKO,UAAU;QAAM;IACrG;IACA,OAAOZ;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,OAAO,SAASc,oBAAoBjB,IAAY,EAAEkB,MAAqB;IACrE,MAAMC,WAAWjB,aAAaF;IAC9B,IAAIkB,OAAOT,MAAM,KAAK,KAAKU,SAASV,MAAM,GAAGS,OAAOT,MAAM,EAAE,OAAO;IACnE,IAAK,IAAIH,QAAQ,GAAGA,SAASa,SAASV,MAAM,GAAGS,OAAOT,MAAM,EAAEH,QAAS;QACrE,IAAIc,UAAU;QACd,IAAK,IAAIC,IAAI,GAAGA,IAAIH,OAAOT,MAAM,EAAEY,IAAK;YACtC,MAAMC,QAAQJ,MAAM,CAACG,EAAE;YACvB,MAAME,WAAWJ,QAAQ,CAACb,QAAQe,EAAE;YACpC,IAAIC,MAAMP,QAAQ,EAAE;gBAClB,IAAI,CAACQ,SAASR,QAAQ,IAAI,CAACQ,SAASvB,IAAI,CAACwB,QAAQ,CAACF,MAAMtB,IAAI,GAAGoB,UAAU;YAC3E,OAAO,IAAIG,SAASR,QAAQ,IAAIrB,QAAQ+B,cAAcF,SAASvB,IAAI,OAAON,QAAQ+B,cAAcH,MAAMtB,IAAI,IAAI;gBAC5GoB,UAAU;YACZ;YACA,IAAI,CAACA,SAAS;QAChB;QACA,IAAIA,SAAS,OAAO;IACtB;IACA,OAAO;AACT;AAEA,8FAA8F;AAC9F,2GAA2G;AAC3G,OAAO,SAASM,aAAa1B,IAAY;IACvC,OAAO2B,MAAMC,IAAI,CAAC5B,KAAKK,QAAQ,CAACT,MAAM,CAACiC,IAAO,CAAA;YAAEvB,OAAOuB,EAAEtB,KAAK;YAAEC,KAAKqB,EAAEtB,KAAK,GAAGsB,CAAC,CAAC,EAAE,CAACpB,MAAM;QAAC,CAAA;AAC7F;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAIqB;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,MAAMC,UAAU;AAEhB,SAASC,UAAUtB,GAAW;IAC5BmB,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAII,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOV,MAAMC,IAAI,CAACE,kBAAkBQ,OAAO,CAAC3B,MAAM,CAAC4B,IAAMA,EAAED,OAAO;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmB7B,GAAW;IACrC,MAAM8B,SAAqB;QAAC,EAAE;KAAC;IAC/B,KAAK,MAAMC,KAAKT,UAAUtB,KAAM;QAC9B,IAAIoB,YAAY9B,IAAI,CAACyC,IAAID,OAAO3B,IAAI,CAAC,EAAE;aAClC2B,MAAM,CAACA,OAAOhC,MAAM,GAAG,EAAE,CAACK,IAAI,CAAC4B;IACtC;IACA,OAAOD,OAAOE,MAAM,CAAC,CAACD,IAAMA,EAAEjC,MAAM,GAAG;AACzC;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,OAAO,SAASmC,aAAa5C,IAAY;IACvC,IAAI,CAACF,QAAQG,IAAI,CAACD,OAAO,OAAO;IAChC,MAAM6C,MAAM7C,KAAK8C,OAAO,CAAClD,KAAK,CAACe;QAC7B,MAAMoC,OAAOP,mBAAmB7B,KAC7BqC,GAAG,CAAC,CAACN,IAAMA,EAAEO,IAAI,CAAC,MAClBA,IAAI,CAAC,CAAC,CAAC,EAAEjB,QAAQ,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,EAAEA,QAAQ,CAAC,EAAEe,KAAK,CAAC,EAAEf,QAAQ,CAAC,CAAC;IAC1C;IACA,OAAOa,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,MAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAAS1C,GAAW,EAAE2C,YAAoB;IACjD,OAAOd,mBAAmB7B,KACvBqC,GAAG,CAAC,CAACN,IAAM,GAAGY,eAAeZ,EAAEjC,MAAM,GAAG,IAAI,CAAC,CAAC,EAAEiC,EAAEO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAGP,CAAC,CAAC,EAAE,EAAE,EACvEO,IAAI,CAAC;AACV;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,OAAO,SAASM,aAAaC,KAAa;IACxC,IAAI,CAAC1D,QAAQG,IAAI,CAACuD,QAAQ,OAAOA;IACjC,IAAIX,MAAM;IACV,IAAIY,SAAS;IACb,MAAMC,SAASF,MAAMG,KAAK,CAAC/D,MAAM,2CAA2C;IAC5E,IAAK,IAAIyB,IAAI,GAAGA,IAAIqC,OAAOjD,MAAM,EAAEY,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;YACf,KAAK,MAAMuC,MAAMF,MAAM,CAACrC,EAAE,CAAE,IAAIuC,OAAO,KAAKH,SAAS,CAACA;YACtDZ,OAAOa,MAAM,CAACrC,EAAE;YAChB;QACF;QACA,IAAIoC,QAAQ;YACVZ,OAAOa,MAAM,CAACrC,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,MAAMQ,IAAIgB,IAAIzC,KAAK,CAAC+C;QACpB,IAAItB,GAAG;YACLgB,MAAM,GAAGA,IAAIjC,KAAK,CAAC,GAAGiB,EAAEtB,KAAK,IAAIsB,CAAC,CAAC,EAAE,GAAGwB,SAASK,MAAM,CAACrC,EAAE,EAAE,GAAGQ,CAAC,CAAC,EAAE,GAAGA,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG;QACtF,OAAO;YACLgB,OAAOQ,SAASK,MAAM,CAACrC,EAAE,EAAE+B;QAC7B;IACF;IACA,OAAOP;AACT;AAEA,kGAAkG;AAClG,2FAA2F;AAC3F,OAAO,SAASpB,cAAczB,IAAY;IACxC,OAAOA,KAAK6D,SAAS,CAAC,OAAOf,OAAO,CAAC,WAAW,IAAIgB,WAAW;AACjE"}

package/dist/esm/watch.d.ts

@@ -1,7 +1,7 @@
 import type { ResolvedConfig } from './config/index.js';
 export type WatchEvent = {
     type: 'started';
-    baseDir: string;
+    rootDir: string;
     dbPath: string;
 } | {
     type: 'reconciled';

package/dist/esm/watch.js

@@ -10,12 +10,13 @@ const STALE_HEARTBEAT_MS = 15000;
 // Runs in the foreground until SIGINT/SIGTERM/signal abort. duckdb/turso hand the cache file to
 // one process at a time, so every cycle opens, reconciles, writes the heartbeat/pid rows, and closes.
 export async function runWatch(cfg, opts = {}) {
-    var _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs;
+    var _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs, _cfg_rootDir;
     const onEvent = (_opts_onEvent = opts.onEvent) !== null && _opts_onEvent !== void 0 ? _opts_onEvent : ()=>{};
     const debounceMs = (_opts_debounceMs = opts.debounceMs) !== null && _opts_debounceMs !== void 0 ? _opts_debounceMs : DEBOUNCE_MS;
     const heartbeatIntervalMs = (_opts_heartbeatIntervalMs = opts.heartbeatIntervalMs) !== null && _opts_heartbeatIntervalMs !== void 0 ? _opts_heartbeatIntervalMs : HEARTBEAT_INTERVAL_MS;
-    const baseDir = cfg.baseDir;
     const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);
+    // openStore normalizes the legacy direct-API baseDir before a watcher touches the filesystem.
+    const rootDir = (_cfg_rootDir = cfg.rootDir) !== null && _cfg_rootDir !== void 0 ? _cfg_rootDir : cfg.baseDir;
     const existingHeartbeat = await getMeta(initialStore, 'watch_heartbeat');
     if (existingHeartbeat && !opts.force) {
         const age = Date.now() - Date.parse(existingHeartbeat);
@@ -31,7 +32,7 @@ export async function runWatch(cfg, opts = {}) {
     await initialStore.close();
     onEvent({
         type: 'started',
-        baseDir,
+        rootDir,
         dbPath
     });
     if (initialWarnings.length > 0 || initialParsed > 0) {
@@ -99,7 +100,7 @@ export async function runWatch(cfg, opts = {}) {
     };
     // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with
     // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.
-    const watcher = fsWatch(baseDir, {
+    const watcher = fsWatch(rootDir, {
         recursive: true
     }, (_event, filename)=>{
         if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. duckdb/turso hand the cache file to\n// one process at a time, so every cycle opens, reconciles, writes the heartbeat/pid rows, and closes.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const baseDir = cfg.baseDir;\n\n  const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const existingHeartbeat = await getMeta(initialStore, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await initialStore.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  // total is read before close: an event fires only once nothing is held open again.\n  const initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n  await setMeta(initialStore, 'watch_heartbeat', new Date().toISOString());\n  await setMeta(initialStore, 'watch_pid', String(process.pid));\n  await initialStore.close();\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n  // coalesced into a single rerun instead of opening a second, overlapping connection.\n  let running: Promise<void> | null = null;\n  let queuedAlways: boolean | null = null;\n\n  const tick = async (alwaysEmit: boolean): Promise<void> => {\n    try {\n      const { store, parsed, warnings } = await openStore(cfg);\n      try {\n        await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n        await setMeta(store, 'watch_pid', String(process.pid));\n        if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        }\n      } finally {\n        await store.close();\n      }\n    } catch (err) {\n      onEvent({ type: 'reconcile-error', message: (err as Error).message });\n    }\n  };\n\n  const startCycle = (alwaysEmit: boolean) => {\n    running = tick(alwaysEmit).then(() => {\n      running = null;\n      if (queuedAlways !== null && !stopping) {\n        const again = queuedAlways;\n        queuedAlways = null;\n        startCycle(again);\n      }\n    });\n  };\n\n  const requestCycle = (alwaysEmit: boolean) => {\n    if (stopping) return;\n    if (running) {\n      queuedAlways = queuedAlways || alwaysEmit;\n      return;\n    }\n    startCycle(alwaysEmit);\n  };\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      requestCycle(true);\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(\n      async () => requestCycle(false),\n      () => stopping\n    ),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await running;\n      const { store: finalStore } = await openStore(cfg);\n      await setMeta(finalStore, 'watch_heartbeat', null);\n      await setMeta(finalStore, 'watch_pid', null);\n      await finalStore.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","baseDir","store","initialStore","dbPath","warnings","initialWarnings","parsed","initialParsed","existingHeartbeat","force","age","Date","now","parse","close","Math","round","initialTotal","length","toISOString","String","process","pid","type","total","stopping","running","queuedAlways","tick","alwaysEmit","err","message","startCycle","then","again","requestCycle","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","finalStore","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,mBAAmB;AAEzE,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,gGAAgG;AAChG,sGAAsG;AACtG,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QACzDA,eACGA,kBACSA;IAF5B,MAAMC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAMQ,UAAUL,IAAIK,OAAO;IAE3B,MAAM,EAAEC,OAAOC,YAAY,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMlB,UAAUM;IAC1G,MAAMa,oBAAoB,MAAMpB,QAAQc,cAAc;IACtD,IAAIM,qBAAqB,CAACZ,KAAKa,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMjB,oBAAoB;YACxC,MAAMS,aAAaY,KAAK;YACxB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEA,mFAAmF;IACnF,MAAMO,eAAeZ,gBAAgBa,MAAM,GAAG,KAAKX,gBAAgB,IAAI,MAAMpB,SAASe,gBAAgB;IACtG,MAAMZ,QAAQY,cAAc,mBAAmB,IAAIS,OAAOQ,WAAW;IACrE,MAAM7B,QAAQY,cAAc,aAAakB,OAAOC,QAAQC,GAAG;IAC3D,MAAMpB,aAAaY,KAAK;IAExBjB,QAAQ;QAAE0B,MAAM;QAAWvB;QAASG;IAAO;IAC3C,IAAIE,gBAAgBa,MAAM,GAAG,KAAKX,gBAAgB,GAAG;QACnDV,QAAQ;YAAE0B,MAAM;YAAcjB,QAAQC;YAAeiB,OAAOP;YAAcb,UAAUC;QAAgB;IACtG;IAEA,IAAIoB,WAAW;IACf,6FAA6F;IAC7F,qFAAqF;IACrF,IAAIC,UAAgC;IACpC,IAAIC,eAA+B;IAEnC,MAAMC,OAAO,OAAOC;QAClB,IAAI;YACF,MAAM,EAAE5B,KAAK,EAAEK,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMf,UAAUM;YACpD,IAAI;gBACF,MAAML,QAAQW,OAAO,mBAAmB,IAAIU,OAAOQ,WAAW;gBAC9D,MAAM7B,QAAQW,OAAO,aAAamB,OAAOC,QAAQC,GAAG;gBACpD,IAAIO,cAAcvB,SAAS,KAAKF,SAASc,MAAM,GAAG,GAAG;oBACnDrB,QAAQ;wBAAE0B,MAAM;wBAAcjB;wBAAQkB,OAAO,MAAMrC,SAASc;wBAAQG;oBAAS;gBAC/E;YACF,SAAU;gBACR,MAAMH,MAAMa,KAAK;YACnB;QACF,EAAE,OAAOgB,KAAK;YACZjC,QAAQ;gBAAE0B,MAAM;gBAAmBQ,SAAS,AAACD,IAAcC,OAAO;YAAC;QACrE;IACF;IAEA,MAAMC,aAAa,CAACH;QAClBH,UAAUE,KAAKC,YAAYI,IAAI,CAAC;YAC9BP,UAAU;YACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;gBACtC,MAAMS,QAAQP;gBACdA,eAAe;gBACfK,WAAWE;YACb;QACF;IACF;IAEA,MAAMC,eAAe,CAACN;QACpB,IAAIJ,UAAU;QACd,IAAIC,SAAS;YACXC,eAAeA,gBAAgBE;YAC/B;QACF;QACAG,WAAWH;IACb;IAEA,IAAIO,gBAAuC;IAC3C,MAAMC,oBAAoB;QACxB,IAAID,eAAeE,aAAaF;QAChCA,gBAAgBG,WAAW;YACzBH,gBAAgB;YAChBD,aAAa;QACf,GAAGrC;IACL;IAEA,iGAAiG;IACjG,uIAAuI;IACvI,MAAM0C,UAAUzD,QAAQiB,SAAS;QAAEyC,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAAC5D,YAAY;QACpEqD;IACF;IACA,MAAMQ,iBAAiBC,YACrB5D,YACE,UAAYiD,aAAa,QACzB,IAAMV,WAER1B;IAGF,OAAO,IAAIgD,QAAc,CAACC;YAqBpBpD,cACCA;QArBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAMqD,WAAW;gBAKfrD;YAJA,IAAI6B,UAAU;YACdA,WAAW;YACXJ,QAAQ6B,GAAG,CAAC,UAAUD;YACtB5B,QAAQ6B,GAAG,CAAC,WAAWD;aACvBrD,eAAAA,KAAKuD,MAAM,cAAXvD,mCAAAA,aAAawD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIT,eAAeE,aAAaF;YAChCI,QAAQ1B,KAAK;YACb,MAAMY;YACN,MAAM,EAAEzB,OAAOqD,UAAU,EAAE,GAAG,MAAMjE,UAAUM;YAC9C,MAAML,QAAQgE,YAAY,mBAAmB;YAC7C,MAAMhE,QAAQgE,YAAY,aAAa;YACvC,MAAMA,WAAWxC,KAAK;YACtBkC;QACF;QACA3B,QAAQkC,IAAI,CAAC,UAAUN;QACvB5B,QAAQkC,IAAI,CAAC,WAAWN;QACxB,KAAIrD,eAAAA,KAAKuD,MAAM,cAAXvD,mCAAAA,aAAa4D,OAAO,EAAEP;cACrBrD,gBAAAA,KAAKuD,MAAM,cAAXvD,oCAAAA,cAAa6D,gBAAgB,CAAC,SAASR,UAAU;YAAEM,MAAM;QAAK;IACrE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; rootDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. duckdb/turso hand the cache file to\n// one process at a time, so every cycle opens, reconciles, writes the heartbeat/pid rows, and closes.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  // openStore normalizes the legacy direct-API baseDir before a watcher touches the filesystem.\n  const rootDir = cfg.rootDir ?? cfg.baseDir;\n  const existingHeartbeat = await getMeta(initialStore, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await initialStore.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  // total is read before close: an event fires only once nothing is held open again.\n  const initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n  await setMeta(initialStore, 'watch_heartbeat', new Date().toISOString());\n  await setMeta(initialStore, 'watch_pid', String(process.pid));\n  await initialStore.close();\n\n  onEvent({ type: 'started', rootDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n  // coalesced into a single rerun instead of opening a second, overlapping connection.\n  let running: Promise<void> | null = null;\n  let queuedAlways: boolean | null = null;\n\n  const tick = async (alwaysEmit: boolean): Promise<void> => {\n    try {\n      const { store, parsed, warnings } = await openStore(cfg);\n      try {\n        await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n        await setMeta(store, 'watch_pid', String(process.pid));\n        if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        }\n      } finally {\n        await store.close();\n      }\n    } catch (err) {\n      onEvent({ type: 'reconcile-error', message: (err as Error).message });\n    }\n  };\n\n  const startCycle = (alwaysEmit: boolean) => {\n    running = tick(alwaysEmit).then(() => {\n      running = null;\n      if (queuedAlways !== null && !stopping) {\n        const again = queuedAlways;\n        queuedAlways = null;\n        startCycle(again);\n      }\n    });\n  };\n\n  const requestCycle = (alwaysEmit: boolean) => {\n    if (stopping) return;\n    if (running) {\n      queuedAlways = queuedAlways || alwaysEmit;\n      return;\n    }\n    startCycle(alwaysEmit);\n  };\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      requestCycle(true);\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(rootDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(\n      async () => requestCycle(false),\n      () => stopping\n    ),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await running;\n      const { store: finalStore } = await openStore(cfg);\n      await setMeta(finalStore, 'watch_heartbeat', null);\n      await setMeta(finalStore, 'watch_pid', null);\n      await finalStore.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","initialStore","dbPath","warnings","initialWarnings","parsed","initialParsed","rootDir","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","initialTotal","length","toISOString","String","process","pid","type","total","stopping","running","queuedAlways","tick","alwaysEmit","err","message","startCycle","then","again","requestCycle","debounceTimer","scheduleReconcile","clearTimeout","setTimeout","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","finalStore","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,mBAAmB;AAEzE,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,gGAAgG;AAChG,sGAAsG;AACtG,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QACzDA,eACGA,kBACSA,2BAGZD;IALhB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,OAAOC,YAAY,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUM;IAC1G,8FAA8F;IAC9F,MAAMY,WAAUZ,eAAAA,IAAIY,OAAO,cAAXZ,0BAAAA,eAAeA,IAAIa,OAAO;IAC1C,MAAMC,oBAAoB,MAAMrB,QAAQa,cAAc;IACtD,IAAIQ,qBAAqB,CAACb,KAAKc,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMlB,oBAAoB;YACxC,MAAMQ,aAAac,KAAK;YACxB,MAAM,IAAI9B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE+B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEA,mFAAmF;IACnF,MAAMO,eAAed,gBAAgBe,MAAM,GAAG,KAAKb,gBAAgB,IAAI,MAAMnB,SAASc,gBAAgB;IACtG,MAAMX,QAAQW,cAAc,mBAAmB,IAAIW,OAAOQ,WAAW;IACrE,MAAM9B,QAAQW,cAAc,aAAaoB,OAAOC,QAAQC,GAAG;IAC3D,MAAMtB,aAAac,KAAK;IAExBlB,QAAQ;QAAE2B,MAAM;QAAWjB;QAASL;IAAO;IAC3C,IAAIE,gBAAgBe,MAAM,GAAG,KAAKb,gBAAgB,GAAG;QACnDT,QAAQ;YAAE2B,MAAM;YAAcnB,QAAQC;YAAemB,OAAOP;YAAcf,UAAUC;QAAgB;IACtG;IAEA,IAAIsB,WAAW;IACf,6FAA6F;IAC7F,qFAAqF;IACrF,IAAIC,UAAgC;IACpC,IAAIC,eAA+B;IAEnC,MAAMC,OAAO,OAAOC;QAClB,IAAI;YACF,MAAM,EAAE9B,KAAK,EAAEK,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMd,UAAUM;YACpD,IAAI;gBACF,MAAML,QAAQU,OAAO,mBAAmB,IAAIY,OAAOQ,WAAW;gBAC9D,MAAM9B,QAAQU,OAAO,aAAaqB,OAAOC,QAAQC,GAAG;gBACpD,IAAIO,cAAczB,SAAS,KAAKF,SAASgB,MAAM,GAAG,GAAG;oBACnDtB,QAAQ;wBAAE2B,MAAM;wBAAcnB;wBAAQoB,OAAO,MAAMtC,SAASa;wBAAQG;oBAAS;gBAC/E;YACF,SAAU;gBACR,MAAMH,MAAMe,KAAK;YACnB;QACF,EAAE,OAAOgB,KAAK;YACZlC,QAAQ;gBAAE2B,MAAM;gBAAmBQ,SAAS,AAACD,IAAcC,OAAO;YAAC;QACrE;IACF;IAEA,MAAMC,aAAa,CAACH;QAClBH,UAAUE,KAAKC,YAAYI,IAAI,CAAC;YAC9BP,UAAU;YACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;gBACtC,MAAMS,QAAQP;gBACdA,eAAe;gBACfK,WAAWE;YACb;QACF;IACF;IAEA,MAAMC,eAAe,CAACN;QACpB,IAAIJ,UAAU;QACd,IAAIC,SAAS;YACXC,eAAeA,gBAAgBE;YAC/B;QACF;QACAG,WAAWH;IACb;IAEA,IAAIO,gBAAuC;IAC3C,MAAMC,oBAAoB;QACxB,IAAID,eAAeE,aAAaF;QAChCA,gBAAgBG,WAAW;YACzBH,gBAAgB;YAChBD,aAAa;QACf,GAAGtC;IACL;IAEA,iGAAiG;IACjG,uIAAuI;IACvI,MAAM2C,UAAU1D,QAAQwB,SAAS;QAAEmC,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAAC7D,YAAY;QACpEsD;IACF;IACA,MAAMQ,iBAAiBC,YACrB7D,YACE,UAAYkD,aAAa,QACzB,IAAMV,WAER3B;IAGF,OAAO,IAAIiD,QAAc,CAACC;YAqBpBrD,cACCA;QArBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAMsD,WAAW;gBAKftD;YAJA,IAAI8B,UAAU;YACdA,WAAW;YACXJ,QAAQ6B,GAAG,CAAC,UAAUD;YACtB5B,QAAQ6B,GAAG,CAAC,WAAWD;aACvBtD,eAAAA,KAAKwD,MAAM,cAAXxD,mCAAAA,aAAayD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIT,eAAeE,aAAaF;YAChCI,QAAQ1B,KAAK;YACb,MAAMY;YACN,MAAM,EAAE3B,OAAOuD,UAAU,EAAE,GAAG,MAAMlE,UAAUM;YAC9C,MAAML,QAAQiE,YAAY,mBAAmB;YAC7C,MAAMjE,QAAQiE,YAAY,aAAa;YACvC,MAAMA,WAAWxC,KAAK;YACtBkC;QACF;QACA3B,QAAQkC,IAAI,CAAC,UAAUN;QACvB5B,QAAQkC,IAAI,CAAC,WAAWN;QACxB,KAAItD,eAAAA,KAAKwD,MAAM,cAAXxD,mCAAAA,aAAa6D,OAAO,EAAEP;cACrBtD,gBAAAA,KAAKwD,MAAM,cAAXxD,oCAAAA,cAAa8D,gBAAgB,CAAC,SAASR,UAAU;YAAEM,MAAM;QAAK;IACrE;AACF"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "sensemaking",
-  "version": "0.23.1",
-  "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
+  "version": "0.24.1",
+  "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step.",
   "keywords": [
     "markdown",
     "frontmatter",
@@ -81,12 +81,13 @@
     "version": "tsds version"
   },
   "dependencies": {
-    "@huggingface/tokenizers": "^0.1.3",
+    "@huggingface/tokenizers": "^0.2.0",
     "franc-min": "^6.2.0",
     "install-module-linked": "^2.0.0",
     "markdown-it": "^15.0.1",
     "markdown-it-footnote": "^4.0.0",
     "markdown-it-task-lists": "^2.1.1",
+    "stemmer": "^2.0.1",
     "tinypool": "^2.1.2",
     "yaml": "^2.9.0"
   },

package/schema.json

@@ -15,6 +15,11 @@
       "enum": [1, 2, 3, 4, 5],
       "description": "Config format version. Older versions are auto-migrated (and the file rewritten) on load; a version newer than this sense build supports makes it exit with an error rather than misinterpret the file. Omit to default to 1."
     },
+    "root": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Optional markdown tree root. Relative paths resolve against the directory containing sense.config.json; omitted means that directory, preserving colocated-config behavior. Preset globs and stored paths are relative to this root, while .sense state remains beside the config."
+    },
     "presets": {
       "type": "object",
       "description": "File selection, index-time, and named search defaults. A preset is self-contained: no inheritance between presets, and no ordering -- presets may overlap freely (they are views, not partitions), and a preset whose globs match nothing is valid. A file is indexed iff any preset's include/exclude covers it (union across presets); `default` is used when a command names no preset, and must be declared. A file is embedded when the top-level `embed` block names a model and some covering preset's `signals` include `vectors`. `sense init` generates `default` plus one `large` example.",
@@ -28,7 +33,7 @@
             "type": "array",
             "items": { "type": "string" },
             "minItems": 1,
-            "description": "Glob patterns (globby syntax), resolved relative to this config file's own directory -- never the invocation cwd. Required for every preset, including `default`."
+            "description": "Glob patterns (globby syntax), resolved relative to the configured root (or the config file's directory when root is omitted) -- never the invocation cwd. Required for every preset, including `default`."
           },
           "exclude": {
             "type": "array",
@@ -104,7 +109,7 @@
     "store": {
       "type": "string",
       "enum": ["sqlite", "duckdb", "turso"],
-      "description": "Backing store engine, defaulting to \"sqlite\" (zero-dependency, Node's built-in SQLite). \"duckdb\" and \"turso\" are experimental: the first command that opens such a tree installs its engine package on its own (@duckdb/node-api, a one-time native download of about 110 MB; @tursodatabase/database, much smaller). The same commands and table names run on all three; what does not port is FTS5 syntax. Under duckdb and turso, `search` text and raw MATCH reject FTS5's prefix (foo*), boolean (AND/OR/NOT), NEAR, initial-token (^), and column-filter (title:foo) operators with a named error (STORE_CAPABILITY_MISSING) that says how to rephrase or set \"store\" to \"sqlite\"; bare words and quoted phrases work on all three. Raw SQL is a per-store dialect: the tables are portable everywhere, but sqlite's FTS5 MATCH, snippet(), and bm25() do not run under duckdb (its fts extension has its own functions) or turso (Tantivy fts_match/fts_score), so saved queries written in FTS5 syntax are sqlite dialect. The has/basename/segment functions are registered on sqlite and duckdb but not turso, whose client cannot register SQL functions at all. `sense watch` requires sqlite; on duckdb and turso it errors at start naming the fix. Each store keeps its own cache file (.sense/cache.db, .sense/cache.duckdb, .sense/cache.turso.db); switching stores rebuilds the index rather than migrating it."
+      "description": "Backing store engine, defaulting to \"sqlite\" (zero-dependency, Node's built-in SQLite). \"duckdb\" and \"turso\" are experimental: the first command that opens such a tree installs its engine package on its own (@duckdb/node-api, a one-time native download of about 110 MB; @tursodatabase/database, much smaller). The same commands and table names run on all three; what does not port is FTS5 syntax. Under duckdb and turso, `search` text and raw MATCH reject FTS5's prefix (foo*), boolean (AND/OR/NOT), NEAR, initial-token (^), and column-filter (title:foo) operators with a named error (STORE_CAPABILITY_MISSING) that says how to rephrase or set \"store\" to \"sqlite\"; bare words and quoted phrases work on all three. Raw SQL is a per-store dialect: the tables are portable everywhere, but sqlite's FTS5 MATCH, snippet(), and bm25() do not run under duckdb (its fts extension has its own functions) or turso (Tantivy fts_match/fts_score), so saved queries written in FTS5 syntax are sqlite dialect. The has/basename/segment functions: has and basename run on all three (turso rewrites them into portable SQL rather than registering them); segment runs on sqlite and duckdb only, so a query calling segment under turso fails with a named error (STORE_CAPABILITY_MISSING) saying to rephrase or set \"store\" to sqlite. `sense watch` runs on all three; under duckdb and turso, which lock the cache file per connection, a concurrent command waits out the watcher's current cycle instead of failing. Each store keeps its own cache file (.sense/cache.db, .sense/cache.duckdb, .sense/cache.turso.db); switching stores rebuilds the index rather than migrating it."
     },
     "queries": {
       "type": "object",

package/skills/sense/EXAMPLES.md

@@ -11,9 +11,9 @@ sense search "pricing OR billing OR invoicing" --k 10 --format json
 ```json
 [
   { "path": "notes/pricing-model.md", "title": "Pricing model",
-    "summary": "Tiered per-seat pricing; floor and discount rules", "hit": "«Pricing» floor for annual…", "via": "match+link", "score": 0.0333, "lines": null },
+    "summary": "Tiered per-seat pricing; floor and discount rules", "snippets": ["«Pricing» floor for annual…"], "via": "match+link", "score": 0.0333, "lines": null },
   { "path": "notes/renewal-playbook.md", "title": "Renewal playbook",
-    "summary": "Renewal sequence and owners", "hit": "…«billing» contact confirms the PO…", "via": "match", "score": 0.0313, "lines": "L81-140" }
+    "summary": "Renewal sequence and owners", "snippets": ["…«billing» contact confirms the PO…"], "via": "match", "score": 0.0313, "lines": "L81-140" }
 ]
 ```
 
@@ -79,7 +79,7 @@ sense search "children dying from poor nutrition" --k 3 --format json
 ```json
 [
   { "path": "notes/malnutrition-outcomes.md", "title": "Malnutrition outcomes",
-    "summary": "Stunting and mortality by region", "hit": null, "via": "vector", "score": 0.0167, "similarity": 0.61, "lines": "L14-52" }
+    "summary": "Stunting and mortality by region", "snippets": [], "via": "vector", "score": 0.0167, "similarity": 0.61, "lines": "L14-52" }
 ]
 ```
 

package/skills/sense/SKILL.md

@@ -7,7 +7,7 @@ description: "Query a markdown tree with the sense CLI: filter notes by frontmat
 
 SQL over a markdown tree, kept fresh by a filesystem check on every query. Every file becomes rows in `frontmatter` (one column per key, plus `path`/`_mtime`/`_ctime`/`_size`/`_rank`/`_parse_error`; `_ctime` is filesystem birthtime, which a clone or copy resets just like `_mtime`), `content` (`title`, `summary`, `text`, `path`; an FTS5 index on the default store, with machine-written `title_seg`/`summary_seg`/`text_seg` sidecars used for matching Chinese, Japanese, Thai, Khmer, Lao, and Burmese text, not for reading), `links` (`src`, `target`, `dst`, `embed`; `NULL` dst = dead link; `embed` 1 for `![[...]]` embeds, 0 for links; one row per distinct written target and kind with alias and anchor stripped, so `[[Foo]]` and `[[Foo|alias]]` are one row while `[[Foo]]` and `[[notes/Foo]]` are two rows that can share a `dst`, and a target both linked and embedded is a row of each kind; extraction matches Obsidian's own graph: comments, code, and link-syntax text yield no rows, `[[#Anchor]]` is a self-edge, a frontmatter value that is exactly `[[X]]` is a link, and a basename collision resolves to the linking note itself, else the shortest path), `tags` (`path`, `tag`: frontmatter and inline `#tags` merged and deduplicated; nested tags stored full, so `book/scifi` matches `tag = 'book' OR tag LIKE 'book/%'`), `sections` (heading outline with line ranges and token estimates), and `preset_files` (`path`, `preset`: which presets cover which files). Features add their own storage; `map` and `status` report which are on.
 
-**Stores.** The config's `store` key picks the backing store: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso` (the first command that opens such a tree installs that engine's package on its own: `@duckdb/node-api`, a one-time native download of about 110 MB, or the much smaller `@tursodatabase/database`). The tables, `?` placeholders, quoted identifiers, and the `scope` binding are the same on all three, so ordinary frontmatter SQL ports as written. Two things do not port. First, FTS5: `content` is an FTS5 table on sqlite and a plain table on the other two, so hand-written `MATCH`, `snippet()`, `bm25()`, and sqlite's date-function forms run only on sqlite (duckdb has its own fts functions and date syntax; turso has Tantivy's `fts_match`/`fts_score`), and `search` text under duckdb and turso rejects FTS5's prefix (`foo*`), boolean (`AND`/`OR`/`NOT`), `NEAR`, initial-token (`^`), and column-filter (`title:foo`) operators with a named error (STORE_CAPABILITY_MISSING) that says how to rephrase or set `store` to `sqlite`; bare words and quoted phrases work on all three. Second, the `has`/`basename`/`segment` functions are registered on sqlite and duckdb but not turso, whose client cannot register SQL functions at all, so a query calling them under turso fails with `no such function`. `sense watch` runs on all three; under duckdb and turso, which lock the cache file per connection, a concurrent command waits out the watcher's current cycle instead of failing.
+**Stores.** The config's `store` key picks the backing store: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso` (the first command that opens such a tree installs that engine's package on its own: `@duckdb/node-api`, a one-time native download of about 110 MB, or the much smaller `@tursodatabase/database`). The tables, `?` placeholders, quoted identifiers, and the `scope` binding are the same on all three, so ordinary frontmatter SQL ports as written. Two things do not port. First, FTS5: `content` is an FTS5 table on sqlite and a plain table on the other two, so hand-written `MATCH`, `snippet()`, `bm25()`, and sqlite's date-function forms run only on sqlite (duckdb has its own fts functions and date syntax; turso has Tantivy's `fts_match`/`fts_score`), and `search` text under duckdb and turso rejects FTS5's prefix (`foo*`), boolean (`AND`/`OR`/`NOT`), `NEAR`, initial-token (`^`), and column-filter (`title:foo`) operators with a named error (STORE_CAPABILITY_MISSING) that says how to rephrase or set `store` to `sqlite`; bare words and quoted phrases work on all three. Second, of the `has`/`basename`/`segment` functions, `has` and `basename` run on all three (turso rewrites them into portable SQL rather than registering them), while `segment` runs on sqlite and duckdb only, so a query calling `segment` under turso fails with a named error (STORE_CAPABILITY_MISSING) saying to rephrase or set `store` to sqlite. `sense watch` runs on all three; under duckdb and turso, which lock the cache file per connection, a concurrent command waits out the watcher's current cycle instead of failing.
 
 ## What each tool is for
 
@@ -31,7 +31,7 @@ Output defaults to a table, built for humans; `--format json` returns the same r
 |---|---|---|---|
 | `words` | the text itself (BM25) | every literal occurrence: identifiers, error strings, names, exact phrases; deterministic | anything phrased differently, e.g. a note saying "compensation floor" for the query "minimum pay" |
 | `links` | connections authors wrote | what the tree's own structure treats as related: context the matching text never restates | notes nobody linked |
-| `vectors` | per-chunk meaning-vectors | notes sharing no words with the query: paraphrase, the category for an instance, a concept restated | proving presence. A vector row cannot show the words occur anywhere; `hit` can |
+| `vectors` | per-chunk meaning-vectors | notes sharing no words with the query: paraphrase, the category for an instance, a concept restated | proving presence. A vector row cannot show the words occur anywhere; `snippets` can |
 
 Each signal in the map carries a weight (`{"words": 1, "links": 1, "vectors": 1}` is the default when the key is omitted entirely); presence turns a signal on, and the number scales its share of the fused ranking. Equal weight (1 for everything) is what every number elsewhere in this doc describes. A weight above 1 pulls the fused ranking toward that signal's own ordering: measured on nfcorpus with the default static model and on MIRACL zh with an HTTP encoder, equal-weight fusion helps English nfcorpus but ranks the encoder's MIRACL-zh results below its own cosine-only ranking (benchmark/reports/2026-08-27-embedding-model-selection.md, weight-sweep table). There is no single weight that is right for both, so a preset that leans on a strong encoder is a candidate for a higher `vectors` weight, checked against that table rather than assumed.
 
@@ -54,13 +54,13 @@ sense --list | status | download
 ```
 
 - Terms pass verbatim to FTS5 MATCH. Bare words AND-join (one absent word means zero rows), so write `OR` yourself when you want any-word matching; double-quote punctuated terms (`"customer-facing"`, `"founder's"`); invalid syntax is an error, not a rewrite. The same rules apply to search commands you write into subagent briefs.
-- When a search misses, both sides have levers. Lexical: OR-in synonyms and concrete instances (the index only knows the words in the files; a note about a specific tool rarely names its category), raise `--k` (a row costs tens of tokens), widen the scope (`--preset`, or `--include` for an ad-hoc glob). Vector: restate the concept in different words. Vectors rank the meaning of the whole query text, so a rephrase moves them even when every literal word still misses. Then pivot through the nearest hit with `related <path>` to walk its meaning-neighbourhood. Vector-only rows in a miss are themselves evidence: `similarity` and the snippet say whether the concept exists in the tree under other words. Each widening adds candidates and dilutes ranking, so the noise trade-off runs both ways.
+- When a search misses, both sides have levers. Lexical: OR-in synonyms and concrete instances (the index only knows the words in the files; a note about a specific tool rarely names its category), raise `--k` (a row costs tens of tokens), widen the scope (`--preset`, or `--include` for an ad-hoc glob). Vector: restate the concept in different words. Vectors rank the meaning of the whole query text, so a rephrase moves them even when every literal word still misses. Then pivot through the nearest hit with `related <path>` to walk its meaning-neighbourhood. Vector-only rows in a miss are leads rather than lexical evidence: use `similarity` and the `lines` range to decide what to read; their `snippets` list is empty because the query words did not match. Each widening adds candidates and dilutes ranking, so the noise trade-off runs both ways.
 - A frontmatter query enumerates its matches deterministically; search ranks by term overlap, so results shift as phrasing shifts. Trade-off: a query needs a known field, search doesn't.
-- The `via` column says what produced each row: `match` (words hit), `link` (connected to notes that hit), `vector` (near in meaning), and combinations. The `lines` column, when set, points at the section that earned the row (the best-matching chunk on vector rows, the term cluster's section on large lexical notes) and is a direct `Read` range; null means the whole note is the reference.
+- The `via` column says what produced each row: `match` (words hit), `link` (connected to notes that hit), `vector` (near in meaning), and combinations. The `lines` column, when set, points at the section that earned the row (the best-matching chunk on vector rows, the term cluster's section on large lexical notes) and is a direct `Read` range; null means the whole note is the reference. A row's `snippets` holds the marked passages that earned the lexical match (`[]` on a row that never contained the terms); `--snippet-char-limit` (default 80, the passage's rendered length, marks and ellipses counted) and `--snippet-count-limit` (default 1, passages per note) bound them, on `search` and a saved search alike.
 - Scope is one vocabulary shared by `search`, `map`, `peek`, `path`, and `related`: bare command uses the config's `default` preset; `--preset <name>` picks another (unknown names error, listing what's declared); `--include <glob>` and `--exclude <glob>` are ad-hoc globs for one command, each overriding its own side of the preset, so one does not clear the other; `--no-exclude` drops the preset's `exclude` for one command, the only way to widen past it without editing config (it widens the query scope, not the index: a file no preset covers is never indexed). `--where` takes any SQL condition against frontmatter alias `f`, not only field equality: `"f.status = 'active' AND has(f.tags, 'x')"`, `"datetime(f.created) >= datetime(?)"`. There is no whole-index flag: a broad `default` preset, or a declared `all` preset (`include ["**/*"]`), is the whole tree. `sense status` shows every preset with its coverage. `sql` scopes differently: it runs over the whole index by default, and `--preset <name>` *binds* the scope as a temporary `scope(path)` table your statement joins, rather than filtering behind the query's back (`JOIN scope ON scope."path" = f.path`). Naming a preset without joining `scope` is a usage error, since it would return everything while reading as scoped. Without the flag, join `preset_files` directly, which is the same coverage under a preset name you write into the SQL.
 - `score` is a rank-fusion value: it ranks rows within one result set and is not comparable across queries, not a relevance magnitude. It encodes how many signals fired and at what rank, so a perfect lexical hit and a weak vector-only hit can read the same number. With vectors active, rows carry `similarity`: the cosine (-1 to 1) of the query against that file's best-matching chunk (the same chunk the `lines` range points at). It orders vector evidence within a result set; the range it spans depends on the corpus and the embedding model, and compresses on small trees, where even a nonsense query has a moderately near neighbour somewhere. Compare similarities within a result set rather than against a fixed cutoff carried between trees.
 - Vector rankings read the whole chunk, boilerplate included, so on a tree whose notes are mostly one template with a line of unique text (a directory of plugins, people, or assets), the shared scaffolding dominates every vector and `related` returns near-ties at the top of the range for any seed. Two signs, both visible in the output already: the top `similarity` values sit within a hair of each other, and the same few notes come back for unrelated seeds. Compare neighbour lists across two unlike seeds when a tree looks like this; matching lists mean the ranking is reading the template, not the content, and `search` over the distinguishing words is the answer instead.
-- Absence evidence lives in the labels: a preset whose `signals` exclude `vectors` (or a tree with no `embed` block) returns 0 rows when the words are nowhere in it. Default `search` always returns up to `k` rows (nearest-neighbour search has a nearest neighbour for any input), so a result of only `via: vector` rows IS the absence signal for the words themselves; `similarity` and the snippet are the evidence for judging whether a vector row is a real conceptual hit.
+- Absence evidence lives in the labels: a preset whose `signals` exclude `vectors` (or a tree with no `embed` block) returns 0 rows when the words are nowhere in it. Default `search` always returns up to `k` rows (nearest-neighbour search has a nearest neighbour for any input), so a result of only `via: vector` rows is the absence signal for the words themselves. Judge whether a vector row is a useful conceptual lead from `similarity`, then read its `lines` range; it has no lexical snippet.
 - A `queries` entry names the verb it runs, mirroring the two commands: `"dead-links": { "sql": "SELECT src, target FROM links WHERE dst IS NULL AND lower(target) NOT GLOB '*.[a-z0-9]*'" }` runs as `sense dead-links`, and `"hot": { "search": "pricing OR billing", "preset": "raw", "k": 20 }` runs as `sense hot` with its settings baked in, so repeat runs need no flags. An invocation-level `--preset`, `--k`, or `--where` overrides a saved search's value; `--list` labels each entry `(sql)` or `(search)`.
 - Running an entry is how it is validated: a typo'd column, stale SQL, bad FTS5 syntax or an unknown preset errors and exits nonzero. A parameterised entry validates with any argument, since SQL is prepared before parameters bind (`sense by-tag zzz` reports `no such column` if the column is wrong, `(0 rows)` if it is right). To sweep a whole config after editing it, read the exit code: 0 ran, 2 means it needs parameters (re-run it with any argument to validate the SQL), anything else is broken.
 
@@ -112,7 +112,7 @@ WHERE a.dst = ? AND b.dst IS NOT NULL AND b.dst <> a.dst;
 - `content MATCH` only works against the fts5 table by its own name, never through an alias or a view: `FROM content c ... WHERE c MATCH 'x'` fails with `no such column: c`. This is why `--preset` binds a table to join rather than shadowing the tables.
 - `content MATCH` takes FTS5 syntax: `a OR b`, `"phrase"`, `pref*`, `NEAR(a b, 5)`, `summary: term`. Stemmed; markdown stripped at index time. Double-quote any term with punctuation. Bare `customer-facing` errors (`-` reads as a column filter), bare apostrophes are syntax errors: write `"customer-facing"`, `"founder's"`. This is the default sqlite store's grammar: under `duckdb` and `turso` the operator forms (`a OR b`, `pref*`, `NEAR`, `^`, column filters) are a named error naming the rephrase, and `MATCH` itself does not run (see Stores).
 - A language written without word spaces (Chinese, Japanese, Thai, Khmer, Lao, Burmese) is indexed per grapheme and searched as an ordered grapheme phrase against the `_seg` sidecar columns: substring semantics, what `grep` gives, a query matches wherever its exact text occurs, including inside a longer run (`京都` matches `东京都政府`, correctly, because it's there at position 2), and needs no minimum length. No decision is needed for these languages. Hand-written SQL is not rewritten for you, so a raw `content MATCH '数据库'` finds nothing: write `content MATCH segment(?)` and bind the terms. `segment()` returns text with no such run unchanged, so it is safe to leave in a query whatever the tree's language.
-- Rank with `ORDER BY bm25(content, 10.0, 5.0, 1.0)` (title > summary > body); the full form `bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)` mirrors the same weights onto the `_seg` sidecars, so a title hit found through `title_seg` ranks like one found through `title` (the three-weight form still runs, FTS5 defaults unnamed columns to 1.0, but ranks a sidecar match at body weight). Excerpt with `snippet(content, 2, '«', '»', '…', 10)`, naming the `text` column explicitly: `-1` means best column, which can surface a machine-spaced sidecar as the excerpt (`search` itself always names column 2). snippet() re-tokenizes each matched doc and its cost grows superlinearly with doc size, measured ~10 s per query on a tree holding one 1 MB note. `search` bounds this itself (docs past 16 KB get an equivalent excerpt another way); in hand-written SQL, guard it: `CASE WHEN length(text) <= 16384 THEN snippet(...) END`, or select `title`/`summary` instead of an excerpt.
+- Rank with `ORDER BY bm25(content, 10.0, 5.0, 1.0)` (title > summary > body); the full form `bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)` mirrors the same weights onto the `_seg` sidecars, so a title hit found through `title_seg` ranks like one found through `title` (the three-weight form still runs, FTS5 defaults unnamed columns to 1.0, but ranks a sidecar match at body weight). In hand-written SQLite SQL, `snippet(content, 2, '«', '»', '…', 10)` names the authored `text` column explicitly; `-1` can surface a machine-spaced sidecar instead. `search` does not call SQLite's `snippet()`: it computes bounded passages in shared code for every store. A raw SQL `snippet()` re-tokenizes its matched document, so guard large text with `CASE WHEN length(text) <= 16384 THEN snippet(...) END`, or select `title`/`summary`. Treat historical timing as diagnostic until a current identical-work sitting replaces it.
 - Select `content.title`/`content.summary` (always exist, empty when absent) rather than `f.title`/`f.summary` (discovered columns; error on trees that never declare them).
 - Frontmatter values keep their YAML type: strings are TEXT, whole numbers and booleans are INTEGER (`true` stores as 1, so `WHERE flag = 1` matches and `WHERE flag = 'true'` matches nothing), fractions are REAL, lists and maps are JSON text. On the `duckdb` store the discovered columns are VARIANT: homogeneous keys, which are nearly all of them, compare identically, but a numeric predicate against a key that holds numbers in some notes and text in others raises a comparison error where sqlite orders by storage class silently. `map` prints the observed type per field, and a field showing two types (`integer,text`) has drifted across notes. A list key written with no items (`tags:` above a bare `-`) is a list holding one null, stored as the JSON text `[null]`: `IS NULL` does not find it (the column holds a string), `json_each` yields one empty member per such row, and `map` counts it as covered because the key is present. `has(tags, 'x')` reads it correctly as no match. To separate written-but-empty from absent, compare against the text: `WHERE tags = '[null]'`.
 - **Dead links need the attachment filter.** `dst IS NULL` alone is not "broken link": a wikilink to anything that is not markdown (`[[Board.base]]`, `![[Pasted image.png]]`, `[[spec.pdf]]`) can never resolve, because sense indexes markdown and resolution only tries the exact path or `+.md`. Those are out of the index's universe, not broken. On a 1,400-note Obsidian vault the unfiltered query returns 143 rows where 14 are real. Exclude anything carrying a file extension, as in the recipe above, and widen the exclusion if your notes have dotted titles (`[[Node.js]]` carries one too, so a stricter list, `'*.png'`, `'*.pdf'`, `'*.base'`, and whatever else your vault attaches, is safer on a tree whose titles use dots). Scope it with `preset_files` as well: template and skill files are full of `[[Note Name]]` examples that are deliberately unresolved.

package/skills/sense-setup/EXAMPLES.md

@@ -106,3 +106,24 @@ sense peek "Plugins/dataview.md"                # outline + links before reading
 ```
 
 Presets earn their place only when a tree has parts deserving different treatment; a tree that is one kind of thing needs none of the vocabulary above. On a big vault, raise `default`'s `k` and read `lines` ranges instead of whole files, or start from the starter's `large` preset.
+
+## E. One vault, separate consumers
+
+Keep each consumer's configuration and cache under its own configuration directory while indexing one shared vault. `root` is relative to this config file, so saved SQL continues to use stable vault-relative paths.
+
+`~/.config/obsidian/tasks/sense.config.json`:
+
+```json
+{
+  "version": 5,
+  "root": "../../../.obsidian",
+  "presets": {
+    "default": { "include": ["vault/tasks/**/*.md"], "k": 20 }
+  },
+  "queries": {
+    "active": { "sql": "SELECT path, title FROM frontmatter WHERE path LIKE 'vault/tasks/%' AND status = 'active' ORDER BY path" }
+  }
+}
+```
+
+The database is `~/.config/obsidian/tasks/.sense/cache.db`; the indexed row remains `vault/tasks/...`, never a path back out through the configuration directory. A `general/` sibling can point at the same root with different presets and a separate `.sense/` cache.

package/skills/sense-setup/SKILL.md

@@ -9,9 +9,9 @@ Querying an existing tree is the `sense` skill. This one covers making a tree: i
 
 ## Setup
 
-- `npm install -g sensemaking`, then `sense init` at the tree root writes `sense.config.json`: two presets (`default`, and `large` showing what a big tree tunes) and an `embed` block naming the model. The model fetches once per machine at the first vector search (progress on stderr); `sense download` prefetches it instead where that timing matters (CI, air-gapped setup). Config discovery walks up from cwd; `--config <path>` overrides.
-- **Backing store.** The config's `store` key: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`, whose engine package the first command that opens such a tree installs on its own (`@duckdb/node-api`, a one-time native download of about 110 MB; `@tursodatabase/database`, much smaller). The same commands and tables run on all three. Two things do not port, and each one decides a tree. **FTS5 syntax:** under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix, boolean, `NEAR`, initial-token and column-filter operators with a named error, and sqlite's FTS5 SQL (`MATCH`, `snippet()`, `bm25()`) does not run, so saved queries written in that syntax are sqlite dialect; a tree whose saved queries or search vocabulary depend on FTS5 operators stays on `sqlite`. **SQL functions:** `has`/`basename`/`segment` are registered on `sqlite` and `duckdb` but not `turso`, whose client cannot register them at all, so a tree whose queries call them stays off `turso`. `sense watch` runs on all three; `duckdb` and `turso` lock their cache file per connection, so a concurrent command waits out the watcher's current cycle instead of failing. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. Indexing speed is the other axis, and it does not follow from any of the above: sqlite builds a cold index fastest and turso slowest, by a wide margin on a large tree, and no setting closes that gap, since turso's engine costs more per write and more again to maintain each index. What turso buys instead is concurrent access, non-blocking I/O and encryption, so reach for it for those rather than for speed. Per-store figures: BENCHMARKING.md in the sensemaking repo.
-- Globs resolve relative to the config file, never the cwd.
+- `npm install -g sensemaking`, then `sense init` at the tree root writes `sense.config.json`: two presets (`default`, and `large` showing what a big tree tunes) and an `embed` block naming the model. The model fetches once per machine at the first vector search (progress on stderr); `sense download` prefetches it instead where that timing matters (CI, air-gapped setup). Config discovery walks up from cwd; `--config <path>` overrides. A top-level `root` can point the config at another markdown tree: it resolves relative to the config directory, makes all globs and stored paths tree-relative, while `.sense/` remains beside the config. Use this when one vault needs independent consumer-specific presets, queries, and caches.
+- **Backing store.** The config's `store` key: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`, whose engine package the first command that opens such a tree installs on its own (`@duckdb/node-api`, a one-time native download of about 110 MB; `@tursodatabase/database`, much smaller). The same commands and tables run on all three. Two things do not port, and each one decides a tree. **FTS5 syntax:** under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix, boolean, `NEAR`, initial-token and column-filter operators with a named error, and sqlite's FTS5 SQL (`MATCH`, `snippet()`, `bm25()`) does not run, so saved queries written in that syntax are sqlite dialect; a tree whose saved queries or search vocabulary depend on FTS5 operators stays on `sqlite`. **SQL functions:** `has`/`basename` run on all three (`turso` rewrites them into portable SQL rather than registering them); `segment` runs on `sqlite` and `duckdb` only, so a tree whose queries call `segment` stays off `turso`. `sense watch` runs on all three; `duckdb` and `turso` lock their cache file per connection, so a concurrent command waits out the watcher's current cycle instead of failing. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. Speed is a separate axis and must come from a current identical-work measurement. Historical per-store rows can include different ranked candidates and are diagnostics, not an overall store leaderboard. The cache remains an ordinary database file, so a tree indexed under duckdb is readable by DuckDB's ecosystem, while turso brings concurrent access, non-blocking I/O and encryption. Pick for the capabilities you need, then review the current evidence in BENCHMARKING.md.
+- Globs resolve relative to `root` when present, otherwise the config file directory; never the cwd.
 - `sense status` and `sense map` show each preset's coverage (files matched, embedded count), so what a config actually indexes is always visible in output. A config edit that changes coverage rebuilds the cache and names the preset that caused it on stderr.
 
 ## Presets