sensemaking 0.21.1 → 0.22.0

package/dist/cjs/store/types.d.ts

@@ -1,7 +1,7 @@
 import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
-export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "watch-concurrency", "lexical", "vectors", "sql-functions"];
+export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "lexical", "vectors", "sql-functions"];
 export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;

package/dist/cjs/store/types.js

@@ -13,7 +13,6 @@ Object.defineProperty(exports, "CAPABILITY_NAMES", {
 var CAPABILITY_NAMES = [
     'phrases',
     'snippets',
-    'watch-concurrency',
     'lexical',
     'vectors',
     'sql-functions'

package/dist/cjs/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n  // Inserts rows whose path cannot already exist in `table` (no ON CONFLICT needed), through a\n  // faster append-only path. Optional: sqlite/turso omit it and every row goes through the upsert.\n  insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n  // Inserts rows whose path cannot already exist in `table` (no ON CONFLICT needed), through a\n  // faster append-only path. Optional: sqlite/turso omit it and every row goes through the upsert.\n  insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB"}

package/dist/cjs/text/segment.js

@@ -35,10 +35,9 @@ function hasUnspacedRun(text) {
     return HAS_RUN.test(text);
 }
 // Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.
-// Built once (construction cost amortizes) and used for both index and query splitting.
-var GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'grapheme'
-});
+// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run
+// never segments at all, so no command pays it at module load.
+var graphemeSegmenter;
 // unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions
 // into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.
 var PUNCTUATION = RegExp("[\\u0021-\\u0023\\u0025-\\u002A\\u002C-\\u002F\\u003A-\\u003B\\u003F-\\u0040\\u005B-\\u005D\\u005F\\u007B\\u007D\\u00A1\\u00A7\\u00AB\\u00B6-\\u00B7\\u00BB\\u00BF\\u037E\\u0387\\u055A-\\u055F\\u0589-\\u058A\\u05BE\\u05C0\\u05C3\\u05C6\\u05F3-\\u05F4\\u0609-\\u060A\\u060C-\\u060D\\u061B\\u061D-\\u061F\\u066A-\\u066D\\u06D4\\u0700-\\u070D\\u07F7-\\u07F9\\u0830-\\u083E\\u085E\\u0964-\\u0965\\u0970\\u09FD\\u0A76\\u0AF0\\u0C77\\u0C84\\u0DF4\\u0E4F\\u0E5A-\\u0E5B\\u0F04-\\u0F12\\u0F14\\u0F3A-\\u0F3D\\u0F85\\u0FD0-\\u0FD4\\u0FD9-\\u0FDA\\u104A-\\u104F\\u10FB\\u1360-\\u1368\\u1400\\u166E\\u169B-\\u169C\\u16EB-\\u16ED\\u1735-\\u1736\\u17D4-\\u17D6\\u17D8-\\u17DA\\u1800-\\u180A\\u1944-\\u1945\\u1A1E-\\u1A1F\\u1AA0-\\u1AA6\\u1AA8-\\u1AAD\\u1B4E-\\u1B4F\\u1B5A-\\u1B60\\u1B7D-\\u1B7F\\u1BFC-\\u1BFF\\u1C3B-\\u1C3F\\u1C7E-\\u1C7F\\u1CC0-\\u1CC7\\u1CD3\\u2010-\\u2027\\u2030-\\u2043\\u2045-\\u2051\\u2053-\\u205E\\u207D-\\u207E\\u208D-\\u208E\\u2308-\\u230B\\u2329-\\u232A\\u2768-\\u2775\\u27C5-\\u27C6\\u27E6-\\u27EF\\u2983-\\u2998\\u29D8-\\u29DB\\u29FC-\\u29FD\\u2CF9-\\u2CFC\\u2CFE-\\u2CFF\\u2D70\\u2E00-\\u2E2E\\u2E30-\\u2E4F\\u2E52-\\u2E5D\\u3001-\\u3003\\u3008-\\u3011\\u3014-\\u301F\\u3030\\u303D\\u30A0\\u30FB\\uA4FE-\\uA4FF\\uA60D-\\uA60F\\uA673\\uA67E\\uA6F2-\\uA6F7\\uA874-\\uA877\\uA8CE-\\uA8CF\\uA8F8-\\uA8FA\\uA8FC\\uA92E-\\uA92F\\uA95F\\uA9C1-\\uA9CD\\uA9DE-\\uA9DF\\uAA5C-\\uAA5F\\uAADE-\\uAADF\\uAAF0-\\uAAF1\\uABEB\\uFD3E-\\uFD3F\\uFE10-\\uFE19\\uFE30-\\uFE52\\uFE54-\\uFE61\\uFE63\\uFE68\\uFE6A-\\uFE6B\\uFF01-\\uFF03\\uFF05-\\uFF0A\\uFF0C-\\uFF0F\\uFF1A-\\uFF1B\\uFF1F-\\uFF20\\uFF3B-\\uFF3D\\uFF3F\\uFF5B\\uFF5D\\uFF5F-\\uFF65\\u{10100}-\\u{10102}\\u{1039F}\\u{103D0}\\u{1056F}\\u{10857}\\u{1091F}\\u{1093F}\\u{10A50}-\\u{10A58}\\u{10A7F}\\u{10AF0}-\\u{10AF6}\\u{10B39}-\\u{10B3F}\\u{10B99}-\\u{10B9C}\\u{10D6E}\\u{10EAD}\\u{10ED0}\\u{10F55}-\\u{10F59}\\u{10F86}-\\u{10F89}\\u{11047}-\\u{1104D}\\u{110BB}-\\u{110BC}\\u{110BE}-\\u{110C1}\\u{11140}-\\u{11143}\\u{11174}-\\u{11175}\\u{111C5}-\\u{111C8}\\u{111CD}\\u{111DB}\\u{111DD}-\\u{111DF}\\u{11238}-\\u{1123D}\\u{112A9}\\u{113D4}-\\u{113D5}\\u{113D7}-\\u{113D8}\\u{1144B}-\\u{1144F}\\u{1145A}-\\u{1145B}\\u{1145D}\\u{114C6}\\u{115C1}-\\u{115D7}\\u{11641}-\\u{11643}\\u{11660}-\\u{1166C}\\u{116B9}\\u{1173C}-\\u{1173E}\\u{1183B}\\u{11944}-\\u{11946}\\u{119E2}\\u{11A3F}-\\u{11A46}\\u{11A9A}-\\u{11A9C}\\u{11A9E}-\\u{11AA2}\\u{11B00}-\\u{11B09}\\u{11BE1}\\u{11C41}-\\u{11C45}\\u{11C70}-\\u{11C71}\\u{11EF7}-\\u{11EF8}\\u{11F43}-\\u{11F4F}\\u{11FFF}\\u{12470}-\\u{12474}\\u{12FF1}-\\u{12FF2}\\u{16A6E}-\\u{16A6F}\\u{16AF5}\\u{16B37}-\\u{16B3B}\\u{16B44}\\u{16D6D}-\\u{16D6F}\\u{16E97}-\\u{16E9A}\\u{16FE2}\\u{1BC9F}\\u{1DA87}-\\u{1DA8B}\\u{1E5FF}\\u{1E95E}-\\u{1E95F}]", "u");
@@ -46,7 +45,10 @@ var PUNCTUATION = RegExp("[\\u0021-\\u0023\\u0025-\\u002A\\u002C-\\u002F\\u003A-
 // are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.
 var BARRIER = 'ꟷ';
 function graphemes(run) {
-    return Array.from(GRAPHEME_SEGMENTER.segment(run), function(s) {
+    graphemeSegmenter !== null && graphemeSegmenter !== void 0 ? graphemeSegmenter : graphemeSegmenter = new Intl.Segmenter(undefined, {
+        granularity: 'grapheme'
+    });
+    return Array.from(graphemeSegmenter.segment(run), function(s) {
         return s.segment;
     });
 }

package/dist/cjs/text/segment.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built once (construction cost amortizes) and used for both index and query splitting.\nconst GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  return Array.from(GRAPHEME_SEGMENTER.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","hasUnspacedRun","segmentField","segmentMatch","RUN","RegExp","HAS_RUN","text","test","GRAPHEME_SEGMENTER","Intl","Segmenter","undefined","granularity","PUNCTUATION","BARRIER","graphemes","run","Array","from","segment","s","splitOnPunctuation","groups","g","push","length","filter","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;;;;;;;;;;;;QACrEA;eAAAA;;QAQGC;eAAAA;;QA8BAC;eAAAA;;QA6BAC;eAAAA;;;AAnET,IAAMH,mBAAmB;AAChC,2FAA2F;AAC3F,gDAAgD;AAChD,IAAMI,MAAM,IAAIC,OAAO,AAAC,QAAwB,OAAjBL,kBAAiB,gBAAc;AAC9D,IAAMM,UAAU,IAAID,OAAO,AAAC,IAAoB,OAAjBL,kBAAiB,MAAI;AAI7C,SAASC,eAAeM,IAAY;IACzC,OAAOD,QAAQE,IAAI,CAACD;AACtB;AACA,6FAA6F;AAC7F,wFAAwF;AACxF,IAAME,qBAAqB,IAAIC,KAAKC,SAAS,CAACC,WAAW;IAAEC,aAAa;AAAW;AACnF,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,IAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5B,OAAOC,MAAMC,IAAI,CAACV,mBAAmBW,OAAO,CAACH,MAAM,SAACI;eAAMA,EAAED,OAAO;;AACrE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBL,GAAW;IACrC,IAAMM,SAAqB;QAAC,EAAE;KAAC;QAC1B,kCAAA,2BAAA;;QAAL,QAAK,YAAWP,UAAUC,yBAArB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMO,IAAN;YACH,IAAIV,YAAYN,IAAI,CAACgB,IAAID,OAAOE,IAAI,CAAC,EAAE;iBAClCF,MAAM,CAACA,OAAOG,MAAM,GAAG,EAAE,CAACD,IAAI,CAACD;QACtC;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,OAAOI,MAAM,CAAC,SAACH;eAAMA,EAAEE,MAAM,GAAG;;AACzC;AAIO,SAASxB,aAAaK,IAAY;IACvC,IAAI,CAACD,QAAQE,IAAI,CAACD,OAAO,OAAO;IAChC,IAAMqB,MAAMrB,KAAKsB,OAAO,CAACzB,KAAK,SAACa;QAC7B,IAAMa,OAAOR,mBAAmBL,KAC7Bc,GAAG,CAAC,SAACP;mBAAMA,EAAEQ,IAAI,CAAC;WAClBA,IAAI,CAAC,AAAC,IAAW,OAARjB,SAAQ;QACpB,OAAO,AAAC,IAAce,OAAXf,SAAQ,KAAWA,OAARe,MAAK,KAAW,OAARf,SAAQ;IACxC;IACA,OAAOa,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,IAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAASnB,GAAW,EAAEoB,YAAoB;IACjD,OAAOf,mBAAmBL,KACvBc,GAAG,CAAC,SAACP;eAAM,AAAC,GAAiBA,OAAfa,cAAwD,OAAzCb,EAAEE,MAAM,GAAG,IAAI,AAAC,IAAe,OAAZF,EAAEQ,IAAI,CAAC,MAAK,OAAKR,CAAC,CAAC,EAAE;OACrEQ,IAAI,CAAC;AACV;AAIO,SAAS7B,aAAamC,KAAa;IACxC,IAAI,CAAChC,QAAQE,IAAI,CAAC8B,QAAQ,OAAOA;IACjC,IAAIV,MAAM;IACV,IAAIW,SAAS;IACb,IAAMC,SAASF,MAAMG,KAAK,CAACrC,MAAM,2CAA2C;IAC5E,IAAK,IAAIsC,IAAI,GAAGA,IAAIF,OAAOd,MAAM,EAAEgB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;gBACV,kCAAA,2BAAA;;gBAAL,QAAK,YAAYF,MAAM,CAACE,EAAE,qBAArB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,KAAN;oBAAuB,IAAIA,OAAO,KAAKJ,SAAS,CAACA;;;gBAAjD;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YACLX,OAAOY,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVX,OAAOY,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,IAAME,IAAIhB,IAAIiB,KAAK,CAACX;QACpB,IAAIU,GAAG;YACLhB,MAAM,AAAC,GAA0BgB,OAAxBhB,IAAIkB,KAAK,CAAC,GAAGF,EAAEG,KAAK,GAAWX,OAAPQ,CAAC,CAAC,EAAE,EAA8C,OAA3CR,SAASI,MAAM,CAACE,EAAE,EAAE,AAAC,GAASE,OAAPA,CAAC,CAAC,EAAE,EAAQ,OAALA,CAAC,CAAC,EAAE,EAAC;QAC7E,OAAO;YACLhB,OAAOQ,SAASI,MAAM,CAACE,EAAE,EAAEP;QAC7B;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","hasUnspacedRun","segmentField","segmentMatch","RUN","RegExp","HAS_RUN","text","test","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","run","Intl","Segmenter","undefined","granularity","Array","from","segment","s","splitOnPunctuation","groups","g","push","length","filter","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;;;;;;;;;;;;QACrEA;eAAAA;;QAQGC;eAAAA;;QAgCAC;eAAAA;;QA6BAC;eAAAA;;;AArET,IAAMH,mBAAmB;AAChC,2FAA2F;AAC3F,gDAAgD;AAChD,IAAMI,MAAM,IAAIC,OAAO,AAAC,QAAwB,OAAjBL,kBAAiB,gBAAc;AAC9D,IAAMM,UAAU,IAAID,OAAO,AAAC,IAAoB,OAAjBL,kBAAiB,MAAI;AAI7C,SAASC,eAAeM,IAAY;IACzC,OAAOD,QAAQE,IAAI,CAACD;AACtB;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAIE;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,IAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5BJ,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAIK,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOC,MAAMC,IAAI,CAACV,kBAAkBW,OAAO,CAACP,MAAM,SAACQ;eAAMA,EAAED,OAAO;;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBT,GAAW;IACrC,IAAMU,SAAqB;QAAC,EAAE;KAAC;QAC1B,kCAAA,2BAAA;;QAAL,QAAK,YAAWX,UAAUC,yBAArB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMW,IAAN;YACH,IAAId,YAAYF,IAAI,CAACgB,IAAID,OAAOE,IAAI,CAAC,EAAE;iBAClCF,MAAM,CAACA,OAAOG,MAAM,GAAG,EAAE,CAACD,IAAI,CAACD;QACtC;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,OAAOI,MAAM,CAAC,SAACH;eAAMA,EAAEE,MAAM,GAAG;;AACzC;AAIO,SAASxB,aAAaK,IAAY;IACvC,IAAI,CAACD,QAAQE,IAAI,CAACD,OAAO,OAAO;IAChC,IAAMqB,MAAMrB,KAAKsB,OAAO,CAACzB,KAAK,SAACS;QAC7B,IAAMiB,OAAOR,mBAAmBT,KAC7BkB,GAAG,CAAC,SAACP;mBAAMA,EAAEQ,IAAI,CAAC;WAClBA,IAAI,CAAC,AAAC,IAAW,OAARrB,SAAQ;QACpB,OAAO,AAAC,IAAcmB,OAAXnB,SAAQ,KAAWA,OAARmB,MAAK,KAAW,OAARnB,SAAQ;IACxC;IACA,OAAOiB,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,IAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAASvB,GAAW,EAAEwB,YAAoB;IACjD,OAAOf,mBAAmBT,KACvBkB,GAAG,CAAC,SAACP;eAAM,AAAC,GAAiBA,OAAfa,cAAwD,OAAzCb,EAAEE,MAAM,GAAG,IAAI,AAAC,IAAe,OAAZF,EAAEQ,IAAI,CAAC,MAAK,OAAKR,CAAC,CAAC,EAAE;OACrEQ,IAAI,CAAC;AACV;AAIO,SAAS7B,aAAamC,KAAa;IACxC,IAAI,CAAChC,QAAQE,IAAI,CAAC8B,QAAQ,OAAOA;IACjC,IAAIV,MAAM;IACV,IAAIW,SAAS;IACb,IAAMC,SAASF,MAAMG,KAAK,CAACrC,MAAM,2CAA2C;IAC5E,IAAK,IAAIsC,IAAI,GAAGA,IAAIF,OAAOd,MAAM,EAAEgB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;gBACV,kCAAA,2BAAA;;gBAAL,QAAK,YAAYF,MAAM,CAACE,EAAE,qBAArB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,KAAN;oBAAuB,IAAIA,OAAO,KAAKJ,SAAS,CAACA;;;gBAAjD;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YACLX,OAAOY,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVX,OAAOY,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,IAAME,IAAIhB,IAAIiB,KAAK,CAACX;QACpB,IAAIU,GAAG;YACLhB,MAAM,AAAC,GAA0BgB,OAAxBhB,IAAIkB,KAAK,CAAC,GAAGF,EAAEG,KAAK,GAAWX,OAAPQ,CAAC,CAAC,EAAE,EAA8C,OAA3CR,SAASI,MAAM,CAACE,EAAE,EAAE,AAAC,GAASE,OAAPA,CAAC,CAAC,EAAE,EAAQ,OAALA,CAAC,CAAC,EAAE,EAAC;QAC7E,OAAO;YACLhB,OAAOQ,SAASI,MAAM,CAACE,EAAE,EAAEP;QAC7B;IACF;IACA,OAAOP;AACT"}

package/dist/cjs/watch.js

@@ -144,93 +144,186 @@ var HEARTBEAT_INTERVAL_MS = 5000;
 var STALE_HEARTBEAT_MS = 15000;
 function runWatch(_0) {
     return _async_to_generator(function(cfg) {
-        var opts, _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs, onEvent, debounceMs, heartbeatIntervalMs, _ref, store, builder, dbPath, initialWarnings, initialParsed, baseDir, existingHeartbeat, age, _tmp, stopping, touchHeartbeat, debounceTimer, inFlight, scheduleReconcile, watcher, heartbeatTimer;
+        var opts, _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs, onEvent, debounceMs, heartbeatIntervalMs, baseDir, _ref, initialStore, dbPath, initialWarnings, initialParsed, existingHeartbeat, age, initialTotal, _tmp, stopping, running, queuedAlways, tick, startCycle, requestCycle, debounceTimer, scheduleReconcile, watcher, heartbeatTimer;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
                     opts = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : {};
-                    (0, _indexts1.requireWatchConcurrency)(cfg);
                     onEvent = (_opts_onEvent = opts.onEvent) !== null && _opts_onEvent !== void 0 ? _opts_onEvent : function() {};
                     debounceMs = (_opts_debounceMs = opts.debounceMs) !== null && _opts_debounceMs !== void 0 ? _opts_debounceMs : DEBOUNCE_MS;
                     heartbeatIntervalMs = (_opts_heartbeatIntervalMs = opts.heartbeatIntervalMs) !== null && _opts_heartbeatIntervalMs !== void 0 ? _opts_heartbeatIntervalMs : HEARTBEAT_INTERVAL_MS;
+                    baseDir = cfg.baseDir;
                     return [
                         4,
-                        (0, _indexts1.openStoreForWatch)(cfg)
+                        (0, _indexts1.openStore)(cfg)
                     ];
                 case 1:
-                    _ref = _state.sent(), store = _ref.store, builder = _ref.builder, dbPath = _ref.dbPath, initialWarnings = _ref.warnings, initialParsed = _ref.parsed;
-                    baseDir = cfg.baseDir;
+                    _ref = _state.sent(), initialStore = _ref.store, dbPath = _ref.dbPath, initialWarnings = _ref.warnings, initialParsed = _ref.parsed;
                     return [
                         4,
-                        (0, _indexts1.getMeta)(store, 'watch_heartbeat')
+                        (0, _indexts1.getMeta)(initialStore, 'watch_heartbeat')
                     ];
                 case 2:
                     existingHeartbeat = _state.sent();
                     if (!(existingHeartbeat && !opts.force)) return [
                         3,
-                        5
+                        4
                     ];
                     age = Date.now() - Date.parse(existingHeartbeat);
                     if (!(age >= 0 && age < STALE_HEARTBEAT_MS)) return [
                         3,
-                        5
+                        4
                     ];
                     return [
                         4,
-                        builder.close()
+                        initialStore.close()
                     ];
                 case 3:
-                    _state.sent();
-                    return [
-                        4,
-                        store.close()
-                    ];
-                case 4:
                     _state.sent();
                     throw new _errorsts.SenseError('WATCH_ACTIVE', "another watcher appears active (heartbeat ".concat(Math.round(age / 1000), "s ago); use --force to override"));
-                case 5:
-                    onEvent({
-                        type: 'started',
-                        baseDir: baseDir,
-                        dbPath: dbPath
-                    });
+                case 4:
                     if (!(initialWarnings.length > 0 || initialParsed > 0)) return [
                         3,
-                        7
+                        6
                     ];
-                    _tmp = {
-                        type: 'reconciled',
-                        parsed: initialParsed
-                    };
                     return [
                         4,
-                        (0, _indexts1.docCount)(store)
+                        (0, _indexts1.docCount)(initialStore)
+                    ];
+                case 5:
+                    _tmp = _state.sent();
+                    return [
+                        3,
+                        7
                     ];
                 case 6:
-                    onEvent.apply(void 0, [
-                        (_tmp.total = _state.sent(), _tmp.warnings = initialWarnings, _tmp)
-                    ]);
+                    _tmp = 0;
                     _state.label = 7;
                 case 7:
+                    initialTotal = _tmp;
+                    return [
+                        4,
+                        (0, _indexts1.setMeta)(initialStore, 'watch_heartbeat', new Date().toISOString())
+                    ];
+                case 8:
+                    _state.sent();
+                    return [
+                        4,
+                        (0, _indexts1.setMeta)(initialStore, 'watch_pid', String(process.pid))
+                    ];
+                case 9:
+                    _state.sent();
+                    return [
+                        4,
+                        initialStore.close()
+                    ];
+                case 10:
+                    _state.sent();
+                    onEvent({
+                        type: 'started',
+                        baseDir: baseDir,
+                        dbPath: dbPath
+                    });
+                    if (initialWarnings.length > 0 || initialParsed > 0) {
+                        onEvent({
+                            type: 'reconciled',
+                            parsed: initialParsed,
+                            total: initialTotal,
+                            warnings: initialWarnings
+                        });
+                    }
                     stopping = false;
-                    touchHeartbeat = function touchHeartbeat() {
+                    // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is
+                    // coalesced into a single rerun instead of opening a second, overlapping connection.
+                    running = null;
+                    queuedAlways = null;
+                    tick = function tick(alwaysEmit) {
                         return _async_to_generator(function() {
+                            var _ref, store, parsed, warnings, _tmp, err;
                             return _ts_generator(this, function(_state) {
                                 switch(_state.label){
                                     case 0:
+                                        _state.trys.push([
+                                            0,
+                                            10,
+                                            ,
+                                            11
+                                        ]);
                                         return [
                                             4,
-                                            (0, _indexts1.setMeta)(store, 'watch_heartbeat', new Date().toISOString())
+                                            (0, _indexts1.openStore)(cfg)
                                         ];
                                     case 1:
+                                        _ref = _state.sent(), store = _ref.store, parsed = _ref.parsed, warnings = _ref.warnings;
+                                        _state.label = 2;
+                                    case 2:
+                                        _state.trys.push([
+                                            2,
+                                            ,
+                                            7,
+                                            9
+                                        ]);
+                                        return [
+                                            4,
+                                            (0, _indexts1.setMeta)(store, 'watch_heartbeat', new Date().toISOString())
+                                        ];
+                                    case 3:
                                         _state.sent();
                                         return [
                                             4,
                                             (0, _indexts1.setMeta)(store, 'watch_pid', String(process.pid))
                                         ];
-                                    case 2:
+                                    case 4:
                                         _state.sent();
+                                        if (!(alwaysEmit || parsed > 0 || warnings.length > 0)) return [
+                                            3,
+                                            6
+                                        ];
+                                        _tmp = {
+                                            type: 'reconciled',
+                                            parsed: parsed
+                                        };
+                                        return [
+                                            4,
+                                            (0, _indexts1.docCount)(store)
+                                        ];
+                                    case 5:
+                                        onEvent.apply(void 0, [
+                                            (_tmp.total = _state.sent(), _tmp.warnings = warnings, _tmp)
+                                        ]);
+                                        _state.label = 6;
+                                    case 6:
+                                        return [
+                                            3,
+                                            9
+                                        ];
+                                    case 7:
+                                        return [
+                                            4,
+                                            store.close()
+                                        ];
+                                    case 8:
+                                        _state.sent();
+                                        return [
+                                            7
+                                        ];
+                                    case 9:
+                                        return [
+                                            3,
+                                            11
+                                        ];
+                                    case 10:
+                                        err = _state.sent();
+                                        onEvent({
+                                            type: 'reconcile-error',
+                                            message: err.message
+                                        });
+                                        return [
+                                            3,
+                                            11
+                                        ];
+                                    case 11:
                                         return [
                                             2
                                         ];
@@ -238,77 +331,30 @@ function runWatch(_0) {
                             });
                         })();
                     };
-                    return [
-                        4,
-                        touchHeartbeat()
-                    ];
-                case 8:
-                    _state.sent();
+                    startCycle = function startCycle1(alwaysEmit) {
+                        running = tick(alwaysEmit).then(function() {
+                            running = null;
+                            if (queuedAlways !== null && !stopping) {
+                                var again = queuedAlways;
+                                queuedAlways = null;
+                                startCycle(again);
+                            }
+                        });
+                    };
+                    requestCycle = function requestCycle(alwaysEmit) {
+                        if (stopping) return;
+                        if (running) {
+                            queuedAlways = queuedAlways || alwaysEmit;
+                            return;
+                        }
+                        startCycle(alwaysEmit);
+                    };
                     debounceTimer = null;
-                    // A reconcile that has already started owns the store, and on a bulk reparse a live worker
-                    // pool as well. Shutdown waits on this rather than closing the connection underneath it.
-                    inFlight = null;
                     scheduleReconcile = function scheduleReconcile() {
                         if (debounceTimer) clearTimeout(debounceTimer);
                         debounceTimer = setTimeout(function() {
                             debounceTimer = null;
-                            inFlight = function() {
-                                return _async_to_generator(function() {
-                                    var _ref, parsed, warnings, _tmp, err;
-                                    return _ts_generator(this, function(_state) {
-                                        switch(_state.label){
-                                            case 0:
-                                                _state.trys.push([
-                                                    0,
-                                                    3,
-                                                    4,
-                                                    5
-                                                ]);
-                                                return [
-                                                    4,
-                                                    builder.build()
-                                                ];
-                                            case 1:
-                                                _ref = _state.sent(), parsed = _ref.parsed, warnings = _ref.warnings;
-                                                _tmp = {
-                                                    type: 'reconciled',
-                                                    parsed: parsed
-                                                };
-                                                return [
-                                                    4,
-                                                    (0, _indexts1.docCount)(store)
-                                                ];
-                                            case 2:
-                                                onEvent.apply(void 0, [
-                                                    (_tmp.total = _state.sent(), _tmp.warnings = warnings, _tmp)
-                                                ]);
-                                                return [
-                                                    3,
-                                                    5
-                                                ];
-                                            case 3:
-                                                err = _state.sent();
-                                                onEvent({
-                                                    type: 'reconcile-error',
-                                                    message: err.message
-                                                });
-                                                return [
-                                                    3,
-                                                    5
-                                                ];
-                                            case 4:
-                                                inFlight = null;
-                                                return [
-                                                    7
-                                                ];
-                                            case 5:
-                                                return [
-                                                    2
-                                                ];
-                                        }
-                                    });
-                                })();
-                            }();
+                            requestCycle(true);
                         }, debounceMs);
                     };
                     // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with
@@ -319,7 +365,16 @@ function runWatch(_0) {
                         if (typeof filename === 'string' && filename.startsWith(_indexts.STATE_DIR)) return;
                         scheduleReconcile();
                     });
-                    heartbeatTimer = setInterval((0, _guardedtickts.guardedTick)(touchHeartbeat, function() {
+                    heartbeatTimer = setInterval((0, _guardedtickts.guardedTick)(function() {
+                        return _async_to_generator(function() {
+                            return _ts_generator(this, function(_state) {
+                                return [
+                                    2,
+                                    requestCycle(false)
+                                ];
+                            });
+                        })();
+                    }, function() {
                         return stopping;
                     }), heartbeatIntervalMs);
                     return [
@@ -330,7 +385,7 @@ function runWatch(_0) {
                             // unregistered here too so a second runWatch call in the same process starts clean.
                             var shutdown = function shutdown1() {
                                 return _async_to_generator(function() {
-                                    var _opts_signal;
+                                    var _opts_signal, _ref, finalStore;
                                     return _ts_generator(this, function(_state) {
                                         switch(_state.label){
                                             case 0:
@@ -346,31 +401,31 @@ function runWatch(_0) {
                                                 watcher.close();
                                                 return [
                                                     4,
-                                                    inFlight
+                                                    running
                                                 ];
                                             case 1:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    builder.close()
+                                                    (0, _indexts1.openStore)(cfg)
                                                 ];
                                             case 2:
-                                                _state.sent();
+                                                _ref = _state.sent(), finalStore = _ref.store;
                                                 return [
                                                     4,
-                                                    (0, _indexts1.setMeta)(store, 'watch_heartbeat', null)
+                                                    (0, _indexts1.setMeta)(finalStore, 'watch_heartbeat', null)
                                                 ];
                                             case 3:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    (0, _indexts1.setMeta)(store, 'watch_pid', null)
+                                                    (0, _indexts1.setMeta)(finalStore, 'watch_pid', null)
                                                 ];
                                             case 4:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    store.close()
+                                                    finalStore.close()
                                                 ];
                                             case 5:
                                                 _state.sent();

package/dist/cjs/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStoreForWatch, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, builder, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStoreForWatch(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await builder.close();\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await builder.build();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await builder.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","builder","dbPath","initialWarnings","initialParsed","baseDir","existingHeartbeat","age","stopping","touchHeartbeat","debounceTimer","inFlight","scheduleReconcile","watcher","heartbeatTimer","requireWatchConcurrency","openStoreForWatch","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","type","length","docCount","total","setMeta","toISOString","String","process","pid","clearTimeout","setTimeout","err","build","message","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;uBAEP;wBACC;6BACC;wBAC2D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvF,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAcpB,SAAeH;wCAASI,GAAmB;YAAEC,MAElCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBAC+E,MAA7EC,OAAOC,SAASC,QAAkBC,iBAAyBC,eAC7DC,SAEAC,mBAEEC,WAaJC,UACEC,gBAMFC,eAGAC,UACEC,mBAmBAC,SAIAC;;;;;oBAzD4ClB,OAAAA,oEAAqB,CAAC;oBACxEmB,IAAAA,iCAAuB,EAACpB;oBAClBE,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBAC6B;;wBAAMuB,IAAAA,2BAAiB,EAACrB;;;oBAAxB,OAAA,eAA7EK,QAA6E,KAA7EA,OAAOC,UAAsE,KAAtEA,SAASC,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDc,UAAmCb,gBAAkB,KAA1Bc;oBACrDb,UAAUV,IAAIU,OAAO;oBAED;;wBAAMc,IAAAA,iBAAO,EAACnB,OAAO;;;oBAAzCM,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACV,KAAKwB,KAAK,AAAD,GAA/Bd;;;;oBACIC,MAAMc,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACjB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMb,kBAAiB,GAAnCa;;;;oBACF;;wBAAMN,QAAQuB,KAAK;;;oBAAnB;oBACA;;wBAAMxB,MAAMwB,KAAK;;;oBAAjB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACpB,MAAM,OAAM;;oBAI7GV,QAAQ;wBAAE+B,MAAM;wBAAWvB,SAAAA;wBAASH,QAAAA;oBAAO;yBACvCC,CAAAA,gBAAgB0B,MAAM,GAAG,KAAKzB,gBAAgB,CAAA,GAA9CD;;;;;wBACQyB,MAAM;wBAAcV,QAAQd;;oBAAsB;;wBAAM0B,IAAAA,kBAAQ,EAAC9B;;;oBAA3EH;8BAAqDkC,QAAO,oBAAuBd,WAAUd;;;;oBAG3FK,WAAW;oBACTC,iBAAiB;;;;;wCACrB;;4CAAMuB,IAAAA,iBAAO,EAAChC,OAAO,mBAAmB,IAAIqB,OAAOY,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAAChC,OAAO,aAAakC,OAAOC,QAAQC,GAAG;;;wCAApD;;;;;;wBACF;;oBACA;;wBAAM3B;;;oBAAN;oBAEIC,gBAAuC;oBAC3C,2FAA2F;oBAC3F,yFAAyF;oBACrFC,WAAiC;oBAC/BC,oBAAoB;wBACxB,IAAIF,eAAe2B,aAAa3B;wBAChCA,gBAAgB4B,WAAW;4BACzB5B,gBAAgB;4BAChBC,WAAW,AAAC;;wCAEqB,MAArBO,QAAQD,gBAETsB;;;;;;;;;;gDAFsB;;oDAAMtC,QAAQuC,KAAK;;;gDAAnB,OAAA,eAArBtB,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;oDACNW,MAAM;oDAAcV,QAAAA;;gDAAe;;oDAAMY,IAAAA,kBAAQ,EAAC9B;;;gDAA5DH;0DAAsCkC,QAAO,oBAAuBd,WAAAA;;;;;;;gDAC7DsB;gDACP1C,QAAQ;oDAAE+B,MAAM;oDAAmBa,SAAS,AAACF,IAAcE,OAAO;gDAAC;;;;;;gDAEnE9B,WAAW;;;;;;;;;;gCAEf;;wBACF,GAAGb;oBACL;oBAEA,iGAAiG;oBACjG,uIAAuI;oBACjIe,UAAU6B,IAAAA,aAAO,EAACrC,SAAS;wBAAEsC,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpEnC;oBACF;oBACME,iBAAiBkC,YACrBC,IAAAA,0BAAW,EAACxC,gBAAgB;+BAAMD;wBAClCT;oBAGF;;wBAAO,IAAImD,QAAc,SAACC;gCAqBpBvD,cACCA;4BArBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAMwD,WAAW;;wCAKfxD;;;;gDAJA,IAAIY,UAAU;;;gDACdA,WAAW;gDACX2B,QAAQkB,GAAG,CAAC,UAAUD;gDACtBjB,QAAQkB,GAAG,CAAC,WAAWD;iDACvBxD,eAAAA,KAAK0D,MAAM,cAAX1D,mCAAAA,aAAa2D,mBAAmB,CAAC,SAASH;gDAC1CI,cAAc1C;gDACd,IAAIJ,eAAe2B,aAAa3B;gDAChCG,QAAQW,KAAK;gDACb;;oDAAMb;;;gDAAN;gDACA;;oDAAMV,QAAQuB,KAAK;;;gDAAnB;gDACA;;oDAAMQ,IAAAA,iBAAO,EAAChC,OAAO,mBAAmB;;;gDAAxC;gDACA;;oDAAMgC,IAAAA,iBAAO,EAAChC,OAAO,aAAa;;;gDAAlC;gDACA;;oDAAMA,MAAMwB,KAAK;;;gDAAjB;gDACA2B;;;;;;gCACF;;4BACAhB,QAAQsB,IAAI,CAAC,UAAUL;4BACvBjB,QAAQsB,IAAI,CAAC,WAAWL;4BACxB,KAAIxD,eAAAA,KAAK0D,MAAM,cAAX1D,mCAAAA,aAAa8D,OAAO,EAAEN;kCACrBxD,gBAAAA,KAAK0D,MAAM,cAAX1D,oCAAAA,cAAa+D,gBAAgB,CAAC,SAASP,UAAU;gCAAEK,MAAM;4BAAK;wBACrE;;;;IACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. duckdb/turso hand the cache file to\n// one process at a time, so every cycle opens, reconciles, writes the heartbeat/pid rows, and closes.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const baseDir = cfg.baseDir;\n\n  const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const existingHeartbeat = await getMeta(initialStore, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await initialStore.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  // total is read before close: an event fires only once nothing is held open again.\n  const initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n  await setMeta(initialStore, 'watch_heartbeat', new Date().toISOString());\n  await setMeta(initialStore, 'watch_pid', String(process.pid));\n  await initialStore.close();\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n  // coalesced into a single rerun instead of opening a second, overlapping connection.\n  let running: Promise<void> | null = null;\n  let queuedAlways: boolean | null = null;\n\n  const tick = async (alwaysEmit: boolean): Promise<void> => {\n    try {\n      const { store, parsed, warnings } = await openStore(cfg);\n      try {\n        await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n        await setMeta(store, 'watch_pid', String(process.pid));\n        if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        }\n      } finally {\n        await store.close();\n      }\n    } catch (err) {\n      onEvent({ type: 'reconcile-error', message: (err as Error).message });\n    }\n  };\n\n  const startCycle = (alwaysEmit: boolean) => {\n    running = tick(alwaysEmit).then(() => {\n      running = null;\n      if (queuedAlways !== null && !stopping) {\n        const again = queuedAlways;\n        queuedAlways = null;\n        startCycle(again);\n      }\n    });\n  };\n\n  const requestCycle = (alwaysEmit: boolean) => {\n    if (stopping) return;\n    if (running) {\n      queuedAlways = queuedAlways || alwaysEmit;\n      return;\n    }\n    startCycle(alwaysEmit);\n  };\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      requestCycle(true);\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(\n      async () => requestCycle(false),\n      () => stopping\n    ),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await running;\n      const { store: finalStore } = await openStore(cfg);\n      await setMeta(finalStore, 'watch_heartbeat', null);\n      await setMeta(finalStore, 'watch_pid', null);\n      await finalStore.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","baseDir","initialStore","dbPath","initialWarnings","initialParsed","existingHeartbeat","age","initialTotal","stopping","running","queuedAlways","tick","startCycle","requestCycle","debounceTimer","scheduleReconcile","watcher","heartbeatTimer","openStore","store","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","length","docCount","setMeta","toISOString","String","process","pid","type","total","alwaysEmit","err","message","then","again","clearTimeout","setTimeout","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","finalStore","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAyBsBA;;;eAAAA;;;sBAzBW;uBAEP;wBACC;6BACC;wBAC0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtD,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAepB,SAAeH;wCAASI,GAAmB;YAAEC,MAClCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBACAC,SAEoF,MAA3EC,cAAcC,QAAkBC,iBAAyBC,eAClEC,mBAEEC,KAQFC,oBAUFC,UAGAC,SACAC,cAEEC,MAiBAC,YAWAC,cASFC,eACEC,mBAUAC,SAIAC;;;;;oBArF4CrB,OAAAA,oEAAqB,CAAC;oBAClEC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBAClDO,UAAUL,IAAIK,OAAO;oBAE+D;;wBAAMkB,IAAAA,mBAAS,EAACvB;;;oBAAhB,OAAA,eAA3EM,eAA2E,KAAlFkB,OAAqBjB,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDiB,UAAmChB,gBAAkB,KAA1BiB;oBACtC;;wBAAMC,IAAAA,iBAAO,EAACrB,cAAc;;;oBAAhDI,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACT,KAAK2B,KAAK,AAAD,GAA/BlB;;;;oBACIC,MAAMkB,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACrB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMZ,kBAAiB,GAAnCY;;;;oBACF;;wBAAML,aAAa0B,KAAK;;;oBAAxB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACxB,MAAM,OAAM;;yBAKxFH,CAAAA,gBAAgB4B,MAAM,GAAG,KAAK3B,gBAAgB,CAAA,GAA9CD;;;;oBAAkD;;wBAAM6B,IAAAA,kBAAQ,EAAC/B;;;2BAAf;;;;;;2BAA+B;;;oBAAhGM;oBACN;;wBAAM0B,IAAAA,iBAAO,EAAChC,cAAc,mBAAmB,IAAIuB,OAAOU,WAAW;;;oBAArE;oBACA;;wBAAMD,IAAAA,iBAAO,EAAChC,cAAc,aAAakC,OAAOC,QAAQC,GAAG;;;oBAA3D;oBACA;;wBAAMpC,aAAa0B,KAAK;;;oBAAxB;oBAEA9B,QAAQ;wBAAEyC,MAAM;wBAAWtC,SAAAA;wBAASE,QAAAA;oBAAO;oBAC3C,IAAIC,gBAAgB4B,MAAM,GAAG,KAAK3B,gBAAgB,GAAG;wBACnDP,QAAQ;4BAAEyC,MAAM;4BAAcjB,QAAQjB;4BAAemC,OAAOhC;4BAAca,UAAUjB;wBAAgB;oBACtG;oBAEIK,WAAW;oBACf,6FAA6F;oBAC7F,qFAAqF;oBACjFC,UAAgC;oBAChCC,eAA+B;oBAE7BC,OAAO,cAAO6B;;gCAEoB,MAA5BrB,OAAOE,QAAQD,gBAUhBqB;;;;;;;;;;wCAV6B;;4CAAMvB,IAAAA,mBAAS,EAACvB;;;wCAAhB,OAAA,eAA5BwB,QAA4B,KAA5BA,OAAOE,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;;;;;;;;wCAErB;;4CAAMa,IAAAA,iBAAO,EAACd,OAAO,mBAAmB,IAAIK,OAAOU,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAACd,OAAO,aAAagB,OAAOC,QAAQC,GAAG;;;wCAApD;6CACIG,CAAAA,cAAcnB,SAAS,KAAKD,SAASW,MAAM,GAAG,CAAA,GAA9CS;;;;;4CACQF,MAAM;4CAAcjB,QAAAA;;wCAAe;;4CAAMW,IAAAA,kBAAQ,EAACb;;;wCAA5DtB;kDAAsC0C,QAAO,oBAAuBnB,WAAAA;;;;;;;;;wCAGtE;;4CAAMD,MAAMQ,KAAK;;;wCAAjB;;;;;;;;;;wCAEKc;wCACP5C,QAAQ;4CAAEyC,MAAM;4CAAmBI,SAAS,AAACD,IAAcC,OAAO;wCAAC;;;;;;;;;;;wBAEvE;;oBAEM9B,aAAa,qBAAC4B;wBAClB/B,UAAUE,KAAK6B,YAAYG,IAAI,CAAC;4BAC9BlC,UAAU;4BACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;gCACtC,IAAMoC,QAAQlC;gCACdA,eAAe;gCACfE,WAAWgC;4BACb;wBACF;oBACF;oBAEM/B,eAAe,sBAAC2B;wBACpB,IAAIhC,UAAU;wBACd,IAAIC,SAAS;4BACXC,eAAeA,gBAAgB8B;4BAC/B;wBACF;wBACA5B,WAAW4B;oBACb;oBAEI1B,gBAAuC;oBACrCC,oBAAoB;wBACxB,IAAID,eAAe+B,aAAa/B;wBAChCA,gBAAgBgC,WAAW;4BACzBhC,gBAAgB;4BAChBD,aAAa;wBACf,GAAGf;oBACL;oBAEA,iGAAiG;oBACjG,uIAAuI;oBACjIkB,UAAU+B,IAAAA,aAAO,EAAC/C,SAAS;wBAAEgD,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpErC;oBACF;oBACME,iBAAiBoC,YACrBC,IAAAA,0BAAW,EACT;;;;;oCAAYzC,aAAa;;;;uBACzB;+BAAML;wBAERT;oBAGF;;wBAAO,IAAIwD,QAAc,SAACC;gCAqBpB5D,cACCA;4BArBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAM6D,WAAW;;wCAKf7D,cAK8B,MAAf8D;;;;gDATf,IAAIlD,UAAU;;;gDACdA,WAAW;gDACX4B,QAAQuB,GAAG,CAAC,UAAUF;gDACtBrB,QAAQuB,GAAG,CAAC,WAAWF;iDACvB7D,eAAAA,KAAKgE,MAAM,cAAXhE,mCAAAA,aAAaiE,mBAAmB,CAAC,SAASJ;gDAC1CK,cAAc7C;gDACd,IAAIH,eAAe+B,aAAa/B;gDAChCE,QAAQW,KAAK;gDACb;;oDAAMlB;;;gDAAN;gDAC8B;;oDAAMS,IAAAA,mBAAS,EAACvB;;;gDAAhB,OAAA,eAAf+D,aAAe,KAAtBvC;gDACR;;oDAAMc,IAAAA,iBAAO,EAACyB,YAAY,mBAAmB;;;gDAA7C;gDACA;;oDAAMzB,IAAAA,iBAAO,EAACyB,YAAY,aAAa;;;gDAAvC;gDACA;;oDAAMA,WAAW/B,KAAK;;;gDAAtB;gDACA6B;;;;;;gCACF;;4BACApB,QAAQ2B,IAAI,CAAC,UAAUN;4BACvBrB,QAAQ2B,IAAI,CAAC,WAAWN;4BACxB,KAAI7D,eAAAA,KAAKgE,MAAM,cAAXhE,mCAAAA,aAAaoE,OAAO,EAAEP;kCACrB7D,gBAAAA,KAAKgE,MAAM,cAAXhE,oCAAAA,cAAaqE,gBAAgB,CAAC,SAASR,UAAU;gCAAEM,MAAM;4BAAK;wBACrE;;;;IACF"}

package/dist/cjs/workers/parse.d.cts

@@ -11,6 +11,7 @@ export type ParseTaskResult = {
     ok: true;
     doc: ParsedDoc;
     warnings: string[];
+    parseMs: number;
 } | {
     ok: false;
     error: WorkerErrorPayload;

package/dist/cjs/workers/parse.d.ts

@@ -11,6 +11,7 @@ export type ParseTaskResult = {
     ok: true;
     doc: ParsedDoc;
     warnings: string[];
+    parseMs: number;
 } | {
     ok: false;
     error: WorkerErrorPayload;

package/dist/cjs/workers/parse.js

@@ -27,11 +27,14 @@ var selected = _indexts.FEATURES.filter(function(feature) {
 });
 function parseTask(file) {
     try {
+        var start = process.hrtime.bigint();
         var _parseFile = (0, _indexts1.parseFile)(file, (0, _reparsets.featuresForFile)(selected, cfg, file), cfg), doc = _parseFile.doc, warnings = _parseFile.warnings;
+        var parseMs = Number(process.hrtime.bigint() - start) / 1e6;
         return {
             ok: true,
             doc: doc,
-            warnings: warnings
+            warnings: warnings,
+            parseMs: parseMs
         };
     } catch (err) {
         return {