sensemaking 0.18.4 → 0.19.1

package/dist/cjs/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event\n  // whose filename the platform could not resolve (null, which fs.watch does deliver under\n  // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs\n  // less than missing a real edit. That is why the guard cannot promise zero reconciles, only\n  // that an identified state-dir write is never one of them.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","initialWarnings","initialParsed","baseDir","existingHeartbeat","age","stopping","touchHeartbeat","debounceTimer","inFlight","scheduleReconcile","watcher","heartbeatTimer","requireWatchConcurrency","openStore","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","type","length","docCount","total","setMeta","toISOString","String","process","pid","clearTimeout","setTimeout","err","reconcile","message","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;uBAEP;wBACC;6BACC;wBACmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE/E,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAcpB,SAAeH;wCAASI,GAAmB;YAAEC,MAElCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBACsE,MAApEC,OAAOC,QAAkBC,iBAAyBC,eACpDC,SAEAC,mBAEEC,WAYJC,UACEC,gBAMFC,eAGAC,UACEC,mBAsBAC,SAIAC;;;;;oBA3D4CjB,OAAAA,oEAAqB,CAAC;oBACxEkB,IAAAA,iCAAuB,EAACnB;oBAClBE,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBACoB;;wBAAMsB,IAAAA,mBAAS,EAACpB;;;oBAAhB,OAAA,eAApEK,QAAoE,KAApEA,OAAOC,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDc,UAAmCb,gBAAkB,KAA1Bc;oBAC5Cb,UAAUT,IAAIS,OAAO;oBAED;;wBAAMc,IAAAA,iBAAO,EAAClB,OAAO;;;oBAAzCK,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACT,KAAKuB,KAAK,AAAD,GAA/Bd;;;;oBACIC,MAAMc,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACjB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMZ,kBAAiB,GAAnCY;;;;oBACF;;wBAAMN,MAAMuB,KAAK;;;oBAAjB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACpB,MAAM,OAAM;;oBAI7GT,QAAQ;wBAAE8B,MAAM;wBAAWvB,SAAAA;wBAASH,QAAAA;oBAAO;yBACvCC,CAAAA,gBAAgB0B,MAAM,GAAG,KAAKzB,gBAAgB,CAAA,GAA9CD;;;;;wBACQyB,MAAM;wBAAcV,QAAQd;;oBAAsB;;wBAAM0B,IAAAA,kBAAQ,EAAC7B;;;oBAA3EH;8BAAqDiC,QAAO,oBAAuBd,WAAUd;;;;oBAG3FK,WAAW;oBACTC,iBAAiB;;;;;wCACrB;;4CAAMuB,IAAAA,iBAAO,EAAC/B,OAAO,mBAAmB,IAAIoB,OAAOY,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAAC/B,OAAO,aAAaiC,OAAOC,QAAQC,GAAG;;;wCAApD;;;;;;wBACF;;oBACA;;wBAAM3B;;;oBAAN;oBAEIC,gBAAuC;oBAC3C,2FAA2F;oBAC3F,yFAAyF;oBACrFC,WAAiC;oBAC/BC,oBAAoB;wBACxB,IAAIF,eAAe2B,aAAa3B;wBAChCA,gBAAgB4B,WAAW;4BACzB5B,gBAAgB;4BAChBC,WAAW,AAAC;;wCAEqB,MAArBO,QAAQD,gBAETsB;;;;;;;;;;gDAFsB;;oDAAMtC,MAAMuC,SAAS;;;gDAArB,OAAA,eAArBtB,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;oDACNW,MAAM;oDAAcV,QAAAA;;gDAAe;;oDAAMY,IAAAA,kBAAQ,EAAC7B;;;gDAA5DH;0DAAsCiC,QAAO,oBAAuBd,WAAAA;;;;;;;gDAC7DsB;gDACPzC,QAAQ;oDAAE8B,MAAM;oDAAmBa,SAAS,AAACF,IAAcE,OAAO;gDAAC;;;;;;gDAEnE9B,WAAW;;;;;;;;;;gCAEf;;wBACF,GAAGZ;oBACL;oBAEA,4FAA4F;oBAC5F,yFAAyF;oBACzF,4FAA4F;oBAC5F,4FAA4F;oBAC5F,2DAA2D;oBACrDc,UAAU6B,IAAAA,aAAO,EAACrC,SAAS;wBAAEsC,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpEnC;oBACF;oBACME,iBAAiBkC,YACrBC,IAAAA,0BAAW,EAACxC,gBAAgB;+BAAMD;wBAClCR;oBAGF;;wBAAO,IAAIkD,QAAc,SAACC;gCAoBpBtD,cACCA;4BApBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAMuD,WAAW;;wCAKfvD;;;;gDAJA,IAAIW,UAAU;;;gDACdA,WAAW;gDACX2B,QAAQkB,GAAG,CAAC,UAAUD;gDACtBjB,QAAQkB,GAAG,CAAC,WAAWD;iDACvBvD,eAAAA,KAAKyD,MAAM,cAAXzD,mCAAAA,aAAa0D,mBAAmB,CAAC,SAASH;gDAC1CI,cAAc1C;gDACd,IAAIJ,eAAe2B,aAAa3B;gDAChCG,QAAQW,KAAK;gDACb;;oDAAMb;;;gDAAN;gDACA;;oDAAMqB,IAAAA,iBAAO,EAAC/B,OAAO,mBAAmB;;;gDAAxC;gDACA;;oDAAM+B,IAAAA,iBAAO,EAAC/B,OAAO,aAAa;;;gDAAlC;gDACA;;oDAAMA,MAAMuB,KAAK;;;gDAAjB;gDACA2B;;;;;;gCACF;;4BACAhB,QAAQsB,IAAI,CAAC,UAAUL;4BACvBjB,QAAQsB,IAAI,CAAC,WAAWL;4BACxB,KAAIvD,eAAAA,KAAKyD,MAAM,cAAXzD,mCAAAA,aAAa6D,OAAO,EAAEN;kCACrBvD,gBAAAA,KAAKyD,MAAM,cAAXzD,oCAAAA,cAAa8D,gBAAgB,CAAC,SAASP,UAAU;gCAAEK,MAAM;4BAAK;wBACrE;;;;IACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","initialWarnings","initialParsed","baseDir","existingHeartbeat","age","stopping","touchHeartbeat","debounceTimer","inFlight","scheduleReconcile","watcher","heartbeatTimer","requireWatchConcurrency","openStore","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","type","length","docCount","total","setMeta","toISOString","String","process","pid","clearTimeout","setTimeout","err","reconcile","message","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;uBAEP;wBACC;6BACC;wBACmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE/E,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAcpB,SAAeH;wCAASI,GAAmB;YAAEC,MAElCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBACsE,MAApEC,OAAOC,QAAkBC,iBAAyBC,eACpDC,SAEAC,mBAEEC,WAYJC,UACEC,gBAMFC,eAGAC,UACEC,mBAmBAC,SAIAC;;;;;oBAxD4CjB,OAAAA,oEAAqB,CAAC;oBACxEkB,IAAAA,iCAAuB,EAACnB;oBAClBE,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBACoB;;wBAAMsB,IAAAA,mBAAS,EAACpB;;;oBAAhB,OAAA,eAApEK,QAAoE,KAApEA,OAAOC,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDc,UAAmCb,gBAAkB,KAA1Bc;oBAC5Cb,UAAUT,IAAIS,OAAO;oBAED;;wBAAMc,IAAAA,iBAAO,EAAClB,OAAO;;;oBAAzCK,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACT,KAAKuB,KAAK,AAAD,GAA/Bd;;;;oBACIC,MAAMc,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACjB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMZ,kBAAiB,GAAnCY;;;;oBACF;;wBAAMN,MAAMuB,KAAK;;;oBAAjB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACpB,MAAM,OAAM;;oBAI7GT,QAAQ;wBAAE8B,MAAM;wBAAWvB,SAAAA;wBAASH,QAAAA;oBAAO;yBACvCC,CAAAA,gBAAgB0B,MAAM,GAAG,KAAKzB,gBAAgB,CAAA,GAA9CD;;;;;wBACQyB,MAAM;wBAAcV,QAAQd;;oBAAsB;;wBAAM0B,IAAAA,kBAAQ,EAAC7B;;;oBAA3EH;8BAAqDiC,QAAO,oBAAuBd,WAAUd;;;;oBAG3FK,WAAW;oBACTC,iBAAiB;;;;;wCACrB;;4CAAMuB,IAAAA,iBAAO,EAAC/B,OAAO,mBAAmB,IAAIoB,OAAOY,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAAC/B,OAAO,aAAaiC,OAAOC,QAAQC,GAAG;;;wCAApD;;;;;;wBACF;;oBACA;;wBAAM3B;;;oBAAN;oBAEIC,gBAAuC;oBAC3C,2FAA2F;oBAC3F,yFAAyF;oBACrFC,WAAiC;oBAC/BC,oBAAoB;wBACxB,IAAIF,eAAe2B,aAAa3B;wBAChCA,gBAAgB4B,WAAW;4BACzB5B,gBAAgB;4BAChBC,WAAW,AAAC;;wCAEqB,MAArBO,QAAQD,gBAETsB;;;;;;;;;;gDAFsB;;oDAAMtC,MAAMuC,SAAS;;;gDAArB,OAAA,eAArBtB,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;oDACNW,MAAM;oDAAcV,QAAAA;;gDAAe;;oDAAMY,IAAAA,kBAAQ,EAAC7B;;;gDAA5DH;0DAAsCiC,QAAO,oBAAuBd,WAAAA;;;;;;;gDAC7DsB;gDACPzC,QAAQ;oDAAE8B,MAAM;oDAAmBa,SAAS,AAACF,IAAcE,OAAO;gDAAC;;;;;;gDAEnE9B,WAAW;;;;;;;;;;gCAEf;;wBACF,GAAGZ;oBACL;oBAEA,iGAAiG;oBACjG,uIAAuI;oBACjIc,UAAU6B,IAAAA,aAAO,EAACrC,SAAS;wBAAEsC,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpEnC;oBACF;oBACME,iBAAiBkC,YACrBC,IAAAA,0BAAW,EAACxC,gBAAgB;+BAAMD;wBAClCR;oBAGF;;wBAAO,IAAIkD,QAAc,SAACC;gCAoBpBtD,cACCA;4BApBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAMuD,WAAW;;wCAKfvD;;;;gDAJA,IAAIW,UAAU;;;gDACdA,WAAW;gDACX2B,QAAQkB,GAAG,CAAC,UAAUD;gDACtBjB,QAAQkB,GAAG,CAAC,WAAWD;iDACvBvD,eAAAA,KAAKyD,MAAM,cAAXzD,mCAAAA,aAAa0D,mBAAmB,CAAC,SAASH;gDAC1CI,cAAc1C;gDACd,IAAIJ,eAAe2B,aAAa3B;gDAChCG,QAAQW,KAAK;gDACb;;oDAAMb;;;gDAAN;gDACA;;oDAAMqB,IAAAA,iBAAO,EAAC/B,OAAO,mBAAmB;;;gDAAxC;gDACA;;oDAAM+B,IAAAA,iBAAO,EAAC/B,OAAO,aAAa;;;gDAAlC;gDACA;;oDAAMA,MAAMuB,KAAK;;;gDAAjB;gDACA2B;;;;;;gCACF;;4BACAhB,QAAQsB,IAAI,CAAC,UAAUL;4BACvBjB,QAAQsB,IAAI,CAAC,WAAWL;4BACxB,KAAIvD,eAAAA,KAAKyD,MAAM,cAAXzD,mCAAAA,aAAa6D,OAAO,EAAEN;kCACrBvD,gBAAAA,KAAKyD,MAAM,cAAXzD,oCAAAA,cAAa8D,gBAAgB,CAAC,SAASP,UAAU;gCAAEK,MAAM;4BAAK;wBACrE;;;;IACF"}

package/dist/cjs/workers/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task.\n// A Feature carries closures and cannot cross the thread boundary; its name can, and the\n// registry on this side resolves it back, which keeps the caller's selection intact.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[] } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    return { ok: true, doc, warnings };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["parseTask","Tinypool","workerData","cfg","featureNames","selected","FEATURES","filter","feature","includes","name","file","parseFile","featuresForFile","doc","warnings","ok","err","error","serializeError"],"mappings":";;;;+BA8BA;;;eAAwBA;;;+DA9BH;uBAEI;wBAEC;yBAEM;6BAED;;;;;;AAgB/B,sCAAsC;AACtC,IAA8BC,uBAAAA,iBAAQ,CAACC,UAAU,EAAzCC,MAAsBF,qBAAtBE,KAAKC,eAAiBH,qBAAjBG;AACb,4FAA4F;AAC5F,qDAAqD;AACrD,IAAMC,WAAWC,iBAAQ,CAACC,MAAM,CAAC,SAACC;WAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;;AAEjE,SAASV,UAAUW,IAAe;IAC/C,IAAI;QACF,IAA0BC,aAAAA,IAAAA,mBAAS,EAACD,MAAME,IAAAA,0BAAe,EAACR,UAAUF,KAAKQ,OAAOR,MAAxEW,MAAkBF,WAAlBE,KAAKC,WAAaH,WAAbG;QACb,OAAO;YAAEC,IAAI;YAAMF,KAAAA;YAAKC,UAAAA;QAAS;IACnC,EAAE,OAAOE,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOC,IAAAA,6BAAc,EAACF;QAAK;IACjD;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[] } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    return { ok: true, doc, warnings };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["parseTask","Tinypool","workerData","cfg","featureNames","selected","FEATURES","filter","feature","includes","name","file","parseFile","featuresForFile","doc","warnings","ok","err","error","serializeError"],"mappings":";;;;+BA6BA;;;eAAwBA;;;+DA7BH;uBAEI;wBAEC;yBAEM;6BAED;;;;;;AAe/B,sCAAsC;AACtC,IAA8BC,uBAAAA,iBAAQ,CAACC,UAAU,EAAzCC,MAAsBF,qBAAtBE,KAAKC,eAAiBH,qBAAjBG;AACb,4FAA4F;AAC5F,qDAAqD;AACrD,IAAMC,WAAWC,iBAAQ,CAACC,MAAM,CAAC,SAACC;WAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;;AAEjE,SAASV,UAAUW,IAAe;IAC/C,IAAI;QACF,IAA0BC,aAAAA,IAAAA,mBAAS,EAACD,MAAME,IAAAA,0BAAe,EAACR,UAAUF,KAAKQ,OAAOR,MAAxEW,MAAkBF,WAAlBE,KAAKC,WAAaH,WAAbG;QACb,OAAO;YAAEC,IAAI;YAAMF,KAAAA;YAAKC,UAAAA;QAAS;IACnC,EAAE,OAAOE,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOC,IAAAA,6BAAc,EAACF;QAAK;IACjD;AACF"}

package/dist/esm/chunk/group.js

@@ -5,7 +5,7 @@ export const DEFAULT_TARGET_TOKENS = 500;
 const PGC_GROUP_SIZE = 2;
 const OVERSIZE_TRIGGER_MULTIPLE = 2;
 // D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is
-// spaced but still token-dense, added here as a conservative bound (one Unicode set, one home).
+// spaced but still token-dense and sits in this set as a conservative bound (one Unicode set, one home).
 const DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\p{scx=Hangul}]`, 'u');
 // D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.
 export function estimateTokens(text) {
@@ -79,12 +79,8 @@ function pack(segments, working) {
     if (current.length > 0) groups.push(current);
     return groups;
 }
-// A single line's text, still over working after rule 5's line-split can't reduce a lone line
-// (the CJK case, one sentence-dense line with no newlines), falls back to Intl.Segmenter sentence
-// boundaries, then word boundaries for any one sentence still over working. Never splits inside a
-// grapheme either way (both are ECMA-402, the same proven component segment.ts uses for
-// graphemes). Mode-agnostic: the caller decides whether `text` is the raw line or its extracted
-// flavor -- this only packs whatever text it is given.
+// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then
+// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.
 function splitLineText(text, working) {
     const sentences = segmentsOf(text, SENTENCE_SEGMENTER);
     const out = [];
@@ -116,10 +112,8 @@ const ATOMIC_TYPES = new Set([
     'table',
     'list'
 ]);
-// An atomic block's (code/table/list) piece is always a raw line slice: re-parsing a table's
-// later pieces without their header/delimiter rows demotes them to paragraph text. A non-atomic
-// piece is a raw line slice too when raw mode requested it; extracted mode re-parses and resolves
-// flavor as before.
+// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces
+// without their header/delimiter rows would demote them to paragraph text.
 function piece(pieceLines, startLine, endLine, blockType, textMode) {
     const text = ATOMIC_TYPES.has(blockType) || textMode === 'raw' ? pieceLines.join('\n') : parse(pieceLines.join('\n')).map((b)=>extractText(b.node)).join('\n');
     return {
@@ -128,13 +122,8 @@ function piece(pieceLines, startLine, endLine, blockType, textMode) {
         text
     };
 }
-// A piece over working size that came from exactly one source line cannot be reduced by another
-// line-boundary pass (rule 5's gap); it is sliced instead, at sentence then word boundaries, over
-// the mode-resolved text piece() already produced -- so raw mode packs the raw line and extracted
-// mode packs its flavor-resolved text. Extents collapse to that one line for every sub-piece (F5:
-// re-derivation is by index over chunk()'s own deterministic output, not by a unique line range
-// per piece); `final` flags each sub-piece so group() knows not to re-derive its text from that
-// shared extent.
+// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it
+// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).
 function finalizePiece(pieceLines, startLine, endLine, working, blockType, textMode) {
     const p = piece(pieceLines, startLine, endLine, blockType, textMode);
     if (pieceLines.length === 1 && estimateTokens(p.text) > working) {
@@ -149,9 +138,8 @@ function finalizePiece(pieceLines, startLine, endLine, working, blockType, textM
         p
     ];
 }
-// A block over 2x working size splits at line boundaries into pieces each <= working size,
-// never mid-line. `seed`: pending tokens (a heading) the first piece must join, checked against
-// the limit too.
+// A block over 2x working size splits at line boundaries into pieces each <= working size, never
+// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.
 function splitOversizeBlock(lines, startLine, endLine, working, blockType, textMode, seed = 0) {
     const pieces = [];
     let pieceLines = [];
@@ -238,8 +226,7 @@ export function group(blocks, body, opts) {
     const chunks = [];
     for (const scope of splitScopes(blocks))chunks.push(...groupScope(scope, lines, resolved));
     // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).
-    // A `final` chunk (a sub-line split) already carries its own slice's raw text -- re-slicing by
-    // extent would return the whole shared source line instead, once per sub-piece.
+    // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.
     const texted = resolved.text === 'raw' ? chunks.map((c)=>c.final ? c : {
             ...c,
             text: lines.slice(c.startLine - 1, c.endLine).join('\n').trim()

package/dist/esm/chunk/group.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { UNSPACED_SCRIPTS } from '../text/segment.ts';\nimport { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nexport const DEFAULT_TARGET_TOKENS = 500;\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\n// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is\n// spaced but still token-dense, added here as a conservative bound (one Unicode set, one home).\nconst DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\\\p{scx=Hangul}]`, 'u');\n\n// D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.\nexport function estimateTokens(text: string): number {\n  let dense = 0;\n  let other = 0;\n  for (const ch of text) {\n    if (DENSE_SCRIPT.test(ch)) dense++;\n    else other++;\n  }\n  return dense + other / 4;\n}\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // true for a sub-line split piece (finalizePiece's splitLineText branch): its text is already\n  // the active mode's final text for its own slice of the line, not the whole line, so group()'s\n  // extent-based raw re-slice (startLine === endLine for every sibling sub-piece) must not touch it.\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\nconst SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'sentence' });\nconst WORD_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'word' });\n\nfunction segmentsOf(text: string, segmenter: Intl.Segmenter): string[] {\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// A single line's text, still over working after rule 5's line-split can't reduce a lone line\n// (the CJK case, one sentence-dense line with no newlines), falls back to Intl.Segmenter sentence\n// boundaries, then word boundaries for any one sentence still over working. Never splits inside a\n// grapheme either way (both are ECMA-402, the same proven component segment.ts uses for\n// graphemes). Mode-agnostic: the caller decides whether `text` is the raw line or its extracted\n// flavor -- this only packs whatever text it is given.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, SENTENCE_SEGMENTER);\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// An atomic block's (code/table/list) piece is always a raw line slice: re-parsing a table's\n// later pieces without their header/delimiter rows demotes them to paragraph text. A non-atomic\n// piece is a raw line slice too when raw mode requested it; extracted mode re-parses and resolves\n// flavor as before.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A piece over working size that came from exactly one source line cannot be reduced by another\n// line-boundary pass (rule 5's gap); it is sliced instead, at sentence then word boundaries, over\n// the mode-resolved text piece() already produced -- so raw mode packs the raw line and extracted\n// mode packs its flavor-resolved text. Extents collapse to that one line for every sub-piece (F5:\n// re-derivation is by index over chunk()'s own deterministic output, not by a unique line range\n// per piece); `final` flags each sub-piece so group() knows not to re-derive its text from that\n// shared extent.\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size,\n// never mid-line. `seed`: pending tokens (a heading) the first piece must join, checked against\n// the limit too.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk (a sub-line split) already carries its own slice's raw text -- re-slicing by\n  // extent would return the whole shared source line instead, once per sub-piece.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["UNSPACED_SCRIPTS","extractText","parse","DEFAULT_TARGET_TOKENS","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","DENSE_SCRIPT","RegExp","estimateTokens","text","dense","other","ch","test","resolveOptions","opts","targetTokens","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","SENTENCE_SEGMENTER","Intl","Segmenter","granularity","WORD_SEGMENTER","segmentsOf","segmenter","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","b","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","group","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","split","chunks","scope","texted","c","trim","filter"],"mappings":"AAAA,SAASA,gBAAgB,QAAQ,qBAAqB;AACtD,SAASC,WAAW,QAAQ,eAAe;AAC3C,SAASC,KAAK,QAAQ,aAAa;AAGnC,OAAO,MAAMC,wBAAwB,IAAI;AACzC,MAAMC,iBAAiB;AACvB,MAAMC,4BAA4B;AAElC,yFAAyF;AACzF,gGAAgG;AAChG,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAEP,iBAAiB,gBAAgB,CAAC,EAAE;AAExE,oFAAoF;AACpF,OAAO,SAASQ,eAAeC,IAAY;IACzC,IAAIC,QAAQ;IACZ,IAAIC,QAAQ;IACZ,KAAK,MAAMC,MAAMH,KAAM;QACrB,IAAIH,aAAaO,IAAI,CAACD,KAAKF;aACtBC;IACP;IACA,OAAOD,QAAQC,QAAQ;AACzB;AAOA,SAASG,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIb;QACpCM,IAAI,WAAEM,iBAAAA,2BAAAA,KAAMN,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASQ,YAAYC,MAAe;IAClC,MAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;IACzB,KAAK,MAAMC,SAASH,OAAQ;QAC1B,IAAIG,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;YAClDJ,OAAOK,IAAI,CAACJ;YACZA,UAAU,EAAE;QACd;QACAA,QAAQI,IAAI,CAACH;IACf;IACA,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAYA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,MAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,MAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEtB,MAAMiB,MAAMM,GAAG,CAAC,CAACC,IAAMA,EAAExB,IAAI,EAAEyB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,CAACH,IAAMA,EAAEE,KAAK;IAAE;AAC3I;AAEA,MAAME,iBAAiB7B,eAAe;AACtC,MAAM8B,qBAAqB,IAAIC,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAW;AACnF,MAAMC,iBAAiB,IAAIH,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAO;AAE3E,SAASE,WAAWlC,IAAY,EAAEmC,SAAyB;IACzD,OAAOC,MAAMC,IAAI,CAACF,UAAUG,OAAO,CAACtC,OAAO,CAACuC,IAAMA,EAAED,OAAO;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,MAAMC,SAAmB,EAAE;IAC3B,IAAIhC,UAAU;IACd,IAAIiC,SAAS;IACb,KAAK,MAAMN,WAAWG,SAAU;QAC9B,MAAMI,gBAAgB9C,eAAeuC;QACrC,IAAI3B,QAAQG,MAAM,GAAG,KAAK8B,SAASC,gBAAgBH,SAAS;YAC1DC,OAAO5B,IAAI,CAACJ;YACZA,UAAU;YACViC,SAAS;QACX;QACAjC,WAAW2B;QACXM,UAAUC;IACZ;IACA,IAAIlC,QAAQG,MAAM,GAAG,GAAG6B,OAAO5B,IAAI,CAACJ;IACpC,OAAOgC;AACT;AAEA,8FAA8F;AAC9F,kGAAkG;AAClG,kGAAkG;AAClG,wFAAwF;AACxF,gGAAgG;AAChG,uDAAuD;AACvD,SAASG,cAAc9C,IAAY,EAAE0C,OAAe;IAClD,MAAMK,YAAYb,WAAWlC,MAAM6B;IACnC,MAAMmB,MAAgB,EAAE;IACxB,IAAIrC,UAAU;IACd,IAAIiC,SAAS;IACb,MAAMK,QAAQ;QACZ,IAAItC,QAAQG,MAAM,GAAG,GAAG;YACtBkC,IAAIjC,IAAI,CAACJ;YACTA,UAAU;YACViC,SAAS;QACX;IACF;IACA,KAAK,MAAMM,YAAYH,UAAW;QAChC,MAAMI,iBAAiBpD,eAAemD;QACtC,IAAIC,iBAAiBT,SAAS;YAC5BO;YACAD,IAAIjC,IAAI,IAAIyB,KAAKN,WAAWgB,UAAUjB,iBAAiBS;YACvD;QACF;QACA,IAAI/B,QAAQG,MAAM,GAAG,KAAK8B,SAASO,iBAAiBT,SAASO;QAC7DtC,WAAWuC;QACXN,UAAUO;IACZ;IACAF;IACA,OAAOD;AACT;AAEA,MAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,6FAA6F;AAC7F,gGAAgG;AAChG,kGAAkG;AAClG,oBAAoB;AACpB,SAASC,MAAMC,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEkC,SAAoB,EAAEC,QAA6B;IAC1H,MAAMzD,OACJoD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAW9B,IAAI,CAAC,QAChBhC,MAAM8D,WAAW9B,IAAI,CAAC,OACnBF,GAAG,CAAC,CAACoC,IAAMnE,YAAYmE,EAAEC,IAAI,GAC7BnC,IAAI,CAAC;IACd,OAAO;QAAEJ;QAAWC;QAAStB;IAAK;AACpC;AAEA,gGAAgG;AAChG,kGAAkG;AAClG,kGAAkG;AAClG,kGAAkG;AAClG,gGAAgG;AAChG,gGAAgG;AAChG,iBAAiB;AACjB,SAAS6D,cAAcN,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,MAAMjC,IAAI8B,MAAMC,YAAYlC,WAAWC,SAASkC,WAAWC;IAC3D,IAAIF,WAAWzC,MAAM,KAAK,KAAKf,eAAeyB,EAAExB,IAAI,IAAI0C,SAAS;QAC/D,OAAOI,cAActB,EAAExB,IAAI,EAAE0C,SAASnB,GAAG,CAAC,CAACvB,OAAU,CAAA;gBAAEqB;gBAAWC;gBAAStB;gBAAM0B,OAAO;YAAK,CAAA;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,2FAA2F;AAC3F,gGAAgG;AAChG,iBAAiB;AACjB,SAASsC,mBAAmBC,KAAe,EAAE1C,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B,EAAEO,OAAO,CAAC;IAC7J,MAAMC,SAAiB,EAAE;IACzB,IAAIV,aAAuB,EAAE;IAC7B,IAAIW,aAAa7C;IACjB,IAAIuB,SAASoB;IACb,IAAK,IAAIG,OAAO9C,WAAW8C,QAAQ7C,SAAS6C,OAAQ;QAClD,MAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,MAAME,MAAMd,WAAWzC,MAAM,GAAG,KAAK8B,SAAS,IAAIhB,iBAAiB;QACnE,MAAM0C,aAAavE,eAAeqE;QAClC,IAAIb,WAAWzC,MAAM,GAAG,KAAK8B,SAASyB,MAAMC,aAAa5B,SAAS;YAChEuB,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAYC,OAAO,GAAGzB,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTsB,aAAaC;YACbZ,WAAWxC,IAAI,CAACqD;YAChBxB,UAAU0B;YACV;QACF;QACAf,WAAWxC,IAAI,CAACqD;QAChBxB,UAAUyB,MAAMC;IAClB;IACA,IAAIf,WAAWzC,MAAM,GAAG,GAAGmD,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAY5C,SAASoB,SAASc,WAAWC;IAC7G,OAAOQ;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,MAAM/B,UAAU+B,SAASlE,YAAY;IACrC,MAAMmE,UAAUhC,UAAU9C;IAC1B,MAAM+E,WAA4C,EAAE;IACpD,IAAI1D,QAAgB,EAAE;IACtB,IAAI2D,iBAAiB;IACrB,IAAIhC,SAAS;IAEb,SAASiC;QACP,MAAMC,QAAQ9D,SAASC;QACvB,IAAI6D,OAAOH,SAAS5D,IAAI,CAAC+D;QACzB7D,QAAQ,EAAE;QACV2D,iBAAiB;QACjBhC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASmC,QAAQ/E,IAAY,EAAEgF,QAAgB,EAAE3D,SAAiB,EAAEC,OAAe;QACjFsB,UAAU,AAAC3B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAK7B,eAAeiF;QACnE/D,MAAMF,IAAI,CAAC;YAAEM;YAAWC;YAAStB;QAAK;IACxC;IAEA,KAAK,MAAMY,SAAS4D,YAAa;QAC/B,MAAMS,MAAMlB,MAAMmB,KAAK,CAACtE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;QACjE,MAAM0D,cAAcpF,eAAekF;QAEnC,IAAIE,cAAcT,SAAS;YACzB,MAAMV,OAAO/C,MAAMH,MAAM,GAAG,IAAI8B,SAAS;YACzC,KAAK,MAAMpB,KAAKsC,mBAAmBC,OAAOnD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEoB,SAAS9B,MAAMC,IAAI,EAAE4D,SAASzE,IAAI,EAAEgE,MAAO;gBACnH/C,MAAMF,IAAI,CAACS;gBACXqD;YACF;YACA;QACF;QAEA,MAAMO,YAAY5F,YAAYoB,MAAMgD,IAAI;QACxC,MAAMoB,WAAWP,SAASzE,IAAI,KAAK,QAAQiF,MAAMG;QAEjD,IAAIxE,MAAMC,IAAI,KAAK,WAAW;YAC5BkE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D;QACF;QAEA,wFAAwF;QACxF,wDAAwD;QACxD,MAAM+D,eAAepE,MAAMH,MAAM,GAAG,KAAK8B,SAAShB,iBAAiBuD,cAAcT;QACjF,IAAIW,cAAcR;QAElBE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;QAC3DsD;QAEA,IAAIA,kBAAkBjF,gBAAgBkF;IACxC;IACAA;IAEA,OAAOF;AACT;AAEA,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASG,MAAMrE,MAAe,EAAE6E,IAAY,EAAEhF,IAAmB;IACtE,MAAMmE,WAAWpE,eAAeC;IAChC,MAAMyD,QAAQuB,KAAKC,KAAK,CAAC;IACzB,MAAMC,SAA0C,EAAE;IAClD,KAAK,MAAMC,SAASjF,YAAYC,QAAS+E,OAAOzE,IAAI,IAAIwD,WAAWkB,OAAO1B,OAAOU;IACjF,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAMiB,SACJjB,SAASzE,IAAI,KAAK,QACdwF,OAAOjE,GAAG,CAAC,CAACoE,IACVA,EAAEjE,KAAK,GACHiE,IACA;YACE,GAAGA,CAAC;YACJ3F,MAAM+D,MACHmB,KAAK,CAACS,EAAEtE,SAAS,GAAG,GAAGsE,EAAErE,OAAO,EAChCG,IAAI,CAAC,MACLmE,IAAI;QACT,KAENJ;IACN,mHAAmH;IACnH,OAAOE,OAAOG,MAAM,CAAC,CAACF,IAAMA,EAAE3F,IAAI,CAAC4F,IAAI,GAAG9E,MAAM,GAAG,GAAGS,GAAG,CAAC,CAACoE,IAAO,CAAA;YAAEtE,WAAWsE,EAAEtE,SAAS;YAAEC,SAASqE,EAAErE,OAAO;YAAEtB,MAAM2F,EAAE3F,IAAI;QAAC,CAAA;AAC/H"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { UNSPACED_SCRIPTS } from '../text/segment.ts';\nimport { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nexport const DEFAULT_TARGET_TOKENS = 500;\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\n// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is\n// spaced but still token-dense and sits in this set as a conservative bound (one Unicode set, one home).\nconst DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\\\p{scx=Hangul}]`, 'u');\n\n// D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.\nexport function estimateTokens(text: string): number {\n  let dense = 0;\n  let other = 0;\n  for (const ch of text) {\n    if (DENSE_SCRIPT.test(ch)) dense++;\n    else other++;\n  }\n  return dense + other / 4;\n}\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\nconst SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'sentence' });\nconst WORD_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'word' });\n\nfunction segmentsOf(text: string, segmenter: Intl.Segmenter): string[] {\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, SENTENCE_SEGMENTER);\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["UNSPACED_SCRIPTS","extractText","parse","DEFAULT_TARGET_TOKENS","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","DENSE_SCRIPT","RegExp","estimateTokens","text","dense","other","ch","test","resolveOptions","opts","targetTokens","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","SENTENCE_SEGMENTER","Intl","Segmenter","granularity","WORD_SEGMENTER","segmentsOf","segmenter","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","b","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","group","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","split","chunks","scope","texted","c","trim","filter"],"mappings":"AAAA,SAASA,gBAAgB,QAAQ,qBAAqB;AACtD,SAASC,WAAW,QAAQ,eAAe;AAC3C,SAASC,KAAK,QAAQ,aAAa;AAGnC,OAAO,MAAMC,wBAAwB,IAAI;AACzC,MAAMC,iBAAiB;AACvB,MAAMC,4BAA4B;AAElC,yFAAyF;AACzF,yGAAyG;AACzG,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAEP,iBAAiB,gBAAgB,CAAC,EAAE;AAExE,oFAAoF;AACpF,OAAO,SAASQ,eAAeC,IAAY;IACzC,IAAIC,QAAQ;IACZ,IAAIC,QAAQ;IACZ,KAAK,MAAMC,MAAMH,KAAM;QACrB,IAAIH,aAAaO,IAAI,CAACD,KAAKF;aACtBC;IACP;IACA,OAAOD,QAAQC,QAAQ;AACzB;AAOA,SAASG,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIb;QACpCM,IAAI,WAAEM,iBAAAA,2BAAAA,KAAMN,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASQ,YAAYC,MAAe;IAClC,MAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;IACzB,KAAK,MAAMC,SAASH,OAAQ;QAC1B,IAAIG,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;YAClDJ,OAAOK,IAAI,CAACJ;YACZA,UAAU,EAAE;QACd;QACAA,QAAQI,IAAI,CAACH;IACf;IACA,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,MAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,MAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEtB,MAAMiB,MAAMM,GAAG,CAAC,CAACC,IAAMA,EAAExB,IAAI,EAAEyB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,CAACH,IAAMA,EAAEE,KAAK;IAAE;AAC3I;AAEA,MAAME,iBAAiB7B,eAAe;AACtC,MAAM8B,qBAAqB,IAAIC,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAW;AACnF,MAAMC,iBAAiB,IAAIH,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAO;AAE3E,SAASE,WAAWlC,IAAY,EAAEmC,SAAyB;IACzD,OAAOC,MAAMC,IAAI,CAACF,UAAUG,OAAO,CAACtC,OAAO,CAACuC,IAAMA,EAAED,OAAO;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,MAAMC,SAAmB,EAAE;IAC3B,IAAIhC,UAAU;IACd,IAAIiC,SAAS;IACb,KAAK,MAAMN,WAAWG,SAAU;QAC9B,MAAMI,gBAAgB9C,eAAeuC;QACrC,IAAI3B,QAAQG,MAAM,GAAG,KAAK8B,SAASC,gBAAgBH,SAAS;YAC1DC,OAAO5B,IAAI,CAACJ;YACZA,UAAU;YACViC,SAAS;QACX;QACAjC,WAAW2B;QACXM,UAAUC;IACZ;IACA,IAAIlC,QAAQG,MAAM,GAAG,GAAG6B,OAAO5B,IAAI,CAACJ;IACpC,OAAOgC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAc9C,IAAY,EAAE0C,OAAe;IAClD,MAAMK,YAAYb,WAAWlC,MAAM6B;IACnC,MAAMmB,MAAgB,EAAE;IACxB,IAAIrC,UAAU;IACd,IAAIiC,SAAS;IACb,MAAMK,QAAQ;QACZ,IAAItC,QAAQG,MAAM,GAAG,GAAG;YACtBkC,IAAIjC,IAAI,CAACJ;YACTA,UAAU;YACViC,SAAS;QACX;IACF;IACA,KAAK,MAAMM,YAAYH,UAAW;QAChC,MAAMI,iBAAiBpD,eAAemD;QACtC,IAAIC,iBAAiBT,SAAS;YAC5BO;YACAD,IAAIjC,IAAI,IAAIyB,KAAKN,WAAWgB,UAAUjB,iBAAiBS;YACvD;QACF;QACA,IAAI/B,QAAQG,MAAM,GAAG,KAAK8B,SAASO,iBAAiBT,SAASO;QAC7DtC,WAAWuC;QACXN,UAAUO;IACZ;IACAF;IACA,OAAOD;AACT;AAEA,MAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEkC,SAAoB,EAAEC,QAA6B;IAC1H,MAAMzD,OACJoD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAW9B,IAAI,CAAC,QAChBhC,MAAM8D,WAAW9B,IAAI,CAAC,OACnBF,GAAG,CAAC,CAACoC,IAAMnE,YAAYmE,EAAEC,IAAI,GAC7BnC,IAAI,CAAC;IACd,OAAO;QAAEJ;QAAWC;QAAStB;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAAS6D,cAAcN,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,MAAMjC,IAAI8B,MAAMC,YAAYlC,WAAWC,SAASkC,WAAWC;IAC3D,IAAIF,WAAWzC,MAAM,KAAK,KAAKf,eAAeyB,EAAExB,IAAI,IAAI0C,SAAS;QAC/D,OAAOI,cAActB,EAAExB,IAAI,EAAE0C,SAASnB,GAAG,CAAC,CAACvB,OAAU,CAAA;gBAAEqB;gBAAWC;gBAAStB;gBAAM0B,OAAO;YAAK,CAAA;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAASsC,mBAAmBC,KAAe,EAAE1C,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B,EAAEO,OAAO,CAAC;IAC7J,MAAMC,SAAiB,EAAE;IACzB,IAAIV,aAAuB,EAAE;IAC7B,IAAIW,aAAa7C;IACjB,IAAIuB,SAASoB;IACb,IAAK,IAAIG,OAAO9C,WAAW8C,QAAQ7C,SAAS6C,OAAQ;QAClD,MAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,MAAME,MAAMd,WAAWzC,MAAM,GAAG,KAAK8B,SAAS,IAAIhB,iBAAiB;QACnE,MAAM0C,aAAavE,eAAeqE;QAClC,IAAIb,WAAWzC,MAAM,GAAG,KAAK8B,SAASyB,MAAMC,aAAa5B,SAAS;YAChEuB,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAYC,OAAO,GAAGzB,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTsB,aAAaC;YACbZ,WAAWxC,IAAI,CAACqD;YAChBxB,UAAU0B;YACV;QACF;QACAf,WAAWxC,IAAI,CAACqD;QAChBxB,UAAUyB,MAAMC;IAClB;IACA,IAAIf,WAAWzC,MAAM,GAAG,GAAGmD,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAY5C,SAASoB,SAASc,WAAWC;IAC7G,OAAOQ;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,MAAM/B,UAAU+B,SAASlE,YAAY;IACrC,MAAMmE,UAAUhC,UAAU9C;IAC1B,MAAM+E,WAA4C,EAAE;IACpD,IAAI1D,QAAgB,EAAE;IACtB,IAAI2D,iBAAiB;IACrB,IAAIhC,SAAS;IAEb,SAASiC;QACP,MAAMC,QAAQ9D,SAASC;QACvB,IAAI6D,OAAOH,SAAS5D,IAAI,CAAC+D;QACzB7D,QAAQ,EAAE;QACV2D,iBAAiB;QACjBhC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASmC,QAAQ/E,IAAY,EAAEgF,QAAgB,EAAE3D,SAAiB,EAAEC,OAAe;QACjFsB,UAAU,AAAC3B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAK7B,eAAeiF;QACnE/D,MAAMF,IAAI,CAAC;YAAEM;YAAWC;YAAStB;QAAK;IACxC;IAEA,KAAK,MAAMY,SAAS4D,YAAa;QAC/B,MAAMS,MAAMlB,MAAMmB,KAAK,CAACtE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;QACjE,MAAM0D,cAAcpF,eAAekF;QAEnC,IAAIE,cAAcT,SAAS;YACzB,MAAMV,OAAO/C,MAAMH,MAAM,GAAG,IAAI8B,SAAS;YACzC,KAAK,MAAMpB,KAAKsC,mBAAmBC,OAAOnD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEoB,SAAS9B,MAAMC,IAAI,EAAE4D,SAASzE,IAAI,EAAEgE,MAAO;gBACnH/C,MAAMF,IAAI,CAACS;gBACXqD;YACF;YACA;QACF;QAEA,MAAMO,YAAY5F,YAAYoB,MAAMgD,IAAI;QACxC,MAAMoB,WAAWP,SAASzE,IAAI,KAAK,QAAQiF,MAAMG;QAEjD,IAAIxE,MAAMC,IAAI,KAAK,WAAW;YAC5BkE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D;QACF;QAEA,wFAAwF;QACxF,wDAAwD;QACxD,MAAM+D,eAAepE,MAAMH,MAAM,GAAG,KAAK8B,SAAShB,iBAAiBuD,cAAcT;QACjF,IAAIW,cAAcR;QAElBE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;QAC3DsD;QAEA,IAAIA,kBAAkBjF,gBAAgBkF;IACxC;IACAA;IAEA,OAAOF;AACT;AAEA,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASG,MAAMrE,MAAe,EAAE6E,IAAY,EAAEhF,IAAmB;IACtE,MAAMmE,WAAWpE,eAAeC;IAChC,MAAMyD,QAAQuB,KAAKC,KAAK,CAAC;IACzB,MAAMC,SAA0C,EAAE;IAClD,KAAK,MAAMC,SAASjF,YAAYC,QAAS+E,OAAOzE,IAAI,IAAIwD,WAAWkB,OAAO1B,OAAOU;IACjF,+FAA+F;IAC/F,qHAAqH;IACrH,MAAMiB,SACJjB,SAASzE,IAAI,KAAK,QACdwF,OAAOjE,GAAG,CAAC,CAACoE,IACVA,EAAEjE,KAAK,GACHiE,IACA;YACE,GAAGA,CAAC;YACJ3F,MAAM+D,MACHmB,KAAK,CAACS,EAAEtE,SAAS,GAAG,GAAGsE,EAAErE,OAAO,EAChCG,IAAI,CAAC,MACLmE,IAAI;QACT,KAENJ;IACN,mHAAmH;IACnH,OAAOE,OAAOG,MAAM,CAAC,CAACF,IAAMA,EAAE3F,IAAI,CAAC4F,IAAI,GAAG9E,MAAM,GAAG,GAAGS,GAAG,CAAC,CAACoE,IAAO,CAAA;YAAEtE,WAAWsE,EAAEtE,SAAS;YAAEC,SAASqE,EAAErE,OAAO;YAAEtB,MAAM2F,EAAE3F,IAAI;QAAC,CAAA;AAC/H"}

package/dist/esm/chunk/parse.js

@@ -18,11 +18,8 @@ const BLOCK_TYPES = {
     list: 'list',
     blockquote: 'blockquote'
 };
-// The five extensions extract.ts/group.ts actually consume, imported individually rather than
-// through micromark-extension-gfm/mdast-util-gfm: those bundles also pull in gfm-tagfilter,
-// an HTML-serialization sanitizer this library never uses (no htmlExtensions call anywhere).
-// micromark and mdast-util-from-markdown each already flatten an extensions array themselves,
-// so no combine-extensions helper is needed here.
+// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull
+// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).
 const EXTENSIONS = [
     gfmAutolinkLiteral(),
     gfmFootnote(),
@@ -37,9 +34,8 @@ const MDAST_EXTENSIONS = [
     gfmTableFromMarkdown(),
     gfmTaskListItemFromMarkdown()
 ];
-// Top-level blocks of a markdown body, typed and line-extent bounded. Extents come from
-// mdast's own node.position, never a regex guess at where a block starts or ends; GFM adds
-// tables, task lists, footnotes and strikethrough to the CommonMark set fromMarkdown parses.
+// Top-level blocks of a markdown body, typed and line-extent bounded from mdast's own
+// node.position (never a regex guess). GFM extensions add tables, task lists, footnotes, strikethrough.
 export function parse(body) {
     const tree = fromMarkdown(body, {
         extensions: EXTENSIONS,

package/dist/esm/chunk/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import type { RootContent } from 'mdast';\nimport { fromMarkdown } from 'mdast-util-from-markdown';\nimport { gfmAutolinkLiteralFromMarkdown } from 'mdast-util-gfm-autolink-literal';\nimport { gfmFootnoteFromMarkdown } from 'mdast-util-gfm-footnote';\nimport { gfmStrikethroughFromMarkdown } from 'mdast-util-gfm-strikethrough';\nimport { gfmTableFromMarkdown } from 'mdast-util-gfm-table';\nimport { gfmTaskListItemFromMarkdown } from 'mdast-util-gfm-task-list-item';\nimport { gfmAutolinkLiteral } from 'micromark-extension-gfm-autolink-literal';\nimport { gfmFootnote } from 'micromark-extension-gfm-footnote';\nimport { gfmStrikethrough } from 'micromark-extension-gfm-strikethrough';\nimport { gfmTable } from 'micromark-extension-gfm-table';\nimport { gfmTaskListItem } from 'micromark-extension-gfm-task-list-item';\nimport { extractText } from './extract.ts';\nimport type { Block, BlockType } from './types.ts';\n\nconst BLOCK_TYPES: Partial<Record<RootContent['type'], BlockType>> = {\n  heading: 'heading',\n  paragraph: 'paragraph',\n  code: 'code',\n  table: 'table',\n  list: 'list',\n  blockquote: 'blockquote',\n};\n\n// The five extensions extract.ts/group.ts actually consume, imported individually rather than\n// through micromark-extension-gfm/mdast-util-gfm: those bundles also pull in gfm-tagfilter,\n// an HTML-serialization sanitizer this library never uses (no htmlExtensions call anywhere).\n// micromark and mdast-util-from-markdown each already flatten an extensions array themselves,\n// so no combine-extensions helper is needed here.\nconst EXTENSIONS = [gfmAutolinkLiteral(), gfmFootnote(), gfmStrikethrough(), gfmTable(), gfmTaskListItem()];\nconst MDAST_EXTENSIONS = [gfmAutolinkLiteralFromMarkdown(), gfmFootnoteFromMarkdown(), gfmStrikethroughFromMarkdown(), gfmTableFromMarkdown(), gfmTaskListItemFromMarkdown()];\n\n// Top-level blocks of a markdown body, typed and line-extent bounded. Extents come from\n// mdast's own node.position, never a regex guess at where a block starts or ends; GFM adds\n// tables, task lists, footnotes and strikethrough to the CommonMark set fromMarkdown parses.\nexport function parse(body: string): Block[] {\n  const tree = fromMarkdown(body, { extensions: EXTENSIONS, mdastExtensions: MDAST_EXTENSIONS });\n  return tree.children.map((node) => {\n    const position = node.position;\n    const block: Block = {\n      type: BLOCK_TYPES[node.type] ?? 'other',\n      startLine: position ? position.start.line : 1,\n      endLine: position ? position.end.line : 1,\n      node,\n    };\n    if (node.type === 'heading') {\n      block.depth = node.depth;\n      block.text = extractText(node);\n    }\n    return block;\n  });\n}\n"],"names":["fromMarkdown","gfmAutolinkLiteralFromMarkdown","gfmFootnoteFromMarkdown","gfmStrikethroughFromMarkdown","gfmTableFromMarkdown","gfmTaskListItemFromMarkdown","gfmAutolinkLiteral","gfmFootnote","gfmStrikethrough","gfmTable","gfmTaskListItem","extractText","BLOCK_TYPES","heading","paragraph","code","table","list","blockquote","EXTENSIONS","MDAST_EXTENSIONS","parse","body","tree","extensions","mdastExtensions","children","map","node","position","block","type","startLine","start","line","endLine","end","depth","text"],"mappings":"AACA,SAASA,YAAY,QAAQ,2BAA2B;AACxD,SAASC,8BAA8B,QAAQ,kCAAkC;AACjF,SAASC,uBAAuB,QAAQ,0BAA0B;AAClE,SAASC,4BAA4B,QAAQ,+BAA+B;AAC5E,SAASC,oBAAoB,QAAQ,uBAAuB;AAC5D,SAASC,2BAA2B,QAAQ,gCAAgC;AAC5E,SAASC,kBAAkB,QAAQ,2CAA2C;AAC9E,SAASC,WAAW,QAAQ,mCAAmC;AAC/D,SAASC,gBAAgB,QAAQ,wCAAwC;AACzE,SAASC,QAAQ,QAAQ,gCAAgC;AACzD,SAASC,eAAe,QAAQ,yCAAyC;AACzE,SAASC,WAAW,QAAQ,eAAe;AAG3C,MAAMC,cAA+D;IACnEC,SAAS;IACTC,WAAW;IACXC,MAAM;IACNC,OAAO;IACPC,MAAM;IACNC,YAAY;AACd;AAEA,8FAA8F;AAC9F,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,kDAAkD;AAClD,MAAMC,aAAa;IAACb;IAAsBC;IAAeC;IAAoBC;IAAYC;CAAkB;AAC3G,MAAMU,mBAAmB;IAACnB;IAAkCC;IAA2BC;IAAgCC;IAAwBC;CAA8B;AAE7K,wFAAwF;AACxF,2FAA2F;AAC3F,6FAA6F;AAC7F,OAAO,SAASgB,MAAMC,IAAY;IAChC,MAAMC,OAAOvB,aAAasB,MAAM;QAAEE,YAAYL;QAAYM,iBAAiBL;IAAiB;IAC5F,OAAOG,KAAKG,QAAQ,CAACC,GAAG,CAAC,CAACC;YAGhBhB;QAFR,MAAMiB,WAAWD,KAAKC,QAAQ;QAC9B,MAAMC,QAAe;YACnBC,IAAI,GAAEnB,yBAAAA,WAAW,CAACgB,KAAKG,IAAI,CAAC,cAAtBnB,oCAAAA,yBAA0B;YAChCoB,WAAWH,WAAWA,SAASI,KAAK,CAACC,IAAI,GAAG;YAC5CC,SAASN,WAAWA,SAASO,GAAG,CAACF,IAAI,GAAG;YACxCN;QACF;QACA,IAAIA,KAAKG,IAAI,KAAK,WAAW;YAC3BD,MAAMO,KAAK,GAAGT,KAAKS,KAAK;YACxBP,MAAMQ,IAAI,GAAG3B,YAAYiB;QAC3B;QACA,OAAOE;IACT;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import type { RootContent } from 'mdast';\nimport { fromMarkdown } from 'mdast-util-from-markdown';\nimport { gfmAutolinkLiteralFromMarkdown } from 'mdast-util-gfm-autolink-literal';\nimport { gfmFootnoteFromMarkdown } from 'mdast-util-gfm-footnote';\nimport { gfmStrikethroughFromMarkdown } from 'mdast-util-gfm-strikethrough';\nimport { gfmTableFromMarkdown } from 'mdast-util-gfm-table';\nimport { gfmTaskListItemFromMarkdown } from 'mdast-util-gfm-task-list-item';\nimport { gfmAutolinkLiteral } from 'micromark-extension-gfm-autolink-literal';\nimport { gfmFootnote } from 'micromark-extension-gfm-footnote';\nimport { gfmStrikethrough } from 'micromark-extension-gfm-strikethrough';\nimport { gfmTable } from 'micromark-extension-gfm-table';\nimport { gfmTaskListItem } from 'micromark-extension-gfm-task-list-item';\nimport { extractText } from './extract.ts';\nimport type { Block, BlockType } from './types.ts';\n\nconst BLOCK_TYPES: Partial<Record<RootContent['type'], BlockType>> = {\n  heading: 'heading',\n  paragraph: 'paragraph',\n  code: 'code',\n  table: 'table',\n  list: 'list',\n  blockquote: 'blockquote',\n};\n\n// Imported individually, not via micromark-extension-gfm/mdast-util-gfm: those bundles also pull\n// in gfm-tagfilter, an HTML sanitizer this library never uses (no htmlExtensions call anywhere).\nconst EXTENSIONS = [gfmAutolinkLiteral(), gfmFootnote(), gfmStrikethrough(), gfmTable(), gfmTaskListItem()];\nconst MDAST_EXTENSIONS = [gfmAutolinkLiteralFromMarkdown(), gfmFootnoteFromMarkdown(), gfmStrikethroughFromMarkdown(), gfmTableFromMarkdown(), gfmTaskListItemFromMarkdown()];\n\n// Top-level blocks of a markdown body, typed and line-extent bounded from mdast's own\n// node.position (never a regex guess). GFM extensions add tables, task lists, footnotes, strikethrough.\nexport function parse(body: string): Block[] {\n  const tree = fromMarkdown(body, { extensions: EXTENSIONS, mdastExtensions: MDAST_EXTENSIONS });\n  return tree.children.map((node) => {\n    const position = node.position;\n    const block: Block = {\n      type: BLOCK_TYPES[node.type] ?? 'other',\n      startLine: position ? position.start.line : 1,\n      endLine: position ? position.end.line : 1,\n      node,\n    };\n    if (node.type === 'heading') {\n      block.depth = node.depth;\n      block.text = extractText(node);\n    }\n    return block;\n  });\n}\n"],"names":["fromMarkdown","gfmAutolinkLiteralFromMarkdown","gfmFootnoteFromMarkdown","gfmStrikethroughFromMarkdown","gfmTableFromMarkdown","gfmTaskListItemFromMarkdown","gfmAutolinkLiteral","gfmFootnote","gfmStrikethrough","gfmTable","gfmTaskListItem","extractText","BLOCK_TYPES","heading","paragraph","code","table","list","blockquote","EXTENSIONS","MDAST_EXTENSIONS","parse","body","tree","extensions","mdastExtensions","children","map","node","position","block","type","startLine","start","line","endLine","end","depth","text"],"mappings":"AACA,SAASA,YAAY,QAAQ,2BAA2B;AACxD,SAASC,8BAA8B,QAAQ,kCAAkC;AACjF,SAASC,uBAAuB,QAAQ,0BAA0B;AAClE,SAASC,4BAA4B,QAAQ,+BAA+B;AAC5E,SAASC,oBAAoB,QAAQ,uBAAuB;AAC5D,SAASC,2BAA2B,QAAQ,gCAAgC;AAC5E,SAASC,kBAAkB,QAAQ,2CAA2C;AAC9E,SAASC,WAAW,QAAQ,mCAAmC;AAC/D,SAASC,gBAAgB,QAAQ,wCAAwC;AACzE,SAASC,QAAQ,QAAQ,gCAAgC;AACzD,SAASC,eAAe,QAAQ,yCAAyC;AACzE,SAASC,WAAW,QAAQ,eAAe;AAG3C,MAAMC,cAA+D;IACnEC,SAAS;IACTC,WAAW;IACXC,MAAM;IACNC,OAAO;IACPC,MAAM;IACNC,YAAY;AACd;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,MAAMC,aAAa;IAACb;IAAsBC;IAAeC;IAAoBC;IAAYC;CAAkB;AAC3G,MAAMU,mBAAmB;IAACnB;IAAkCC;IAA2BC;IAAgCC;IAAwBC;CAA8B;AAE7K,sFAAsF;AACtF,wGAAwG;AACxG,OAAO,SAASgB,MAAMC,IAAY;IAChC,MAAMC,OAAOvB,aAAasB,MAAM;QAAEE,YAAYL;QAAYM,iBAAiBL;IAAiB;IAC5F,OAAOG,KAAKG,QAAQ,CAACC,GAAG,CAAC,CAACC;YAGhBhB;QAFR,MAAMiB,WAAWD,KAAKC,QAAQ;QAC9B,MAAMC,QAAe;YACnBC,IAAI,GAAEnB,yBAAAA,WAAW,CAACgB,KAAKG,IAAI,CAAC,cAAtBnB,oCAAAA,yBAA0B;YAChCoB,WAAWH,WAAWA,SAASI,KAAK,CAACC,IAAI,GAAG;YAC5CC,SAASN,WAAWA,SAASO,GAAG,CAACF,IAAI,GAAG;YACxCN;QACF;QACA,IAAIA,KAAKG,IAAI,KAAK,WAAW;YAC3BD,MAAMO,KAAK,GAAGT,KAAKS,KAAK;YACxBP,MAAMQ,IAAI,GAAG3B,YAAYiB;QAC3B;QACA,OAAOE;IACT;AACF"}

package/dist/esm/cli/index.js

@@ -1,6 +1,5 @@
-// Each command's usage line, minus the leading "usage: <name> " -- shared by cli.ts's
-// top-level usage() and the command's own parse() error/--help output, so there is one
-// copy of each line, not two that can drift.
+// Each command's usage line, minus the leading "usage: <name> ": shared by cli.ts's
+// usage() and the command's own parse() error/--help output so there is one copy, not two.
 export const USAGE = {
     init: 'init [--model id-or-path] [--provider static|openai|cohere] [--url url]',
     watch: 'watch [--force] [--config path]',
@@ -14,8 +13,7 @@ export const USAGE = {
     related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where "<sql>"] [--k n] [--format table|json|csv] [--config path]'
 };
 // Lazy registry: a command's imports load only when it runs, so a heavy dependency in one
-// command never taxes the others (or --version/--help). These names are reserved: a saved
-// entry in config can never shadow them.
+// command never taxes the others (or --version/--help). These names are reserved from saved queries.
 export const COMMANDS = {
     init: ()=>import('./init.js'),
     watch: ()=>import('./watch.js'),

package/dist/esm/cli/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \" -- shared by cli.ts's\n// top-level usage() and the command's own parse() error/--help output, so there is one\n// copy of each line, not two that can drift.\nexport const USAGE = {\n  init: 'init [--model id-or-path] [--provider static|openai|cohere] [--url url]',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--preset name] [--format table|json|csv] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved: a saved\n// entry in config can never shadow them.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,sFAAsF;AACtF,uFAAuF;AACvF,6CAA6C;AAC7C,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,0FAA0F;AAC1F,yCAAyC;AACzC,OAAO,MAAMC,WAAgE;IAC3EV,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \": shared by cli.ts's\n// usage() and the command's own parse() error/--help output so there is one copy, not two.\nexport const USAGE = {\n  init: 'init [--model id-or-path] [--provider static|openai|cohere] [--url url]',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--preset name] [--format table|json|csv] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved from saved queries.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,oFAAoF;AACpF,2FAA2F;AAC3F,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,MAAMC,WAAgE;IAC3EV,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}

package/dist/esm/cli/shared.js

@@ -46,9 +46,8 @@ export const SEARCH_FLAGS = {
         type: 'string'
     }
 };
-// The SCOPE fragment's parsed values as the overrides every scoped command passes down. One
-// place to read them, so a new scope field is added to the fragment and here, not in each
-// command's call.
+// The SCOPE fragment's parsed values as the overrides every scoped command passes down.
+// One place to read them: a new scope field is added to the fragment and here, not per-command.
 export function scopeOf(values) {
     return {
         preset: values.preset,
@@ -58,8 +57,8 @@ export function scopeOf(values) {
         noExclude: values['no-exclude'] === true
     };
 }
-// An unrecognised value used to fall through to `table`, so a typo looked like it worked.
-// parseArgs is strict about flag names; this is the same strictness for the value.
+// parseArgs is strict about flag names; this applies the same strictness to the value.
+// An unrecognised --format is a usage error, not a silent fallback to `table`.
 function pickFormat(values, allowed) {
     const format = String(values.format);
     if (allowed.includes(format)) return format;
@@ -113,9 +112,8 @@ export function parse(argv, usage, options) {
         positionals
     };
 }
-// --k must be a positive integer: SQLite reads a bound LIMIT of -1 as "unlimited" and 0 as
-// "nothing", and parseInt would silently truncate "5.9" -- all three are caller mistakes
-// worth a usage error, matching the config-level SavedSearch validation.
+// --k must be a positive integer: SQLite reads a bound LIMIT of -1 as "unlimited" and 0 as "nothing", and parseInt would silently truncate "5.9".
+// All three are caller mistakes worth a usage error, matching the config-level SavedSearch validation.
 export function parseK(k, usageError) {
     if (k === undefined) return undefined;
     const parsed = Number(k);
@@ -136,11 +134,8 @@ export async function withDb(ctx, configPath, fn) {
         await store.close();
     }
 }
-// `--preset` on SQL binds the scope rather than applying it: the statement joins `scope`
-// itself. Filtering behind the query's back is not available -- the obvious version, temp views
-// shadowing the base tables, cannot cover `content`, because FTS5 `MATCH` uses the table name
-// as a hidden column and a view has none. A flag that silently scoped three tables of four
-// would look scoped and not be.
+// `--preset` on SQL binds a temp `scope` table rather than transparently filtering: temp views shadowing the base tables can't cover `content`, since FTS5 MATCH treats the table name as a hidden column.
+// A flag that silently scoped three tables of four would look scoped and not be.
 async function bindScope(store, cfg, preset) {
     const { name } = resolvePreset(cfg, preset); // unknown names throw, listing what is declared
     await store.exec('DROP TABLE IF EXISTS temp.scope');
@@ -166,12 +161,8 @@ export async function runSql(cfg, sql, params, format, label, preset) {
     const { store, warnings } = await openStore(cfg);
     printWarnings(warnings);
     if (preset !== undefined) await bindScope(store, cfg, preset);
-    // Streamed rather than collected: `sql` is the one caller whose result size is the
-    // statement's business, not this package's, so the rows are never held here in bulk.
-    // columns() reads the statement's own metadata, so a 0-row csv still has its header.
-    // A mid-stream SQLite error (e.g. SQLITE_BUSY outlasting busy_timeout) can still leave
-    // partial output; the nonzero exit code is the caller's signal, output completeness is not
-    // guaranteed on failure.
+    // Streamed, not collected: `sql`'s result size is the caller's business, not ours. columns() reads the statement's own metadata, so a 0-row csv gets a header too.
+    // A mid-stream SQLite error (e.g. SQLITE_BUSY outlasting busy_timeout) may leave partial output; the nonzero exit code is the failure signal, not a completeness guarantee.
     try {
         const statement = await store.raw.prepare(sql);
         const columns = statement.columns().map((c)=>c.name);
@@ -179,9 +170,8 @@ export async function runSql(cfg, sql, params, format, label, preset) {
     } catch (err) {
         const hinted = columnHint(await store.docs.columns(), err); // columns read while the store is still open
         await store.close();
-        // A saved query or ad-hoc SQL can carry `content MATCH ?` too, so the same FTS5
-        // punctuation trap applies -- the bound parameters are the search terms there. SQL
-        // without MATCH gets its error verbatim; search advice on a plain typo would mislead.
+        // A saved query or ad-hoc SQL can carry `content MATCH ?` too, so the same FTS5 punctuation trap applies to its bound parameters.
+        // SQL without MATCH gets its error verbatim; search advice on a plain typo would mislead.
         if (/\bMATCH\b/i.test(sql)) throw searchError(hinted, params.join(' '));
         throw hinted;
     }

package/dist/esm/cli/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/shared.ts"],"sourcesContent":["import type { ParseArgsOptionsConfig } from 'node:util';\nimport { parseArgs } from 'node:util';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { resolvePreset } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\nimport type { Row, RowFormat } from '../output/output.ts';\nimport { printRowStream } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport type { OpenResult } from '../store/index.ts';\nimport { openStore } from '../store/index.ts';\nimport type { Store } from '../store/types.ts';\nimport type { Ctx } from './types.ts';\n\n// Spreadable option fragments -- one flag name keeps one meaning across every command's table.\nexport const FORMAT: ParseArgsOptionsConfig = { format: { type: 'string', default: 'table' } };\nexport const CONFIG: ParseArgsOptionsConfig = { config: { type: 'string' } };\n// The scope vocabulary every scoped command shares: named preset, ad hoc include/exclude\n// globs, a where SQL condition. search adds k on top.\nexport const SCOPE: ParseArgsOptionsConfig = {\n  where: { type: 'string' },\n  preset: { type: 'string' },\n  include: { type: 'string', multiple: true },\n  exclude: { type: 'string', multiple: true },\n  // Widening, which no other flag can express: --include and --exclude each override their\n  // own side of the preset, so neither can drop an exclusion the preset declares.\n  'no-exclude': { type: 'boolean', default: false },\n};\nexport const SEARCH_FLAGS: ParseArgsOptionsConfig = { ...SCOPE, k: { type: 'string' } };\n\ntype Values = Record<string, string | boolean | string[] | undefined>;\n\n// The SCOPE fragment's parsed values as the overrides every scoped command passes down. One\n// place to read them, so a new scope field is added to the fragment and here, not in each\n// command's call.\nexport function scopeOf(values: Values): SearchOverrides {\n  return {\n    preset: values.preset as string | undefined,\n    include: values.include as string[] | undefined,\n    exclude: values.exclude as string[] | undefined,\n    where: values.where as string | undefined,\n    noExclude: values['no-exclude'] === true,\n  };\n}\n\n// An unrecognised value used to fall through to `table`, so a typo looked like it worked.\n// parseArgs is strict about flag names; this is the same strictness for the value.\nfunction pickFormat<T extends string>(values: Values, allowed: readonly T[]): T {\n  const format = String(values.format);\n  if ((allowed as readonly string[]).includes(format)) return format as T;\n  console.error(`unknown --format \"${format}\"; expected ${allowed.join(', ')}`);\n  process.exit(2);\n}\n\nexport function formatOf(values: Values): 'table' | 'json' {\n  return pickFormat(values, ['table', 'json'] as const);\n}\n\n// csv is a row set rendered as rows; map, peek, status, and path render structures instead,\n// and take formatOf.\nexport function rowFormatOf(values: Values): RowFormat {\n  return pickFormat(values, ['table', 'json', 'csv'] as const);\n}\n\n// Per-command parseArgs: strict (a foreign flag exits 2), and every table gets --help for free.\nexport function parse(argv: string[], usage: string, options: ParseArgsOptionsConfig): { values: Values; positionals: string[] } {\n  let values: Values;\n  let positionals: string[];\n  try {\n    ({ values, positionals } = parseArgs({\n      args: argv,\n      options: { ...options, help: { type: 'boolean', default: false, short: 'h' } },\n      strict: true,\n      allowPositionals: true,\n    }));\n  } catch (err) {\n    console.error((err as Error).message);\n    console.error(usage);\n    process.exit(2);\n  }\n  if (values.help) {\n    console.log(usage);\n    process.exit(0);\n  }\n  return { values, positionals };\n}\n\n// --k must be a positive integer: SQLite reads a bound LIMIT of -1 as \"unlimited\" and 0 as\n// \"nothing\", and parseInt would silently truncate \"5.9\" -- all three are caller mistakes\n// worth a usage error, matching the config-level SavedSearch validation.\nexport function parseK(k: string | undefined, usageError: (message: string) => never): number | undefined {\n  if (k === undefined) return undefined;\n  const parsed = Number(k);\n  if (!Number.isInteger(parsed) || parsed <= 0) usageError(`--k expects a positive integer, got \"${k}\"`);\n  return parsed;\n}\n\n// Shared open-query-close envelope for commands that touch the tree.\n\nexport function printWarnings(warnings: string[]): void {\n  for (const w of warnings) console.warn(w);\n}\n\nexport async function withDb(ctx: Ctx, configPath: string | undefined, fn: (store: OpenResult['store'], cfg: ResolvedConfig) => void | Promise<void>): Promise<void> {\n  const cfg = ctx.resolveConfig(configPath);\n  const { store, warnings } = await openStore(cfg);\n  printWarnings(warnings);\n  try {\n    await fn(store, cfg);\n  } finally {\n    await store.close();\n  }\n}\n\n// `--preset` on SQL binds the scope rather than applying it: the statement joins `scope`\n// itself. Filtering behind the query's back is not available -- the obvious version, temp views\n// shadowing the base tables, cannot cover `content`, because FTS5 `MATCH` uses the table name\n// as a hidden column and a view has none. A flag that silently scoped three tables of four\n// would look scoped and not be.\nasync function bindScope(store: Store, cfg: ResolvedConfig, preset: string): Promise<void> {\n  const { name } = resolvePreset(cfg, preset); // unknown names throw, listing what is declared\n  await store.exec('DROP TABLE IF EXISTS temp.scope');\n  await store.exec('CREATE TEMP TABLE scope (\"path\" TEXT PRIMARY KEY)');\n  const insert = await store.prepare('INSERT INTO temp.scope (\"path\") SELECT \"path\" FROM preset_files WHERE preset = ?');\n  await insert.run(name);\n}\n\n// An unbound `?` silently binds NULL, so mismatched param counts fail loudly instead.\nexport async function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: RowFormat, label: string, preset?: string): Promise<void> {\n  const placeholderCount = (sql.match(/\\?/g) ?? []).length;\n  if (params.length !== placeholderCount) {\n    console.error(`${label} expects ${placeholderCount} parameter(s), got ${params.length}`);\n    process.exit(2);\n  }\n  // Naming a preset and never joining it would return the whole index while reading as scoped,\n  // which is the one hazard of binding rather than applying. Refuse instead.\n  if (preset !== undefined && !/\\bscope\\b/i.test(sql)) {\n    console.error(`--preset binds a temporary \"scope\" table of the preset's paths, and ${label} never joins it, so the preset would have no effect`);\n    console.error(`add: JOIN scope ON scope.\"path\" = <table>.\"path\"`);\n    process.exit(2);\n  }\n  const { store, warnings } = await openStore(cfg);\n  printWarnings(warnings);\n  if (preset !== undefined) await bindScope(store, cfg, preset);\n  // Streamed rather than collected: `sql` is the one caller whose result size is the\n  // statement's business, not this package's, so the rows are never held here in bulk.\n  // columns() reads the statement's own metadata, so a 0-row csv still has its header.\n  // A mid-stream SQLite error (e.g. SQLITE_BUSY outlasting busy_timeout) can still leave\n  // partial output; the nonzero exit code is the caller's signal, output completeness is not\n  // guaranteed on failure.\n  try {\n    const statement = await store.raw.prepare(sql);\n    const columns = statement.columns().map((c) => c.name);\n    await printRowStream(statement.iterate(...params) as AsyncIterable<Row>, format, columns);\n  } catch (err) {\n    const hinted = columnHint(await store.docs.columns(), err as Error); // columns read while the store is still open\n    await store.close();\n    // A saved query or ad-hoc SQL can carry `content MATCH ?` too, so the same FTS5\n    // punctuation trap applies -- the bound parameters are the search terms there. SQL\n    // without MATCH gets its error verbatim; search advice on a plain typo would mislead.\n    if (/\\bMATCH\\b/i.test(sql)) throw searchError(hinted, params.join(' '));\n    throw hinted;\n  }\n  await store.close();\n}\n"],"names":["parseArgs","resolvePreset","columnHint","printRowStream","searchError","openStore","FORMAT","format","type","default","CONFIG","config","SCOPE","where","preset","include","multiple","exclude","SEARCH_FLAGS","k","scopeOf","values","noExclude","pickFormat","allowed","String","includes","console","error","join","process","exit","formatOf","rowFormatOf","parse","argv","usage","options","positionals","args","help","short","strict","allowPositionals","err","message","log","parseK","usageError","undefined","parsed","Number","isInteger","printWarnings","warnings","w","warn","withDb","ctx","configPath","fn","cfg","resolveConfig","store","close","bindScope","name","exec","insert","prepare","run","runSql","sql","params","label","placeholderCount","match","length","test","statement","raw","columns","map","c","iterate","hinted","docs"],"mappings":"AACA,SAASA,SAAS,QAAQ,YAAY;AAEtC,SAASC,aAAa,QAAQ,qBAAqB;AACnD,SAASC,UAAU,QAAQ,2BAA2B;AAEtD,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,WAAW,QAAQ,4BAA4B;AAExD,SAASC,SAAS,QAAQ,oBAAoB;AAI9C,+FAA+F;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEC,MAAM;QAAUC,SAAS;IAAQ;AAAE,EAAE;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEH,MAAM;IAAS;AAAE,EAAE;AAC7E,yFAAyF;AACzF,sDAAsD;AACtD,OAAO,MAAMI,QAAgC;IAC3CC,OAAO;QAAEL,MAAM;IAAS;IACxBM,QAAQ;QAAEN,MAAM;IAAS;IACzBO,SAAS;QAAEP,MAAM;QAAUQ,UAAU;IAAK;IAC1CC,SAAS;QAAET,MAAM;QAAUQ,UAAU;IAAK;IAC1C,yFAAyF;IACzF,gFAAgF;IAChF,cAAc;QAAER,MAAM;QAAWC,SAAS;IAAM;AAClD,EAAE;AACF,OAAO,MAAMS,eAAuC;IAAE,GAAGN,KAAK;IAAEO,GAAG;QAAEX,MAAM;IAAS;AAAE,EAAE;AAIxF,4FAA4F;AAC5F,0FAA0F;AAC1F,kBAAkB;AAClB,OAAO,SAASY,QAAQC,MAAc;IACpC,OAAO;QACLP,QAAQO,OAAOP,MAAM;QACrBC,SAASM,OAAON,OAAO;QACvBE,SAASI,OAAOJ,OAAO;QACvBJ,OAAOQ,OAAOR,KAAK;QACnBS,WAAWD,MAAM,CAAC,aAAa,KAAK;IACtC;AACF;AAEA,0FAA0F;AAC1F,mFAAmF;AACnF,SAASE,WAA6BF,MAAc,EAAEG,OAAqB;IACzE,MAAMjB,SAASkB,OAAOJ,OAAOd,MAAM;IACnC,IAAI,AAACiB,QAA8BE,QAAQ,CAACnB,SAAS,OAAOA;IAC5DoB,QAAQC,KAAK,CAAC,CAAC,kBAAkB,EAAErB,OAAO,YAAY,EAAEiB,QAAQK,IAAI,CAAC,OAAO;IAC5EC,QAAQC,IAAI,CAAC;AACf;AAEA,OAAO,SAASC,SAASX,MAAc;IACrC,OAAOE,WAAWF,QAAQ;QAAC;QAAS;KAAO;AAC7C;AAEA,4FAA4F;AAC5F,qBAAqB;AACrB,OAAO,SAASY,YAAYZ,MAAc;IACxC,OAAOE,WAAWF,QAAQ;QAAC;QAAS;QAAQ;KAAM;AACpD;AAEA,gGAAgG;AAChG,OAAO,SAASa,MAAMC,IAAc,EAAEC,KAAa,EAAEC,OAA+B;IAClF,IAAIhB;IACJ,IAAIiB;IACJ,IAAI;QACD,CAAA,EAAEjB,MAAM,EAAEiB,WAAW,EAAE,GAAGtC,UAAU;YACnCuC,MAAMJ;YACNE,SAAS;gBAAE,GAAGA,OAAO;gBAAEG,MAAM;oBAAEhC,MAAM;oBAAWC,SAAS;oBAAOgC,OAAO;gBAAI;YAAE;YAC7EC,QAAQ;YACRC,kBAAkB;QACpB,EAAC;IACH,EAAE,OAAOC,KAAK;QACZjB,QAAQC,KAAK,CAAC,AAACgB,IAAcC,OAAO;QACpClB,QAAQC,KAAK,CAACQ;QACdN,QAAQC,IAAI,CAAC;IACf;IACA,IAAIV,OAAOmB,IAAI,EAAE;QACfb,QAAQmB,GAAG,CAACV;QACZN,QAAQC,IAAI,CAAC;IACf;IACA,OAAO;QAAEV;QAAQiB;IAAY;AAC/B;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,yEAAyE;AACzE,OAAO,SAASS,OAAO5B,CAAqB,EAAE6B,UAAsC;IAClF,IAAI7B,MAAM8B,WAAW,OAAOA;IAC5B,MAAMC,SAASC,OAAOhC;IACtB,IAAI,CAACgC,OAAOC,SAAS,CAACF,WAAWA,UAAU,GAAGF,WAAW,CAAC,qCAAqC,EAAE7B,EAAE,CAAC,CAAC;IACrG,OAAO+B;AACT;AAEA,qEAAqE;AAErE,OAAO,SAASG,cAAcC,QAAkB;IAC9C,KAAK,MAAMC,KAAKD,SAAU3B,QAAQ6B,IAAI,CAACD;AACzC;AAEA,OAAO,eAAeE,OAAOC,GAAQ,EAAEC,UAA8B,EAAEC,EAA6E;IAClJ,MAAMC,MAAMH,IAAII,aAAa,CAACH;IAC9B,MAAM,EAAEI,KAAK,EAAET,QAAQ,EAAE,GAAG,MAAMjD,UAAUwD;IAC5CR,cAAcC;IACd,IAAI;QACF,MAAMM,GAAGG,OAAOF;IAClB,SAAU;QACR,MAAME,MAAMC,KAAK;IACnB;AACF;AAEA,yFAAyF;AACzF,gGAAgG;AAChG,8FAA8F;AAC9F,2FAA2F;AAC3F,gCAAgC;AAChC,eAAeC,UAAUF,KAAY,EAAEF,GAAmB,EAAE/C,MAAc;IACxE,MAAM,EAAEoD,IAAI,EAAE,GAAGjE,cAAc4D,KAAK/C,SAAS,gDAAgD;IAC7F,MAAMiD,MAAMI,IAAI,CAAC;IACjB,MAAMJ,MAAMI,IAAI,CAAC;IACjB,MAAMC,SAAS,MAAML,MAAMM,OAAO,CAAC;IACnC,MAAMD,OAAOE,GAAG,CAACJ;AACnB;AAEA,sFAAsF;AACtF,OAAO,eAAeK,OAAOV,GAAmB,EAAEW,GAAW,EAAEC,MAAgB,EAAElE,MAAiB,EAAEmE,KAAa,EAAE5D,MAAe;QACtG0D;IAA1B,MAAMG,mBAAmB,EAACH,aAAAA,IAAII,KAAK,CAAC,oBAAVJ,wBAAAA,aAAoB,EAAE,EAAEK,MAAM;IACxD,IAAIJ,OAAOI,MAAM,KAAKF,kBAAkB;QACtChD,QAAQC,KAAK,CAAC,GAAG8C,MAAM,SAAS,EAAEC,iBAAiB,mBAAmB,EAAEF,OAAOI,MAAM,EAAE;QACvF/C,QAAQC,IAAI,CAAC;IACf;IACA,6FAA6F;IAC7F,2EAA2E;IAC3E,IAAIjB,WAAWmC,aAAa,CAAC,aAAa6B,IAAI,CAACN,MAAM;QACnD7C,QAAQC,KAAK,CAAC,CAAC,oEAAoE,EAAE8C,MAAM,mDAAmD,CAAC;QAC/I/C,QAAQC,KAAK,CAAC,CAAC,gDAAgD,CAAC;QAChEE,QAAQC,IAAI,CAAC;IACf;IACA,MAAM,EAAEgC,KAAK,EAAET,QAAQ,EAAE,GAAG,MAAMjD,UAAUwD;IAC5CR,cAAcC;IACd,IAAIxC,WAAWmC,WAAW,MAAMgB,UAAUF,OAAOF,KAAK/C;IACtD,mFAAmF;IACnF,qFAAqF;IACrF,qFAAqF;IACrF,uFAAuF;IACvF,2FAA2F;IAC3F,yBAAyB;IACzB,IAAI;QACF,MAAMiE,YAAY,MAAMhB,MAAMiB,GAAG,CAACX,OAAO,CAACG;QAC1C,MAAMS,UAAUF,UAAUE,OAAO,GAAGC,GAAG,CAAC,CAACC,IAAMA,EAAEjB,IAAI;QACrD,MAAM/D,eAAe4E,UAAUK,OAAO,IAAIX,SAA+BlE,QAAQ0E;IACnF,EAAE,OAAOrC,KAAK;QACZ,MAAMyC,SAASnF,WAAW,MAAM6D,MAAMuB,IAAI,CAACL,OAAO,IAAIrC,MAAe,6CAA6C;QAClH,MAAMmB,MAAMC,KAAK;QACjB,gFAAgF;QAChF,mFAAmF;QACnF,sFAAsF;QACtF,IAAI,aAAac,IAAI,CAACN,MAAM,MAAMpE,YAAYiF,QAAQZ,OAAO5C,IAAI,CAAC;QAClE,MAAMwD;IACR;IACA,MAAMtB,MAAMC,KAAK;AACnB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/shared.ts"],"sourcesContent":["import type { ParseArgsOptionsConfig } from 'node:util';\nimport { parseArgs } from 'node:util';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { resolvePreset } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\nimport type { Row, RowFormat } from '../output/output.ts';\nimport { printRowStream } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport type { OpenResult } from '../store/index.ts';\nimport { openStore } from '../store/index.ts';\nimport type { Store } from '../store/types.ts';\nimport type { Ctx } from './types.ts';\n\n// Spreadable option fragments -- one flag name keeps one meaning across every command's table.\nexport const FORMAT: ParseArgsOptionsConfig = { format: { type: 'string', default: 'table' } };\nexport const CONFIG: ParseArgsOptionsConfig = { config: { type: 'string' } };\n// The scope vocabulary every scoped command shares: named preset, ad hoc include/exclude\n// globs, a where SQL condition. search adds k on top.\nexport const SCOPE: ParseArgsOptionsConfig = {\n  where: { type: 'string' },\n  preset: { type: 'string' },\n  include: { type: 'string', multiple: true },\n  exclude: { type: 'string', multiple: true },\n  // Widening, which no other flag can express: --include and --exclude each override their\n  // own side of the preset, so neither can drop an exclusion the preset declares.\n  'no-exclude': { type: 'boolean', default: false },\n};\nexport const SEARCH_FLAGS: ParseArgsOptionsConfig = { ...SCOPE, k: { type: 'string' } };\n\ntype Values = Record<string, string | boolean | string[] | undefined>;\n\n// The SCOPE fragment's parsed values as the overrides every scoped command passes down.\n// One place to read them: a new scope field is added to the fragment and here, not per-command.\nexport function scopeOf(values: Values): SearchOverrides {\n  return {\n    preset: values.preset as string | undefined,\n    include: values.include as string[] | undefined,\n    exclude: values.exclude as string[] | undefined,\n    where: values.where as string | undefined,\n    noExclude: values['no-exclude'] === true,\n  };\n}\n\n// parseArgs is strict about flag names; this applies the same strictness to the value.\n// An unrecognised --format is a usage error, not a silent fallback to `table`.\nfunction pickFormat<T extends string>(values: Values, allowed: readonly T[]): T {\n  const format = String(values.format);\n  if ((allowed as readonly string[]).includes(format)) return format as T;\n  console.error(`unknown --format \"${format}\"; expected ${allowed.join(', ')}`);\n  process.exit(2);\n}\n\nexport function formatOf(values: Values): 'table' | 'json' {\n  return pickFormat(values, ['table', 'json'] as const);\n}\n\n// csv is a row set rendered as rows; map, peek, status, and path render structures instead,\n// and take formatOf.\nexport function rowFormatOf(values: Values): RowFormat {\n  return pickFormat(values, ['table', 'json', 'csv'] as const);\n}\n\n// Per-command parseArgs: strict (a foreign flag exits 2), and every table gets --help for free.\nexport function parse(argv: string[], usage: string, options: ParseArgsOptionsConfig): { values: Values; positionals: string[] } {\n  let values: Values;\n  let positionals: string[];\n  try {\n    ({ values, positionals } = parseArgs({\n      args: argv,\n      options: { ...options, help: { type: 'boolean', default: false, short: 'h' } },\n      strict: true,\n      allowPositionals: true,\n    }));\n  } catch (err) {\n    console.error((err as Error).message);\n    console.error(usage);\n    process.exit(2);\n  }\n  if (values.help) {\n    console.log(usage);\n    process.exit(0);\n  }\n  return { values, positionals };\n}\n\n// --k must be a positive integer: SQLite reads a bound LIMIT of -1 as \"unlimited\" and 0 as \"nothing\", and parseInt would silently truncate \"5.9\".\n// All three are caller mistakes worth a usage error, matching the config-level SavedSearch validation.\nexport function parseK(k: string | undefined, usageError: (message: string) => never): number | undefined {\n  if (k === undefined) return undefined;\n  const parsed = Number(k);\n  if (!Number.isInteger(parsed) || parsed <= 0) usageError(`--k expects a positive integer, got \"${k}\"`);\n  return parsed;\n}\n\n// Shared open-query-close envelope for commands that touch the tree.\n\nexport function printWarnings(warnings: string[]): void {\n  for (const w of warnings) console.warn(w);\n}\n\nexport async function withDb(ctx: Ctx, configPath: string | undefined, fn: (store: OpenResult['store'], cfg: ResolvedConfig) => void | Promise<void>): Promise<void> {\n  const cfg = ctx.resolveConfig(configPath);\n  const { store, warnings } = await openStore(cfg);\n  printWarnings(warnings);\n  try {\n    await fn(store, cfg);\n  } finally {\n    await store.close();\n  }\n}\n\n// `--preset` on SQL binds a temp `scope` table rather than transparently filtering: temp views shadowing the base tables can't cover `content`, since FTS5 MATCH treats the table name as a hidden column.\n// A flag that silently scoped three tables of four would look scoped and not be.\nasync function bindScope(store: Store, cfg: ResolvedConfig, preset: string): Promise<void> {\n  const { name } = resolvePreset(cfg, preset); // unknown names throw, listing what is declared\n  await store.exec('DROP TABLE IF EXISTS temp.scope');\n  await store.exec('CREATE TEMP TABLE scope (\"path\" TEXT PRIMARY KEY)');\n  const insert = await store.prepare('INSERT INTO temp.scope (\"path\") SELECT \"path\" FROM preset_files WHERE preset = ?');\n  await insert.run(name);\n}\n\n// An unbound `?` silently binds NULL, so mismatched param counts fail loudly instead.\nexport async function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: RowFormat, label: string, preset?: string): Promise<void> {\n  const placeholderCount = (sql.match(/\\?/g) ?? []).length;\n  if (params.length !== placeholderCount) {\n    console.error(`${label} expects ${placeholderCount} parameter(s), got ${params.length}`);\n    process.exit(2);\n  }\n  // Naming a preset and never joining it would return the whole index while reading as scoped,\n  // which is the one hazard of binding rather than applying. Refuse instead.\n  if (preset !== undefined && !/\\bscope\\b/i.test(sql)) {\n    console.error(`--preset binds a temporary \"scope\" table of the preset's paths, and ${label} never joins it, so the preset would have no effect`);\n    console.error(`add: JOIN scope ON scope.\"path\" = <table>.\"path\"`);\n    process.exit(2);\n  }\n  const { store, warnings } = await openStore(cfg);\n  printWarnings(warnings);\n  if (preset !== undefined) await bindScope(store, cfg, preset);\n  // Streamed, not collected: `sql`'s result size is the caller's business, not ours. columns() reads the statement's own metadata, so a 0-row csv gets a header too.\n  // A mid-stream SQLite error (e.g. SQLITE_BUSY outlasting busy_timeout) may leave partial output; the nonzero exit code is the failure signal, not a completeness guarantee.\n  try {\n    const statement = await store.raw.prepare(sql);\n    const columns = statement.columns().map((c) => c.name);\n    await printRowStream(statement.iterate(...params) as AsyncIterable<Row>, format, columns);\n  } catch (err) {\n    const hinted = columnHint(await store.docs.columns(), err as Error); // columns read while the store is still open\n    await store.close();\n    // A saved query or ad-hoc SQL can carry `content MATCH ?` too, so the same FTS5 punctuation trap applies to its bound parameters.\n    // SQL without MATCH gets its error verbatim; search advice on a plain typo would mislead.\n    if (/\\bMATCH\\b/i.test(sql)) throw searchError(hinted, params.join(' '));\n    throw hinted;\n  }\n  await store.close();\n}\n"],"names":["parseArgs","resolvePreset","columnHint","printRowStream","searchError","openStore","FORMAT","format","type","default","CONFIG","config","SCOPE","where","preset","include","multiple","exclude","SEARCH_FLAGS","k","scopeOf","values","noExclude","pickFormat","allowed","String","includes","console","error","join","process","exit","formatOf","rowFormatOf","parse","argv","usage","options","positionals","args","help","short","strict","allowPositionals","err","message","log","parseK","usageError","undefined","parsed","Number","isInteger","printWarnings","warnings","w","warn","withDb","ctx","configPath","fn","cfg","resolveConfig","store","close","bindScope","name","exec","insert","prepare","run","runSql","sql","params","label","placeholderCount","match","length","test","statement","raw","columns","map","c","iterate","hinted","docs"],"mappings":"AACA,SAASA,SAAS,QAAQ,YAAY;AAEtC,SAASC,aAAa,QAAQ,qBAAqB;AACnD,SAASC,UAAU,QAAQ,2BAA2B;AAEtD,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,WAAW,QAAQ,4BAA4B;AAExD,SAASC,SAAS,QAAQ,oBAAoB;AAI9C,+FAA+F;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEC,MAAM;QAAUC,SAAS;IAAQ;AAAE,EAAE;AAC/F,OAAO,MAAMC,SAAiC;IAAEC,QAAQ;QAAEH,MAAM;IAAS;AAAE,EAAE;AAC7E,yFAAyF;AACzF,sDAAsD;AACtD,OAAO,MAAMI,QAAgC;IAC3CC,OAAO;QAAEL,MAAM;IAAS;IACxBM,QAAQ;QAAEN,MAAM;IAAS;IACzBO,SAAS;QAAEP,MAAM;QAAUQ,UAAU;IAAK;IAC1CC,SAAS;QAAET,MAAM;QAAUQ,UAAU;IAAK;IAC1C,yFAAyF;IACzF,gFAAgF;IAChF,cAAc;QAAER,MAAM;QAAWC,SAAS;IAAM;AAClD,EAAE;AACF,OAAO,MAAMS,eAAuC;IAAE,GAAGN,KAAK;IAAEO,GAAG;QAAEX,MAAM;IAAS;AAAE,EAAE;AAIxF,wFAAwF;AACxF,gGAAgG;AAChG,OAAO,SAASY,QAAQC,MAAc;IACpC,OAAO;QACLP,QAAQO,OAAOP,MAAM;QACrBC,SAASM,OAAON,OAAO;QACvBE,SAASI,OAAOJ,OAAO;QACvBJ,OAAOQ,OAAOR,KAAK;QACnBS,WAAWD,MAAM,CAAC,aAAa,KAAK;IACtC;AACF;AAEA,uFAAuF;AACvF,+EAA+E;AAC/E,SAASE,WAA6BF,MAAc,EAAEG,OAAqB;IACzE,MAAMjB,SAASkB,OAAOJ,OAAOd,MAAM;IACnC,IAAI,AAACiB,QAA8BE,QAAQ,CAACnB,SAAS,OAAOA;IAC5DoB,QAAQC,KAAK,CAAC,CAAC,kBAAkB,EAAErB,OAAO,YAAY,EAAEiB,QAAQK,IAAI,CAAC,OAAO;IAC5EC,QAAQC,IAAI,CAAC;AACf;AAEA,OAAO,SAASC,SAASX,MAAc;IACrC,OAAOE,WAAWF,QAAQ;QAAC;QAAS;KAAO;AAC7C;AAEA,4FAA4F;AAC5F,qBAAqB;AACrB,OAAO,SAASY,YAAYZ,MAAc;IACxC,OAAOE,WAAWF,QAAQ;QAAC;QAAS;QAAQ;KAAM;AACpD;AAEA,gGAAgG;AAChG,OAAO,SAASa,MAAMC,IAAc,EAAEC,KAAa,EAAEC,OAA+B;IAClF,IAAIhB;IACJ,IAAIiB;IACJ,IAAI;QACD,CAAA,EAAEjB,MAAM,EAAEiB,WAAW,EAAE,GAAGtC,UAAU;YACnCuC,MAAMJ;YACNE,SAAS;gBAAE,GAAGA,OAAO;gBAAEG,MAAM;oBAAEhC,MAAM;oBAAWC,SAAS;oBAAOgC,OAAO;gBAAI;YAAE;YAC7EC,QAAQ;YACRC,kBAAkB;QACpB,EAAC;IACH,EAAE,OAAOC,KAAK;QACZjB,QAAQC,KAAK,CAAC,AAACgB,IAAcC,OAAO;QACpClB,QAAQC,KAAK,CAACQ;QACdN,QAAQC,IAAI,CAAC;IACf;IACA,IAAIV,OAAOmB,IAAI,EAAE;QACfb,QAAQmB,GAAG,CAACV;QACZN,QAAQC,IAAI,CAAC;IACf;IACA,OAAO;QAAEV;QAAQiB;IAAY;AAC/B;AAEA,kJAAkJ;AAClJ,uGAAuG;AACvG,OAAO,SAASS,OAAO5B,CAAqB,EAAE6B,UAAsC;IAClF,IAAI7B,MAAM8B,WAAW,OAAOA;IAC5B,MAAMC,SAASC,OAAOhC;IACtB,IAAI,CAACgC,OAAOC,SAAS,CAACF,WAAWA,UAAU,GAAGF,WAAW,CAAC,qCAAqC,EAAE7B,EAAE,CAAC,CAAC;IACrG,OAAO+B;AACT;AAEA,qEAAqE;AAErE,OAAO,SAASG,cAAcC,QAAkB;IAC9C,KAAK,MAAMC,KAAKD,SAAU3B,QAAQ6B,IAAI,CAACD;AACzC;AAEA,OAAO,eAAeE,OAAOC,GAAQ,EAAEC,UAA8B,EAAEC,EAA6E;IAClJ,MAAMC,MAAMH,IAAII,aAAa,CAACH;IAC9B,MAAM,EAAEI,KAAK,EAAET,QAAQ,EAAE,GAAG,MAAMjD,UAAUwD;IAC5CR,cAAcC;IACd,IAAI;QACF,MAAMM,GAAGG,OAAOF;IAClB,SAAU;QACR,MAAME,MAAMC,KAAK;IACnB;AACF;AAEA,2MAA2M;AAC3M,iFAAiF;AACjF,eAAeC,UAAUF,KAAY,EAAEF,GAAmB,EAAE/C,MAAc;IACxE,MAAM,EAAEoD,IAAI,EAAE,GAAGjE,cAAc4D,KAAK/C,SAAS,gDAAgD;IAC7F,MAAMiD,MAAMI,IAAI,CAAC;IACjB,MAAMJ,MAAMI,IAAI,CAAC;IACjB,MAAMC,SAAS,MAAML,MAAMM,OAAO,CAAC;IACnC,MAAMD,OAAOE,GAAG,CAACJ;AACnB;AAEA,sFAAsF;AACtF,OAAO,eAAeK,OAAOV,GAAmB,EAAEW,GAAW,EAAEC,MAAgB,EAAElE,MAAiB,EAAEmE,KAAa,EAAE5D,MAAe;QACtG0D;IAA1B,MAAMG,mBAAmB,EAACH,aAAAA,IAAII,KAAK,CAAC,oBAAVJ,wBAAAA,aAAoB,EAAE,EAAEK,MAAM;IACxD,IAAIJ,OAAOI,MAAM,KAAKF,kBAAkB;QACtChD,QAAQC,KAAK,CAAC,GAAG8C,MAAM,SAAS,EAAEC,iBAAiB,mBAAmB,EAAEF,OAAOI,MAAM,EAAE;QACvF/C,QAAQC,IAAI,CAAC;IACf;IACA,6FAA6F;IAC7F,2EAA2E;IAC3E,IAAIjB,WAAWmC,aAAa,CAAC,aAAa6B,IAAI,CAACN,MAAM;QACnD7C,QAAQC,KAAK,CAAC,CAAC,oEAAoE,EAAE8C,MAAM,mDAAmD,CAAC;QAC/I/C,QAAQC,KAAK,CAAC,CAAC,gDAAgD,CAAC;QAChEE,QAAQC,IAAI,CAAC;IACf;IACA,MAAM,EAAEgC,KAAK,EAAET,QAAQ,EAAE,GAAG,MAAMjD,UAAUwD;IAC5CR,cAAcC;IACd,IAAIxC,WAAWmC,WAAW,MAAMgB,UAAUF,OAAOF,KAAK/C;IACtD,mKAAmK;IACnK,4KAA4K;IAC5K,IAAI;QACF,MAAMiE,YAAY,MAAMhB,MAAMiB,GAAG,CAACX,OAAO,CAACG;QAC1C,MAAMS,UAAUF,UAAUE,OAAO,GAAGC,GAAG,CAAC,CAACC,IAAMA,EAAEjB,IAAI;QACrD,MAAM/D,eAAe4E,UAAUK,OAAO,IAAIX,SAA+BlE,QAAQ0E;IACnF,EAAE,OAAOrC,KAAK;QACZ,MAAMyC,SAASnF,WAAW,MAAM6D,MAAMuB,IAAI,CAACL,OAAO,IAAIrC,MAAe,6CAA6C;QAClH,MAAMmB,MAAMC,KAAK;QACjB,kIAAkI;QAClI,0FAA0F;QAC1F,IAAI,aAAac,IAAI,CAACN,MAAM,MAAMpE,YAAYiF,QAAQZ,OAAO5C,IAAI,CAAC;QAClE,MAAMwD;IACR;IACA,MAAMtB,MAAMC,KAAK;AACnB"}

package/dist/esm/cli/status.js

@@ -8,13 +8,10 @@ import { featuresLine, presetsLines, stringifyJson } from '../output/output.js';
 import { docCount, getMeta, openStore } from '../store/index.js';
 import { USAGE } from './index.js';
 import { CONFIG, FORMAT, formatOf, parse, printWarnings } from './shared.js';
-// status names what you cannot read off the config file: locations sense chose, values it
-// derived, state it stored. Config content is not echoed -- the config line says which file to
-// read, and duplicating its globs here would create two places that disagree after an edit.
+// status names what you cannot read off the config file: locations sense chose, values it derived, state it stored. Config content itself is not echoed, since duplicating its globs here would create two places that disagree after an edit.
 // The exception is the embed block's url and key, whose effect is remote and otherwise invisible.
-// Vectors are computed on the first semantic query, not at reconcile, so a tree can have a
-// semantic-on preset with nothing embedded yet -- reported here because a fast reconcile on
-// such a tree otherwise reads as "the feature isn't on".
+// Vectors are computed on the first semantic query, not at reconcile, so a tree can have a semantic-on preset with nothing embedded yet.
+// Reported here because a fast reconcile on such a tree otherwise reads as "the feature isn't on".
 async function vectorState(store) {
     try {
         const stmt = await store.prepare('SELECT COUNT(vector) AS embedded, COUNT(*) - COUNT(vector) AS pending FROM embeddings');
@@ -48,6 +45,36 @@ async function embedReachable(e) {
     const base = ((_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : 'https://api.cohere.com').replace(/\/+$/, '');
     return probeReachable(e.provider === 'openai' ? `${base}/models` : base);
 }
+// Config-declared languages win over the model card, for every provider, as registry.ts does.
+// A local-path model has no card by design, so it is never "unresolved": that means an HF id.
+function languagesInfo(e) {
+    if (e.languages) return {
+        languages: e.languages,
+        state: 'declared',
+        source: 'config'
+    };
+    if (e.provider !== 'static' || !isDownloadable(e.model)) return {
+        languages: null,
+        state: null,
+        source: null
+    };
+    const cardLanguages = readLanguages(e.model);
+    if (cardLanguages === undefined) return {
+        languages: null,
+        state: 'unresolved',
+        source: null
+    };
+    if (cardLanguages.length === 0) return {
+        languages: null,
+        state: 'none',
+        source: null
+    };
+    return {
+        languages: cardLanguages,
+        state: 'declared',
+        source: 'card'
+    };
+}
 const status = async (ctx)=>{
     var _ref, _process_env_e_key, _cfg_migratedFrom, _cfg_unknownKeys, _cfg_queries;
     const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.status}`, {
@@ -64,8 +91,11 @@ const status = async (ctx)=>{
     const reachable = e ? await embedReachable(e) : null;
     const vectors = e ? await vectorState(store) : null;
     const presets = await presetCoverage(store, cfg);
-    const staticLanguages = e && e.provider === 'static' ? readLanguages(e.model) : undefined;
-    const declaredLanguages = staticLanguages && staticLanguages.length > 0 ? staticLanguages : null;
+    const { languages: declaredLanguages, state: languagesState, source: languagesSource } = e ? languagesInfo(e) : {
+        languages: null,
+        state: null,
+        source: null
+    };
     const detectedLanguages = (_ref = await languageDistribution(store)) !== null && _ref !== void 0 ? _ref : null;
     const heartbeat = await getMeta(store, 'watch_heartbeat');
     // Written by the store's open() on every open, so it is this cache's true version on either
@@ -101,6 +131,7 @@ const status = async (ctx)=>{
             keyEnv,
             downloaded: hasModel,
             languages: declaredLanguages,
+            languagesState,
             detectedLanguages,
             ...vectors !== null && vectors !== void 0 ? vectors : {}
         } : null,
@@ -128,7 +159,10 @@ const status = async (ctx)=>{
         else {
             console.log(`embed:    ${e.provider} ${e.model}`);
             if (e.provider === 'static') console.log(`          model:   ${modelDir(e.model)} (${hasModel ? 'present' : `missing, ${isDownloadable(e.model) ? `fetches automatically at first search, or run \`${ctx.name} download\` to prefetch` : `no ${MODEL_FILENAMES} in that directory`}`})`);
-            if (declaredLanguages) console.log(`          languages: ${declaredLanguages.join(', ')} (declared by the model card)`);
+            if (declaredLanguages && languagesSource === 'config') console.log(`          languages: ${declaredLanguages.join(', ')} (declared in embed.languages)`);
+            else if (declaredLanguages && languagesSource === 'card') console.log(`          languages: ${declaredLanguages.join(', ')} (declared by the model card)`);
+            else if (languagesState === 'none') console.log('          languages: none declared by the model card');
+            else if (languagesState === 'unresolved') console.log(`          languages: unresolved (model card unreachable; run \`${ctx.name} download\`)`);
             // Cohere defaults to api.cohere.com with no url in the config; show it anyway so the
             // reachability probe below has something to name.
             if (e.url || reachable !== null) {