sensemaking 0.21.1 → 0.22.1

package/dist/cjs/watch.js

@@ -144,93 +144,186 @@ var HEARTBEAT_INTERVAL_MS = 5000;
 var STALE_HEARTBEAT_MS = 15000;
 function runWatch(_0) {
     return _async_to_generator(function(cfg) {
-        var opts, _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs, onEvent, debounceMs, heartbeatIntervalMs, _ref, store, builder, dbPath, initialWarnings, initialParsed, baseDir, existingHeartbeat, age, _tmp, stopping, touchHeartbeat, debounceTimer, inFlight, scheduleReconcile, watcher, heartbeatTimer;
+        var opts, _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs, onEvent, debounceMs, heartbeatIntervalMs, baseDir, _ref, initialStore, dbPath, initialWarnings, initialParsed, existingHeartbeat, age, initialTotal, _tmp, stopping, running, queuedAlways, tick, startCycle, requestCycle, debounceTimer, scheduleReconcile, watcher, heartbeatTimer;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
                     opts = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : {};
-                    (0, _indexts1.requireWatchConcurrency)(cfg);
                     onEvent = (_opts_onEvent = opts.onEvent) !== null && _opts_onEvent !== void 0 ? _opts_onEvent : function() {};
                     debounceMs = (_opts_debounceMs = opts.debounceMs) !== null && _opts_debounceMs !== void 0 ? _opts_debounceMs : DEBOUNCE_MS;
                     heartbeatIntervalMs = (_opts_heartbeatIntervalMs = opts.heartbeatIntervalMs) !== null && _opts_heartbeatIntervalMs !== void 0 ? _opts_heartbeatIntervalMs : HEARTBEAT_INTERVAL_MS;
+                    baseDir = cfg.baseDir;
                     return [
                         4,
-                        (0, _indexts1.openStoreForWatch)(cfg)
+                        (0, _indexts1.openStore)(cfg)
                     ];
                 case 1:
-                    _ref = _state.sent(), store = _ref.store, builder = _ref.builder, dbPath = _ref.dbPath, initialWarnings = _ref.warnings, initialParsed = _ref.parsed;
-                    baseDir = cfg.baseDir;
+                    _ref = _state.sent(), initialStore = _ref.store, dbPath = _ref.dbPath, initialWarnings = _ref.warnings, initialParsed = _ref.parsed;
                     return [
                         4,
-                        (0, _indexts1.getMeta)(store, 'watch_heartbeat')
+                        (0, _indexts1.getMeta)(initialStore, 'watch_heartbeat')
                     ];
                 case 2:
                     existingHeartbeat = _state.sent();
                     if (!(existingHeartbeat && !opts.force)) return [
                         3,
-                        5
+                        4
                     ];
                     age = Date.now() - Date.parse(existingHeartbeat);
                     if (!(age >= 0 && age < STALE_HEARTBEAT_MS)) return [
                         3,
-                        5
+                        4
                     ];
                     return [
                         4,
-                        builder.close()
+                        initialStore.close()
                     ];
                 case 3:
-                    _state.sent();
-                    return [
-                        4,
-                        store.close()
-                    ];
-                case 4:
                     _state.sent();
                     throw new _errorsts.SenseError('WATCH_ACTIVE', "another watcher appears active (heartbeat ".concat(Math.round(age / 1000), "s ago); use --force to override"));
-                case 5:
-                    onEvent({
-                        type: 'started',
-                        baseDir: baseDir,
-                        dbPath: dbPath
-                    });
+                case 4:
                     if (!(initialWarnings.length > 0 || initialParsed > 0)) return [
                         3,
-                        7
+                        6
                     ];
-                    _tmp = {
-                        type: 'reconciled',
-                        parsed: initialParsed
-                    };
                     return [
                         4,
-                        (0, _indexts1.docCount)(store)
+                        (0, _indexts1.docCount)(initialStore)
+                    ];
+                case 5:
+                    _tmp = _state.sent();
+                    return [
+                        3,
+                        7
                     ];
                 case 6:
-                    onEvent.apply(void 0, [
-                        (_tmp.total = _state.sent(), _tmp.warnings = initialWarnings, _tmp)
-                    ]);
+                    _tmp = 0;
                     _state.label = 7;
                 case 7:
+                    initialTotal = _tmp;
+                    return [
+                        4,
+                        (0, _indexts1.setMeta)(initialStore, 'watch_heartbeat', new Date().toISOString())
+                    ];
+                case 8:
+                    _state.sent();
+                    return [
+                        4,
+                        (0, _indexts1.setMeta)(initialStore, 'watch_pid', String(process.pid))
+                    ];
+                case 9:
+                    _state.sent();
+                    return [
+                        4,
+                        initialStore.close()
+                    ];
+                case 10:
+                    _state.sent();
+                    onEvent({
+                        type: 'started',
+                        baseDir: baseDir,
+                        dbPath: dbPath
+                    });
+                    if (initialWarnings.length > 0 || initialParsed > 0) {
+                        onEvent({
+                            type: 'reconciled',
+                            parsed: initialParsed,
+                            total: initialTotal,
+                            warnings: initialWarnings
+                        });
+                    }
                     stopping = false;
-                    touchHeartbeat = function touchHeartbeat() {
+                    // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is
+                    // coalesced into a single rerun instead of opening a second, overlapping connection.
+                    running = null;
+                    queuedAlways = null;
+                    tick = function tick(alwaysEmit) {
                         return _async_to_generator(function() {
+                            var _ref, store, parsed, warnings, _tmp, err;
                             return _ts_generator(this, function(_state) {
                                 switch(_state.label){
                                     case 0:
+                                        _state.trys.push([
+                                            0,
+                                            10,
+                                            ,
+                                            11
+                                        ]);
                                         return [
                                             4,
-                                            (0, _indexts1.setMeta)(store, 'watch_heartbeat', new Date().toISOString())
+                                            (0, _indexts1.openStore)(cfg)
                                         ];
                                     case 1:
+                                        _ref = _state.sent(), store = _ref.store, parsed = _ref.parsed, warnings = _ref.warnings;
+                                        _state.label = 2;
+                                    case 2:
+                                        _state.trys.push([
+                                            2,
+                                            ,
+                                            7,
+                                            9
+                                        ]);
+                                        return [
+                                            4,
+                                            (0, _indexts1.setMeta)(store, 'watch_heartbeat', new Date().toISOString())
+                                        ];
+                                    case 3:
                                         _state.sent();
                                         return [
                                             4,
                                             (0, _indexts1.setMeta)(store, 'watch_pid', String(process.pid))
                                         ];
-                                    case 2:
+                                    case 4:
                                         _state.sent();
+                                        if (!(alwaysEmit || parsed > 0 || warnings.length > 0)) return [
+                                            3,
+                                            6
+                                        ];
+                                        _tmp = {
+                                            type: 'reconciled',
+                                            parsed: parsed
+                                        };
+                                        return [
+                                            4,
+                                            (0, _indexts1.docCount)(store)
+                                        ];
+                                    case 5:
+                                        onEvent.apply(void 0, [
+                                            (_tmp.total = _state.sent(), _tmp.warnings = warnings, _tmp)
+                                        ]);
+                                        _state.label = 6;
+                                    case 6:
+                                        return [
+                                            3,
+                                            9
+                                        ];
+                                    case 7:
+                                        return [
+                                            4,
+                                            store.close()
+                                        ];
+                                    case 8:
+                                        _state.sent();
+                                        return [
+                                            7
+                                        ];
+                                    case 9:
+                                        return [
+                                            3,
+                                            11
+                                        ];
+                                    case 10:
+                                        err = _state.sent();
+                                        onEvent({
+                                            type: 'reconcile-error',
+                                            message: err.message
+                                        });
+                                        return [
+                                            3,
+                                            11
+                                        ];
+                                    case 11:
                                         return [
                                             2
                                         ];
@@ -238,77 +331,30 @@ function runWatch(_0) {
                             });
                         })();
                     };
-                    return [
-                        4,
-                        touchHeartbeat()
-                    ];
-                case 8:
-                    _state.sent();
+                    startCycle = function startCycle1(alwaysEmit) {
+                        running = tick(alwaysEmit).then(function() {
+                            running = null;
+                            if (queuedAlways !== null && !stopping) {
+                                var again = queuedAlways;
+                                queuedAlways = null;
+                                startCycle(again);
+                            }
+                        });
+                    };
+                    requestCycle = function requestCycle(alwaysEmit) {
+                        if (stopping) return;
+                        if (running) {
+                            queuedAlways = queuedAlways || alwaysEmit;
+                            return;
+                        }
+                        startCycle(alwaysEmit);
+                    };
                     debounceTimer = null;
-                    // A reconcile that has already started owns the store, and on a bulk reparse a live worker
-                    // pool as well. Shutdown waits on this rather than closing the connection underneath it.
-                    inFlight = null;
                     scheduleReconcile = function scheduleReconcile() {
                         if (debounceTimer) clearTimeout(debounceTimer);
                         debounceTimer = setTimeout(function() {
                             debounceTimer = null;
-                            inFlight = function() {
-                                return _async_to_generator(function() {
-                                    var _ref, parsed, warnings, _tmp, err;
-                                    return _ts_generator(this, function(_state) {
-                                        switch(_state.label){
-                                            case 0:
-                                                _state.trys.push([
-                                                    0,
-                                                    3,
-                                                    4,
-                                                    5
-                                                ]);
-                                                return [
-                                                    4,
-                                                    builder.build()
-                                                ];
-                                            case 1:
-                                                _ref = _state.sent(), parsed = _ref.parsed, warnings = _ref.warnings;
-                                                _tmp = {
-                                                    type: 'reconciled',
-                                                    parsed: parsed
-                                                };
-                                                return [
-                                                    4,
-                                                    (0, _indexts1.docCount)(store)
-                                                ];
-                                            case 2:
-                                                onEvent.apply(void 0, [
-                                                    (_tmp.total = _state.sent(), _tmp.warnings = warnings, _tmp)
-                                                ]);
-                                                return [
-                                                    3,
-                                                    5
-                                                ];
-                                            case 3:
-                                                err = _state.sent();
-                                                onEvent({
-                                                    type: 'reconcile-error',
-                                                    message: err.message
-                                                });
-                                                return [
-                                                    3,
-                                                    5
-                                                ];
-                                            case 4:
-                                                inFlight = null;
-                                                return [
-                                                    7
-                                                ];
-                                            case 5:
-                                                return [
-                                                    2
-                                                ];
-                                        }
-                                    });
-                                })();
-                            }();
+                            requestCycle(true);
                         }, debounceMs);
                     };
                     // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with
@@ -319,7 +365,16 @@ function runWatch(_0) {
                         if (typeof filename === 'string' && filename.startsWith(_indexts.STATE_DIR)) return;
                         scheduleReconcile();
                     });
-                    heartbeatTimer = setInterval((0, _guardedtickts.guardedTick)(touchHeartbeat, function() {
+                    heartbeatTimer = setInterval((0, _guardedtickts.guardedTick)(function() {
+                        return _async_to_generator(function() {
+                            return _ts_generator(this, function(_state) {
+                                return [
+                                    2,
+                                    requestCycle(false)
+                                ];
+                            });
+                        })();
+                    }, function() {
                         return stopping;
                     }), heartbeatIntervalMs);
                     return [
@@ -330,7 +385,7 @@ function runWatch(_0) {
                             // unregistered here too so a second runWatch call in the same process starts clean.
                             var shutdown = function shutdown1() {
                                 return _async_to_generator(function() {
-                                    var _opts_signal;
+                                    var _opts_signal, _ref, finalStore;
                                     return _ts_generator(this, function(_state) {
                                         switch(_state.label){
                                             case 0:
@@ -346,31 +401,31 @@ function runWatch(_0) {
                                                 watcher.close();
                                                 return [
                                                     4,
-                                                    inFlight
+                                                    running
                                                 ];
                                             case 1:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    builder.close()
+                                                    (0, _indexts1.openStore)(cfg)
                                                 ];
                                             case 2:
-                                                _state.sent();
+                                                _ref = _state.sent(), finalStore = _ref.store;
                                                 return [
                                                     4,
-                                                    (0, _indexts1.setMeta)(store, 'watch_heartbeat', null)
+                                                    (0, _indexts1.setMeta)(finalStore, 'watch_heartbeat', null)
                                                 ];
                                             case 3:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    (0, _indexts1.setMeta)(store, 'watch_pid', null)
+                                                    (0, _indexts1.setMeta)(finalStore, 'watch_pid', null)
                                                 ];
                                             case 4:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    store.close()
+                                                    finalStore.close()
                                                 ];
                                             case 5:
                                                 _state.sent();

package/dist/cjs/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStoreForWatch, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, builder, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStoreForWatch(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await builder.close();\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await builder.build();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await builder.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","builder","dbPath","initialWarnings","initialParsed","baseDir","existingHeartbeat","age","stopping","touchHeartbeat","debounceTimer","inFlight","scheduleReconcile","watcher","heartbeatTimer","requireWatchConcurrency","openStoreForWatch","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","type","length","docCount","total","setMeta","toISOString","String","process","pid","clearTimeout","setTimeout","err","build","message","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;uBAEP;wBACC;6BACC;wBAC2D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvF,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAcpB,SAAeH;wCAASI,GAAmB;YAAEC,MAElCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBAC+E,MAA7EC,OAAOC,SAASC,QAAkBC,iBAAyBC,eAC7DC,SAEAC,mBAEEC,WAaJC,UACEC,gBAMFC,eAGAC,UACEC,mBAmBAC,SAIAC;;;;;oBAzD4ClB,OAAAA,oEAAqB,CAAC;oBACxEmB,IAAAA,iCAAuB,EAACpB;oBAClBE,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBAC6B;;wBAAMuB,IAAAA,2BAAiB,EAACrB;;;oBAAxB,OAAA,eAA7EK,QAA6E,KAA7EA,OAAOC,UAAsE,KAAtEA,SAASC,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDc,UAAmCb,gBAAkB,KAA1Bc;oBACrDb,UAAUV,IAAIU,OAAO;oBAED;;wBAAMc,IAAAA,iBAAO,EAACnB,OAAO;;;oBAAzCM,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACV,KAAKwB,KAAK,AAAD,GAA/Bd;;;;oBACIC,MAAMc,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACjB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMb,kBAAiB,GAAnCa;;;;oBACF;;wBAAMN,QAAQuB,KAAK;;;oBAAnB;oBACA;;wBAAMxB,MAAMwB,KAAK;;;oBAAjB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACpB,MAAM,OAAM;;oBAI7GV,QAAQ;wBAAE+B,MAAM;wBAAWvB,SAAAA;wBAASH,QAAAA;oBAAO;yBACvCC,CAAAA,gBAAgB0B,MAAM,GAAG,KAAKzB,gBAAgB,CAAA,GAA9CD;;;;;wBACQyB,MAAM;wBAAcV,QAAQd;;oBAAsB;;wBAAM0B,IAAAA,kBAAQ,EAAC9B;;;oBAA3EH;8BAAqDkC,QAAO,oBAAuBd,WAAUd;;;;oBAG3FK,WAAW;oBACTC,iBAAiB;;;;;wCACrB;;4CAAMuB,IAAAA,iBAAO,EAAChC,OAAO,mBAAmB,IAAIqB,OAAOY,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAAChC,OAAO,aAAakC,OAAOC,QAAQC,GAAG;;;wCAApD;;;;;;wBACF;;oBACA;;wBAAM3B;;;oBAAN;oBAEIC,gBAAuC;oBAC3C,2FAA2F;oBAC3F,yFAAyF;oBACrFC,WAAiC;oBAC/BC,oBAAoB;wBACxB,IAAIF,eAAe2B,aAAa3B;wBAChCA,gBAAgB4B,WAAW;4BACzB5B,gBAAgB;4BAChBC,WAAW,AAAC;;wCAEqB,MAArBO,QAAQD,gBAETsB;;;;;;;;;;gDAFsB;;oDAAMtC,QAAQuC,KAAK;;;gDAAnB,OAAA,eAArBtB,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;oDACNW,MAAM;oDAAcV,QAAAA;;gDAAe;;oDAAMY,IAAAA,kBAAQ,EAAC9B;;;gDAA5DH;0DAAsCkC,QAAO,oBAAuBd,WAAAA;;;;;;;gDAC7DsB;gDACP1C,QAAQ;oDAAE+B,MAAM;oDAAmBa,SAAS,AAACF,IAAcE,OAAO;gDAAC;;;;;;gDAEnE9B,WAAW;;;;;;;;;;gCAEf;;wBACF,GAAGb;oBACL;oBAEA,iGAAiG;oBACjG,uIAAuI;oBACjIe,UAAU6B,IAAAA,aAAO,EAACrC,SAAS;wBAAEsC,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpEnC;oBACF;oBACME,iBAAiBkC,YACrBC,IAAAA,0BAAW,EAACxC,gBAAgB;+BAAMD;wBAClCT;oBAGF;;wBAAO,IAAImD,QAAc,SAACC;gCAqBpBvD,cACCA;4BArBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAMwD,WAAW;;wCAKfxD;;;;gDAJA,IAAIY,UAAU;;;gDACdA,WAAW;gDACX2B,QAAQkB,GAAG,CAAC,UAAUD;gDACtBjB,QAAQkB,GAAG,CAAC,WAAWD;iDACvBxD,eAAAA,KAAK0D,MAAM,cAAX1D,mCAAAA,aAAa2D,mBAAmB,CAAC,SAASH;gDAC1CI,cAAc1C;gDACd,IAAIJ,eAAe2B,aAAa3B;gDAChCG,QAAQW,KAAK;gDACb;;oDAAMb;;;gDAAN;gDACA;;oDAAMV,QAAQuB,KAAK;;;gDAAnB;gDACA;;oDAAMQ,IAAAA,iBAAO,EAAChC,OAAO,mBAAmB;;;gDAAxC;gDACA;;oDAAMgC,IAAAA,iBAAO,EAAChC,OAAO,aAAa;;;gDAAlC;gDACA;;oDAAMA,MAAMwB,KAAK;;;gDAAjB;gDACA2B;;;;;;gCACF;;4BACAhB,QAAQsB,IAAI,CAAC,UAAUL;4BACvBjB,QAAQsB,IAAI,CAAC,WAAWL;4BACxB,KAAIxD,eAAAA,KAAK0D,MAAM,cAAX1D,mCAAAA,aAAa8D,OAAO,EAAEN;kCACrBxD,gBAAAA,KAAK0D,MAAM,cAAX1D,oCAAAA,cAAa+D,gBAAgB,CAAC,SAASP,UAAU;gCAAEK,MAAM;4BAAK;wBACrE;;;;IACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. duckdb/turso hand the cache file to\n// one process at a time, so every cycle opens, reconciles, writes the heartbeat/pid rows, and closes.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const baseDir = cfg.baseDir;\n\n  const { store: initialStore, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const existingHeartbeat = await getMeta(initialStore, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await initialStore.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  // total is read before close: an event fires only once nothing is held open again.\n  const initialTotal = initialWarnings.length > 0 || initialParsed > 0 ? await docCount(initialStore) : 0;\n  await setMeta(initialStore, 'watch_heartbeat', new Date().toISOString());\n  await setMeta(initialStore, 'watch_pid', String(process.pid));\n  await initialStore.close();\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: initialTotal, warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  // At most one open+reconcile+close cycle runs at a time; a trigger that arrives mid-cycle is\n  // coalesced into a single rerun instead of opening a second, overlapping connection.\n  let running: Promise<void> | null = null;\n  let queuedAlways: boolean | null = null;\n\n  const tick = async (alwaysEmit: boolean): Promise<void> => {\n    try {\n      const { store, parsed, warnings } = await openStore(cfg);\n      try {\n        await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n        await setMeta(store, 'watch_pid', String(process.pid));\n        if (alwaysEmit || parsed > 0 || warnings.length > 0) {\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        }\n      } finally {\n        await store.close();\n      }\n    } catch (err) {\n      onEvent({ type: 'reconcile-error', message: (err as Error).message });\n    }\n  };\n\n  const startCycle = (alwaysEmit: boolean) => {\n    running = tick(alwaysEmit).then(() => {\n      running = null;\n      if (queuedAlways !== null && !stopping) {\n        const again = queuedAlways;\n        queuedAlways = null;\n        startCycle(again);\n      }\n    });\n  };\n\n  const requestCycle = (alwaysEmit: boolean) => {\n    if (stopping) return;\n    if (running) {\n      queuedAlways = queuedAlways || alwaysEmit;\n      return;\n    }\n    startCycle(alwaysEmit);\n  };\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      requestCycle(true);\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(\n      async () => requestCycle(false),\n      () => stopping\n    ),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await running;\n      const { store: finalStore } = await openStore(cfg);\n      await setMeta(finalStore, 'watch_heartbeat', null);\n      await setMeta(finalStore, 'watch_pid', null);\n      await finalStore.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","baseDir","initialStore","dbPath","initialWarnings","initialParsed","existingHeartbeat","age","initialTotal","stopping","running","queuedAlways","tick","startCycle","requestCycle","debounceTimer","scheduleReconcile","watcher","heartbeatTimer","openStore","store","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","length","docCount","setMeta","toISOString","String","process","pid","type","total","alwaysEmit","err","message","then","again","clearTimeout","setTimeout","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","finalStore","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAyBsBA;;;eAAAA;;;sBAzBW;uBAEP;wBACC;6BACC;wBAC0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtD,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAepB,SAAeH;wCAASI,GAAmB;YAAEC,MAClCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBACAC,SAEoF,MAA3EC,cAAcC,QAAkBC,iBAAyBC,eAClEC,mBAEEC,KAQFC,oBAUFC,UAGAC,SACAC,cAEEC,MAiBAC,YAWAC,cASFC,eACEC,mBAUAC,SAIAC;;;;;oBArF4CrB,OAAAA,oEAAqB,CAAC;oBAClEC,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBAClDO,UAAUL,IAAIK,OAAO;oBAE+D;;wBAAMkB,IAAAA,mBAAS,EAACvB;;;oBAAhB,OAAA,eAA3EM,eAA2E,KAAlFkB,OAAqBjB,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDiB,UAAmChB,gBAAkB,KAA1BiB;oBACtC;;wBAAMC,IAAAA,iBAAO,EAACrB,cAAc;;;oBAAhDI,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACT,KAAK2B,KAAK,AAAD,GAA/BlB;;;;oBACIC,MAAMkB,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACrB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMZ,kBAAiB,GAAnCY;;;;oBACF;;wBAAML,aAAa0B,KAAK;;;oBAAxB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACxB,MAAM,OAAM;;yBAKxFH,CAAAA,gBAAgB4B,MAAM,GAAG,KAAK3B,gBAAgB,CAAA,GAA9CD;;;;oBAAkD;;wBAAM6B,IAAAA,kBAAQ,EAAC/B;;;2BAAf;;;;;;2BAA+B;;;oBAAhGM;oBACN;;wBAAM0B,IAAAA,iBAAO,EAAChC,cAAc,mBAAmB,IAAIuB,OAAOU,WAAW;;;oBAArE;oBACA;;wBAAMD,IAAAA,iBAAO,EAAChC,cAAc,aAAakC,OAAOC,QAAQC,GAAG;;;oBAA3D;oBACA;;wBAAMpC,aAAa0B,KAAK;;;oBAAxB;oBAEA9B,QAAQ;wBAAEyC,MAAM;wBAAWtC,SAAAA;wBAASE,QAAAA;oBAAO;oBAC3C,IAAIC,gBAAgB4B,MAAM,GAAG,KAAK3B,gBAAgB,GAAG;wBACnDP,QAAQ;4BAAEyC,MAAM;4BAAcjB,QAAQjB;4BAAemC,OAAOhC;4BAAca,UAAUjB;wBAAgB;oBACtG;oBAEIK,WAAW;oBACf,6FAA6F;oBAC7F,qFAAqF;oBACjFC,UAAgC;oBAChCC,eAA+B;oBAE7BC,OAAO,cAAO6B;;gCAEoB,MAA5BrB,OAAOE,QAAQD,gBAUhBqB;;;;;;;;;;wCAV6B;;4CAAMvB,IAAAA,mBAAS,EAACvB;;;wCAAhB,OAAA,eAA5BwB,QAA4B,KAA5BA,OAAOE,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;;;;;;;;wCAErB;;4CAAMa,IAAAA,iBAAO,EAACd,OAAO,mBAAmB,IAAIK,OAAOU,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAACd,OAAO,aAAagB,OAAOC,QAAQC,GAAG;;;wCAApD;6CACIG,CAAAA,cAAcnB,SAAS,KAAKD,SAASW,MAAM,GAAG,CAAA,GAA9CS;;;;;4CACQF,MAAM;4CAAcjB,QAAAA;;wCAAe;;4CAAMW,IAAAA,kBAAQ,EAACb;;;wCAA5DtB;kDAAsC0C,QAAO,oBAAuBnB,WAAAA;;;;;;;;;wCAGtE;;4CAAMD,MAAMQ,KAAK;;;wCAAjB;;;;;;;;;;wCAEKc;wCACP5C,QAAQ;4CAAEyC,MAAM;4CAAmBI,SAAS,AAACD,IAAcC,OAAO;wCAAC;;;;;;;;;;;wBAEvE;;oBAEM9B,aAAa,qBAAC4B;wBAClB/B,UAAUE,KAAK6B,YAAYG,IAAI,CAAC;4BAC9BlC,UAAU;4BACV,IAAIC,iBAAiB,QAAQ,CAACF,UAAU;gCACtC,IAAMoC,QAAQlC;gCACdA,eAAe;gCACfE,WAAWgC;4BACb;wBACF;oBACF;oBAEM/B,eAAe,sBAAC2B;wBACpB,IAAIhC,UAAU;wBACd,IAAIC,SAAS;4BACXC,eAAeA,gBAAgB8B;4BAC/B;wBACF;wBACA5B,WAAW4B;oBACb;oBAEI1B,gBAAuC;oBACrCC,oBAAoB;wBACxB,IAAID,eAAe+B,aAAa/B;wBAChCA,gBAAgBgC,WAAW;4BACzBhC,gBAAgB;4BAChBD,aAAa;wBACf,GAAGf;oBACL;oBAEA,iGAAiG;oBACjG,uIAAuI;oBACjIkB,UAAU+B,IAAAA,aAAO,EAAC/C,SAAS;wBAAEgD,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpErC;oBACF;oBACME,iBAAiBoC,YACrBC,IAAAA,0BAAW,EACT;;;;;oCAAYzC,aAAa;;;;uBACzB;+BAAML;wBAERT;oBAGF;;wBAAO,IAAIwD,QAAc,SAACC;gCAqBpB5D,cACCA;4BArBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAM6D,WAAW;;wCAKf7D,cAK8B,MAAf8D;;;;gDATf,IAAIlD,UAAU;;;gDACdA,WAAW;gDACX4B,QAAQuB,GAAG,CAAC,UAAUF;gDACtBrB,QAAQuB,GAAG,CAAC,WAAWF;iDACvB7D,eAAAA,KAAKgE,MAAM,cAAXhE,mCAAAA,aAAaiE,mBAAmB,CAAC,SAASJ;gDAC1CK,cAAc7C;gDACd,IAAIH,eAAe+B,aAAa/B;gDAChCE,QAAQW,KAAK;gDACb;;oDAAMlB;;;gDAAN;gDAC8B;;oDAAMS,IAAAA,mBAAS,EAACvB;;;gDAAhB,OAAA,eAAf+D,aAAe,KAAtBvC;gDACR;;oDAAMc,IAAAA,iBAAO,EAACyB,YAAY,mBAAmB;;;gDAA7C;gDACA;;oDAAMzB,IAAAA,iBAAO,EAACyB,YAAY,aAAa;;;gDAAvC;gDACA;;oDAAMA,WAAW/B,KAAK;;;gDAAtB;gDACA6B;;;;;;gCACF;;4BACApB,QAAQ2B,IAAI,CAAC,UAAUN;4BACvBrB,QAAQ2B,IAAI,CAAC,WAAWN;4BACxB,KAAI7D,eAAAA,KAAKgE,MAAM,cAAXhE,mCAAAA,aAAaoE,OAAO,EAAEP;kCACrB7D,gBAAAA,KAAKgE,MAAM,cAAXhE,oCAAAA,cAAaqE,gBAAgB,CAAC,SAASR,UAAU;gCAAEM,MAAM;4BAAK;wBACrE;;;;IACF"}

package/dist/cjs/workers/parse.d.cts

@@ -11,6 +11,7 @@ export type ParseTaskResult = {
     ok: true;
     doc: ParsedDoc;
     warnings: string[];
+    parseMs: number;
 } | {
     ok: false;
     error: WorkerErrorPayload;

package/dist/cjs/workers/parse.d.ts

@@ -11,6 +11,7 @@ export type ParseTaskResult = {
     ok: true;
     doc: ParsedDoc;
     warnings: string[];
+    parseMs: number;
 } | {
     ok: false;
     error: WorkerErrorPayload;

package/dist/cjs/workers/parse.js

@@ -27,11 +27,14 @@ var selected = _indexts.FEATURES.filter(function(feature) {
 });
 function parseTask(file) {
     try {
+        var start = process.hrtime.bigint();
         var _parseFile = (0, _indexts1.parseFile)(file, (0, _reparsets.featuresForFile)(selected, cfg, file), cfg), doc = _parseFile.doc, warnings = _parseFile.warnings;
+        var parseMs = Number(process.hrtime.bigint() - start) / 1e6;
         return {
             ok: true,
             doc: doc,
-            warnings: warnings
+            warnings: warnings,
+            parseMs: parseMs
         };
     } catch (err) {
         return {

package/dist/cjs/workers/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[] } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    return { ok: true, doc, warnings };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["parseTask","Tinypool","workerData","cfg","featureNames","selected","FEATURES","filter","feature","includes","name","file","parseFile","featuresForFile","doc","warnings","ok","err","error","serializeError"],"mappings":";;;;+BA6BA;;;eAAwBA;;;+DA7BH;uBAEI;wBAEC;yBAEM;6BAED;;;;;;AAe/B,sCAAsC;AACtC,IAA8BC,uBAAAA,iBAAQ,CAACC,UAAU,EAAzCC,MAAsBF,qBAAtBE,KAAKC,eAAiBH,qBAAjBG;AACb,4FAA4F;AAC5F,qDAAqD;AACrD,IAAMC,WAAWC,iBAAQ,CAACC,MAAM,CAAC,SAACC;WAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;;AAEjE,SAASV,UAAUW,IAAe;IAC/C,IAAI;QACF,IAA0BC,aAAAA,IAAAA,mBAAS,EAACD,MAAME,IAAAA,0BAAe,EAACR,UAAUF,KAAKQ,OAAOR,MAAxEW,MAAkBF,WAAlBE,KAAKC,WAAaH,WAAbG;QACb,OAAO;YAAEC,IAAI;YAAMF,KAAAA;YAAKC,UAAAA;QAAS;IACnC,EAAE,OAAOE,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOC,IAAAA,6BAAc,EAACF;QAAK;IACjD;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\n// parseMs is the worker's own hrtime for parseFile, excluding dispatch and the return clone --\n// what separates it from the pool's dispatch-to-drain `parse` stage.\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[]; parseMs: number } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const start = process.hrtime.bigint();\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    const parseMs = Number(process.hrtime.bigint() - start) / 1e6;\n    return { ok: true, doc, warnings, parseMs };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["parseTask","Tinypool","workerData","cfg","featureNames","selected","FEATURES","filter","feature","includes","name","file","start","process","hrtime","bigint","parseFile","featuresForFile","doc","warnings","parseMs","Number","ok","err","error","serializeError"],"mappings":";;;;+BA+BA;;;eAAwBA;;;+DA/BH;uBAEI;wBAEC;yBAEM;6BAED;;;;;;AAiB/B,sCAAsC;AACtC,IAA8BC,uBAAAA,iBAAQ,CAACC,UAAU,EAAzCC,MAAsBF,qBAAtBE,KAAKC,eAAiBH,qBAAjBG;AACb,4FAA4F;AAC5F,qDAAqD;AACrD,IAAMC,WAAWC,iBAAQ,CAACC,MAAM,CAAC,SAACC;WAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;;AAEjE,SAASV,UAAUW,IAAe;IAC/C,IAAI;QACF,IAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;QACnC,IAA0BC,aAAAA,IAAAA,mBAAS,EAACL,MAAMM,IAAAA,0BAAe,EAACZ,UAAUF,KAAKQ,OAAOR,MAAxEe,MAAkBF,WAAlBE,KAAKC,WAAaH,WAAbG;QACb,IAAMC,UAAUC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QAC1D,OAAO;YAAEU,IAAI;YAAMJ,KAAAA;YAAKC,UAAAA;YAAUC,SAAAA;QAAQ;IAC5C,EAAE,OAAOG,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOC,IAAAA,6BAAc,EAACF;QAAK;IACjD;AACF"}

package/dist/esm/chunk/group.js

@@ -37,13 +37,17 @@ function finalize(parts) {
     };
 }
 const NEWLINE_TOKENS = estimateTokens('\n');
-const SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'sentence'
-});
-const WORD_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'word'
-});
-function segmentsOf(text, segmenter) {
+// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever
+// split, so no command pays for a segmenter it never reaches.
+const SEGMENTERS = new Map();
+function segmentsOf(text, granularity) {
+    let segmenter = SEGMENTERS.get(granularity);
+    if (!segmenter) {
+        segmenter = new Intl.Segmenter(undefined, {
+            granularity
+        });
+        SEGMENTERS.set(granularity, segmenter);
+    }
     return Array.from(segmenter.segment(text), (s)=>s.segment);
 }
 // Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups
@@ -68,7 +72,7 @@ function pack(segments, working) {
 // Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then
 // word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.
 function splitLineText(text, working) {
-    const sentences = segmentsOf(text, SENTENCE_SEGMENTER);
+    const sentences = segmentsOf(text, 'sentence');
     const out = [];
     let current = '';
     let tokens = 0;
@@ -83,7 +87,7 @@ function splitLineText(text, working) {
         const sentenceTokens = estimateTokens(sentence);
         if (sentenceTokens > working) {
             flush();
-            out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));
+            out.push(...pack(segmentsOf(sentence, 'word'), working));
             continue;
         }
         if (current.length > 0 && tokens + sentenceTokens > working) flush();

package/dist/esm/chunk/group.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\nconst SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'sentence' });\nconst WORD_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'word' });\n\nfunction segmentsOf(text: string, segmenter: Intl.Segmenter): string[] {\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, SENTENCE_SEGMENTER);\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["extractText","parse","DEFAULT_TARGET_TOKENS","estimateTokens","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","resolveOptions","opts","targetTokens","text","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","SENTENCE_SEGMENTER","Intl","Segmenter","granularity","WORD_SEGMENTER","segmentsOf","segmenter","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","b","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","group","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","split","chunks","scope","texted","c","trim","filter"],"mappings":"AAAA,SAASA,WAAW,QAAQ,eAAe;AAC3C,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,qBAAqB,EAAEC,cAAc,QAAQ,cAAc;AAGpE,MAAMC,iBAAiB;AACvB,MAAMC,4BAA4B;AAOlC,SAASC,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIN;QACpCO,IAAI,WAAEF,iBAAAA,2BAAAA,KAAME,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASC,YAAYC,MAAe;IAClC,MAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;IACzB,KAAK,MAAMC,SAASH,OAAQ;QAC1B,IAAIG,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;YAClDJ,OAAOK,IAAI,CAACJ;YACZA,UAAU,EAAE;QACd;QACAA,QAAQI,IAAI,CAACH;IACf;IACA,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,MAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,MAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEf,MAAMU,MAAMM,GAAG,CAAC,CAACC,IAAMA,EAAEjB,IAAI,EAAEkB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,CAACH,IAAMA,EAAEE,KAAK;IAAE;AAC3I;AAEA,MAAME,iBAAiB3B,eAAe;AACtC,MAAM4B,qBAAqB,IAAIC,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAW;AACnF,MAAMC,iBAAiB,IAAIH,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAO;AAE3E,SAASE,WAAW3B,IAAY,EAAE4B,SAAyB;IACzD,OAAOC,MAAMC,IAAI,CAACF,UAAUG,OAAO,CAAC/B,OAAO,CAACgC,IAAMA,EAAED,OAAO;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,MAAMC,SAAmB,EAAE;IAC3B,IAAIhC,UAAU;IACd,IAAIiC,SAAS;IACb,KAAK,MAAMN,WAAWG,SAAU;QAC9B,MAAMI,gBAAgB5C,eAAeqC;QACrC,IAAI3B,QAAQG,MAAM,GAAG,KAAK8B,SAASC,gBAAgBH,SAAS;YAC1DC,OAAO5B,IAAI,CAACJ;YACZA,UAAU;YACViC,SAAS;QACX;QACAjC,WAAW2B;QACXM,UAAUC;IACZ;IACA,IAAIlC,QAAQG,MAAM,GAAG,GAAG6B,OAAO5B,IAAI,CAACJ;IACpC,OAAOgC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAcvC,IAAY,EAAEmC,OAAe;IAClD,MAAMK,YAAYb,WAAW3B,MAAMsB;IACnC,MAAMmB,MAAgB,EAAE;IACxB,IAAIrC,UAAU;IACd,IAAIiC,SAAS;IACb,MAAMK,QAAQ;QACZ,IAAItC,QAAQG,MAAM,GAAG,GAAG;YACtBkC,IAAIjC,IAAI,CAACJ;YACTA,UAAU;YACViC,SAAS;QACX;IACF;IACA,KAAK,MAAMM,YAAYH,UAAW;QAChC,MAAMI,iBAAiBlD,eAAeiD;QACtC,IAAIC,iBAAiBT,SAAS;YAC5BO;YACAD,IAAIjC,IAAI,IAAIyB,KAAKN,WAAWgB,UAAUjB,iBAAiBS;YACvD;QACF;QACA,IAAI/B,QAAQG,MAAM,GAAG,KAAK8B,SAASO,iBAAiBT,SAASO;QAC7DtC,WAAWuC;QACXN,UAAUO;IACZ;IACAF;IACA,OAAOD;AACT;AAEA,MAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEkC,SAAoB,EAAEC,QAA6B;IAC1H,MAAMlD,OACJ6C,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAW9B,IAAI,CAAC,QAChB1B,MAAMwD,WAAW9B,IAAI,CAAC,OACnBF,GAAG,CAAC,CAACoC,IAAM7D,YAAY6D,EAAEC,IAAI,GAC7BnC,IAAI,CAAC;IACd,OAAO;QAAEJ;QAAWC;QAASf;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAASsD,cAAcN,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,MAAMjC,IAAI8B,MAAMC,YAAYlC,WAAWC,SAASkC,WAAWC;IAC3D,IAAIF,WAAWzC,MAAM,KAAK,KAAKb,eAAeuB,EAAEjB,IAAI,IAAImC,SAAS;QAC/D,OAAOI,cAActB,EAAEjB,IAAI,EAAEmC,SAASnB,GAAG,CAAC,CAAChB,OAAU,CAAA;gBAAEc;gBAAWC;gBAASf;gBAAMmB,OAAO;YAAK,CAAA;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAASsC,mBAAmBC,KAAe,EAAE1C,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B,EAAEO,OAAO,CAAC;IAC7J,MAAMC,SAAiB,EAAE;IACzB,IAAIV,aAAuB,EAAE;IAC7B,IAAIW,aAAa7C;IACjB,IAAIuB,SAASoB;IACb,IAAK,IAAIG,OAAO9C,WAAW8C,QAAQ7C,SAAS6C,OAAQ;QAClD,MAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,MAAME,MAAMd,WAAWzC,MAAM,GAAG,KAAK8B,SAAS,IAAIhB,iBAAiB;QACnE,MAAM0C,aAAarE,eAAemE;QAClC,IAAIb,WAAWzC,MAAM,GAAG,KAAK8B,SAASyB,MAAMC,aAAa5B,SAAS;YAChEuB,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAYC,OAAO,GAAGzB,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTsB,aAAaC;YACbZ,WAAWxC,IAAI,CAACqD;YAChBxB,UAAU0B;YACV;QACF;QACAf,WAAWxC,IAAI,CAACqD;QAChBxB,UAAUyB,MAAMC;IAClB;IACA,IAAIf,WAAWzC,MAAM,GAAG,GAAGmD,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAY5C,SAASoB,SAASc,WAAWC;IAC7G,OAAOQ;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,MAAM/B,UAAU+B,SAASnE,YAAY;IACrC,MAAMoE,UAAUhC,UAAUvC;IAC1B,MAAMwE,WAA4C,EAAE;IACpD,IAAI1D,QAAgB,EAAE;IACtB,IAAI2D,iBAAiB;IACrB,IAAIhC,SAAS;IAEb,SAASiC;QACP,MAAMC,QAAQ9D,SAASC;QACvB,IAAI6D,OAAOH,SAAS5D,IAAI,CAAC+D;QACzB7D,QAAQ,EAAE;QACV2D,iBAAiB;QACjBhC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASmC,QAAQxE,IAAY,EAAEyE,QAAgB,EAAE3D,SAAiB,EAAEC,OAAe;QACjFsB,UAAU,AAAC3B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAK3B,eAAe+E;QACnE/D,MAAMF,IAAI,CAAC;YAAEM;YAAWC;YAASf;QAAK;IACxC;IAEA,KAAK,MAAMK,SAAS4D,YAAa;QAC/B,MAAMS,MAAMlB,MAAMmB,KAAK,CAACtE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;QACjE,MAAM0D,cAAclF,eAAegF;QAEnC,IAAIE,cAAcT,SAAS;YACzB,MAAMV,OAAO/C,MAAMH,MAAM,GAAG,IAAI8B,SAAS;YACzC,KAAK,MAAMpB,KAAKsC,mBAAmBC,OAAOnD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEoB,SAAS9B,MAAMC,IAAI,EAAE4D,SAASlE,IAAI,EAAEyD,MAAO;gBACnH/C,MAAMF,IAAI,CAACS;gBACXqD;YACF;YACA;QACF;QAEA,MAAMO,YAAYtF,YAAYc,MAAMgD,IAAI;QACxC,MAAMoB,WAAWP,SAASlE,IAAI,KAAK,QAAQ0E,MAAMG;QAEjD,IAAIxE,MAAMC,IAAI,KAAK,WAAW;YAC5BkE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D;QACF;QAEA,wFAAwF;QACxF,wDAAwD;QACxD,MAAM+D,eAAepE,MAAMH,MAAM,GAAG,KAAK8B,SAAShB,iBAAiBuD,cAAcT;QACjF,IAAIW,cAAcR;QAElBE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;QAC3DsD;QAEA,IAAIA,kBAAkB1E,gBAAgB2E;IACxC;IACAA;IAEA,OAAOF;AACT;AAEA,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASG,MAAMrE,MAAe,EAAE6E,IAAY,EAAEjF,IAAmB;IACtE,MAAMoE,WAAWrE,eAAeC;IAChC,MAAM0D,QAAQuB,KAAKC,KAAK,CAAC;IACzB,MAAMC,SAA0C,EAAE;IAClD,KAAK,MAAMC,SAASjF,YAAYC,QAAS+E,OAAOzE,IAAI,IAAIwD,WAAWkB,OAAO1B,OAAOU;IACjF,+FAA+F;IAC/F,qHAAqH;IACrH,MAAMiB,SACJjB,SAASlE,IAAI,KAAK,QACdiF,OAAOjE,GAAG,CAAC,CAACoE,IACVA,EAAEjE,KAAK,GACHiE,IACA;YACE,GAAGA,CAAC;YACJpF,MAAMwD,MACHmB,KAAK,CAACS,EAAEtE,SAAS,GAAG,GAAGsE,EAAErE,OAAO,EAChCG,IAAI,CAAC,MACLmE,IAAI;QACT,KAENJ;IACN,mHAAmH;IACnH,OAAOE,OAAOG,MAAM,CAAC,CAACF,IAAMA,EAAEpF,IAAI,CAACqF,IAAI,GAAG9E,MAAM,GAAG,GAAGS,GAAG,CAAC,CAACoE,IAAO,CAAA;YAAEtE,WAAWsE,EAAEtE,SAAS;YAAEC,SAASqE,EAAErE,OAAO;YAAEf,MAAMoF,EAAEpF,IAAI;QAAC,CAAA;AAC/H"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport { DEFAULT_TARGET_TOKENS, estimateTokens } from './tokens.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // True for a sub-line split piece: its text is already the final slice, not the whole line --\n  // group()'s extent-based raw re-slice must not touch it (siblings share startLine === endLine).\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\n// Built on first use and kept: each construction is ~3.5 ms, and only an oversize block is ever\n// split, so no command pays for a segmenter it never reaches.\nconst SEGMENTERS = new Map<string, Intl.Segmenter>();\n\nfunction segmentsOf(text: string, granularity: 'sentence' | 'word'): string[] {\n  let segmenter = SEGMENTERS.get(granularity);\n  if (!segmenter) {\n    segmenter = new Intl.Segmenter(undefined, { granularity });\n    SEGMENTERS.set(granularity, segmenter);\n  }\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// Line-split alone can't shrink a lone dense line (the CJK case): falls back to sentence then\n// word boundaries (Intl.Segmenter, the same grapheme-safe engine as segment.ts), mode-agnostic on `text`.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, 'sentence');\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, 'word'), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// Atomic (code/table/list) pieces are always a raw line slice: re-parsing a table's later pieces\n// without their header/delimiter rows would demote them to paragraph text.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A one-line piece over working can't shrink via another line-boundary pass (rule 5's gap), so it\n// splits at sentence/word boundaries instead; `final` stops group() re-deriving its text by extent (F5).\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size, never\n// mid-line. `seed`: pending tokens (e.g. a heading) the first piece must join, checked against the limit.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk already carries its own slice's raw text; re-slicing by extent would return the whole shared line.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["extractText","parse","DEFAULT_TARGET_TOKENS","estimateTokens","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","resolveOptions","opts","targetTokens","text","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","SEGMENTERS","Map","segmentsOf","granularity","segmenter","get","Intl","Segmenter","set","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","b","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","group","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","split","chunks","scope","texted","c","trim","filter"],"mappings":"AAAA,SAASA,WAAW,QAAQ,eAAe;AAC3C,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,qBAAqB,EAAEC,cAAc,QAAQ,cAAc;AAGpE,MAAMC,iBAAiB;AACvB,MAAMC,4BAA4B;AAOlC,SAASC,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIN;QACpCO,IAAI,WAAEF,iBAAAA,2BAAAA,KAAME,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASC,YAAYC,MAAe;IAClC,MAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;IACzB,KAAK,MAAMC,SAASH,OAAQ;QAC1B,IAAIG,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;YAClDJ,OAAOK,IAAI,CAACJ;YACZA,UAAU,EAAE;QACd;QACAA,QAAQI,IAAI,CAACH;IACf;IACA,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAWA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,MAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,MAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEf,MAAMU,MAAMM,GAAG,CAAC,CAACC,IAAMA,EAAEjB,IAAI,EAAEkB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,CAACH,IAAMA,EAAEE,KAAK;IAAE;AAC3I;AAEA,MAAME,iBAAiB3B,eAAe;AACtC,gGAAgG;AAChG,8DAA8D;AAC9D,MAAM4B,aAAa,IAAIC;AAEvB,SAASC,WAAWxB,IAAY,EAAEyB,WAAgC;IAChE,IAAIC,YAAYJ,WAAWK,GAAG,CAACF;IAC/B,IAAI,CAACC,WAAW;QACdA,YAAY,IAAIE,KAAKC,SAAS,CAAClB,WAAW;YAAEc;QAAY;QACxDH,WAAWQ,GAAG,CAACL,aAAaC;IAC9B;IACA,OAAOK,MAAMC,IAAI,CAACN,UAAUO,OAAO,CAACjC,OAAO,CAACkC,IAAMA,EAAED,OAAO;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,MAAMC,SAAmB,EAAE;IAC3B,IAAIlC,UAAU;IACd,IAAImC,SAAS;IACb,KAAK,MAAMN,WAAWG,SAAU;QAC9B,MAAMI,gBAAgB9C,eAAeuC;QACrC,IAAI7B,QAAQG,MAAM,GAAG,KAAKgC,SAASC,gBAAgBH,SAAS;YAC1DC,OAAO9B,IAAI,CAACJ;YACZA,UAAU;YACVmC,SAAS;QACX;QACAnC,WAAW6B;QACXM,UAAUC;IACZ;IACA,IAAIpC,QAAQG,MAAM,GAAG,GAAG+B,OAAO9B,IAAI,CAACJ;IACpC,OAAOkC;AACT;AAEA,8FAA8F;AAC9F,0GAA0G;AAC1G,SAASG,cAAczC,IAAY,EAAEqC,OAAe;IAClD,MAAMK,YAAYlB,WAAWxB,MAAM;IACnC,MAAM2C,MAAgB,EAAE;IACxB,IAAIvC,UAAU;IACd,IAAImC,SAAS;IACb,MAAMK,QAAQ;QACZ,IAAIxC,QAAQG,MAAM,GAAG,GAAG;YACtBoC,IAAInC,IAAI,CAACJ;YACTA,UAAU;YACVmC,SAAS;QACX;IACF;IACA,KAAK,MAAMM,YAAYH,UAAW;QAChC,MAAMI,iBAAiBpD,eAAemD;QACtC,IAAIC,iBAAiBT,SAAS;YAC5BO;YACAD,IAAInC,IAAI,IAAI2B,KAAKX,WAAWqB,UAAU,SAASR;YAC/C;QACF;QACA,IAAIjC,QAAQG,MAAM,GAAG,KAAKgC,SAASO,iBAAiBT,SAASO;QAC7DxC,WAAWyC;QACXN,UAAUO;IACZ;IACAF;IACA,OAAOD;AACT;AAEA,MAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,iGAAiG;AACjG,2EAA2E;AAC3E,SAASC,MAAMC,UAAoB,EAAEpC,SAAiB,EAAEC,OAAe,EAAEoC,SAAoB,EAAEC,QAA6B;IAC1H,MAAMpD,OACJ+C,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAWhC,IAAI,CAAC,QAChB1B,MAAM0D,WAAWhC,IAAI,CAAC,OACnBF,GAAG,CAAC,CAACsC,IAAM/D,YAAY+D,EAAEC,IAAI,GAC7BrC,IAAI,CAAC;IACd,OAAO;QAAEJ;QAAWC;QAASf;IAAK;AACpC;AAEA,kGAAkG;AAClG,yGAAyG;AACzG,SAASwD,cAAcN,UAAoB,EAAEpC,SAAiB,EAAEC,OAAe,EAAEsB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,MAAMnC,IAAIgC,MAAMC,YAAYpC,WAAWC,SAASoC,WAAWC;IAC3D,IAAIF,WAAW3C,MAAM,KAAK,KAAKb,eAAeuB,EAAEjB,IAAI,IAAIqC,SAAS;QAC/D,OAAOI,cAAcxB,EAAEjB,IAAI,EAAEqC,SAASrB,GAAG,CAAC,CAAChB,OAAU,CAAA;gBAAEc;gBAAWC;gBAASf;gBAAMmB,OAAO;YAAK,CAAA;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,iGAAiG;AACjG,0GAA0G;AAC1G,SAASwC,mBAAmBC,KAAe,EAAE5C,SAAiB,EAAEC,OAAe,EAAEsB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B,EAAEO,OAAO,CAAC;IAC7J,MAAMC,SAAiB,EAAE;IACzB,IAAIV,aAAuB,EAAE;IAC7B,IAAIW,aAAa/C;IACjB,IAAIyB,SAASoB;IACb,IAAK,IAAIG,OAAOhD,WAAWgD,QAAQ/C,SAAS+C,OAAQ;QAClD,MAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,MAAME,MAAMd,WAAW3C,MAAM,GAAG,KAAKgC,SAAS,IAAIlB,iBAAiB;QACnE,MAAM4C,aAAavE,eAAeqE;QAClC,IAAIb,WAAW3C,MAAM,GAAG,KAAKgC,SAASyB,MAAMC,aAAa5B,SAAS;YAChEuB,OAAOpD,IAAI,IAAIgD,cAAcN,YAAYW,YAAYC,OAAO,GAAGzB,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTsB,aAAaC;YACbZ,WAAW1C,IAAI,CAACuD;YAChBxB,UAAU0B;YACV;QACF;QACAf,WAAW1C,IAAI,CAACuD;QAChBxB,UAAUyB,MAAMC;IAClB;IACA,IAAIf,WAAW3C,MAAM,GAAG,GAAGqD,OAAOpD,IAAI,IAAIgD,cAAcN,YAAYW,YAAY9C,SAASsB,SAASc,WAAWC;IAC7G,OAAOQ;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,MAAM/B,UAAU+B,SAASrE,YAAY;IACrC,MAAMsE,UAAUhC,UAAUzC;IAC1B,MAAM0E,WAA4C,EAAE;IACpD,IAAI5D,QAAgB,EAAE;IACtB,IAAI6D,iBAAiB;IACrB,IAAIhC,SAAS;IAEb,SAASiC;QACP,MAAMC,QAAQhE,SAASC;QACvB,IAAI+D,OAAOH,SAAS9D,IAAI,CAACiE;QACzB/D,QAAQ,EAAE;QACV6D,iBAAiB;QACjBhC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASmC,QAAQ1E,IAAY,EAAE2E,QAAgB,EAAE7D,SAAiB,EAAEC,OAAe;QACjFwB,UAAU,AAAC7B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAK3B,eAAeiF;QACnEjE,MAAMF,IAAI,CAAC;YAAEM;YAAWC;YAASf;QAAK;IACxC;IAEA,KAAK,MAAMK,SAAS8D,YAAa;QAC/B,MAAMS,MAAMlB,MAAMmB,KAAK,CAACxE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;QACjE,MAAM4D,cAAcpF,eAAekF;QAEnC,IAAIE,cAAcT,SAAS;YACzB,MAAMV,OAAOjD,MAAMH,MAAM,GAAG,IAAIgC,SAAS;YACzC,KAAK,MAAMtB,KAAKwC,mBAAmBC,OAAOrD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEsB,SAAShC,MAAMC,IAAI,EAAE8D,SAASpE,IAAI,EAAE2D,MAAO;gBACnHjD,MAAMF,IAAI,CAACS;gBACXuD;YACF;YACA;QACF;QAEA,MAAMO,YAAYxF,YAAYc,MAAMkD,IAAI;QACxC,MAAMoB,WAAWP,SAASpE,IAAI,KAAK,QAAQ4E,MAAMG;QAEjD,IAAI1E,MAAMC,IAAI,KAAK,WAAW;YAC5BoE,QAAQK,WAAWJ,UAAUtE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D;QACF;QAEA,wFAAwF;QACxF,wDAAwD;QACxD,MAAMiE,eAAetE,MAAMH,MAAM,GAAG,KAAKgC,SAASlB,iBAAiByD,cAAcT;QACjF,IAAIW,cAAcR;QAElBE,QAAQK,WAAWJ,UAAUtE,MAAMS,SAAS,EAAET,MAAMU,OAAO;QAC3DwD;QAEA,IAAIA,kBAAkB5E,gBAAgB6E;IACxC;IACAA;IAEA,OAAOF;AACT;AAEA,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASG,MAAMvE,MAAe,EAAE+E,IAAY,EAAEnF,IAAmB;IACtE,MAAMsE,WAAWvE,eAAeC;IAChC,MAAM4D,QAAQuB,KAAKC,KAAK,CAAC;IACzB,MAAMC,SAA0C,EAAE;IAClD,KAAK,MAAMC,SAASnF,YAAYC,QAASiF,OAAO3E,IAAI,IAAI0D,WAAWkB,OAAO1B,OAAOU;IACjF,+FAA+F;IAC/F,qHAAqH;IACrH,MAAMiB,SACJjB,SAASpE,IAAI,KAAK,QACdmF,OAAOnE,GAAG,CAAC,CAACsE,IACVA,EAAEnE,KAAK,GACHmE,IACA;YACE,GAAGA,CAAC;YACJtF,MAAM0D,MACHmB,KAAK,CAACS,EAAExE,SAAS,GAAG,GAAGwE,EAAEvE,OAAO,EAChCG,IAAI,CAAC,MACLqE,IAAI;QACT,KAENJ;IACN,mHAAmH;IACnH,OAAOE,OAAOG,MAAM,CAAC,CAACF,IAAMA,EAAEtF,IAAI,CAACuF,IAAI,GAAGhF,MAAM,GAAG,GAAGS,GAAG,CAAC,CAACsE,IAAO,CAAA;YAAExE,WAAWwE,EAAExE,SAAS;YAAEC,SAASuE,EAAEvE,OAAO;YAAEf,MAAMsF,EAAEtF,IAAI;QAAC,CAAA;AAC/H"}

package/dist/esm/commands/search.js

@@ -7,7 +7,7 @@ import { searchError } from '../output/search-error.js';
 import { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.js';
 import { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.js';
 // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
-// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, rows get the JS excerpt.
+// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md). Past this bound, rows get the JS excerpt.
 const EXCERPT_WINDOW = 160;
 // Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt
 // scan matches the same words the query matched on, not FTS5 syntax.