sensemaking 0.18.0 → 0.18.2

package/dist/cjs/scan/reparse.js

@@ -2,13 +2,25 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
-Object.defineProperty(exports, "reparseFiles", {
-    enumerable: true,
-    get: function() {
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get featuresForFile () {
+        return featuresForFile;
+    },
+    get reparseFiles () {
         return reparseFiles;
     }
 });
+var _nodemodule = require("node:module");
+var _nodeos = require("node:os");
+var _nodepath = require("node:path");
 var _indexts = require("./index.js");
+var _workererrorts = require("./worker-error.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -17,6 +29,32 @@ function _array_like_to_array(arr, len) {
 function _array_without_holes(arr) {
     if (Array.isArray(arr)) return _array_like_to_array(arr);
 }
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
 function _iterable_to_array(iter) {
     if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
         return Array.from(iter);
@@ -28,6 +66,105 @@ function _non_iterable_spread() {
 function _to_consumable_array(arr) {
     return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
 }
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
 function _unsupported_iterable_to_array(o, minLen) {
     if (!o) return;
     if (typeof o === "string") return _array_like_to_array(o, minLen);
@@ -36,51 +173,60 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-function reparseFiles(files, features, cfg, knownColumns, onParsed) {
-    var docs = [];
-    var warnings = [];
-    var newColumns = [];
-    var seen = new Set(knownColumns);
+// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house
+// deferral reaches it. Deferred (not a top-level import) because most reconciles stay below
+// DEFAULT_THRESHOLD and should not pay for constructing pool machinery they never use.
+var _require = typeof require === 'undefined' ? (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString()) : require;
+// Measured 2026-08-29 on a 14-logical-core Apple M4 Pro: serial and pooled cross between 160
+// and 180 files (pool fixed cost here is ~85-100ms, not the ~300ms a heavier module graph
+// costs elsewhere), so 200 keeps a safety margin without giving up real wins. 8 workers matches
+// an interleaved worker-count sweep at 6,566 files -- 8 and 10 tie within noise, 12+ trends
+// worse (main-thread + OS contention on the 6 non-parse-worker cores).
+var DEFAULT_THRESHOLD = 200;
+var DEFAULT_MAX_WORKERS = 8;
+// Worker MUST always load from dist/cjs/ (house pattern: install-module-linked,
+// install-optional, node-version-install, node-exec-path all carry this same rule). A
+// worker_threads thread is a fresh realm and does not inherit the main thread's TS loader
+// hook, so a source path cannot work on any platform; dist/cjs loads from the esm and cjs
+// builds alike. Root is found the way cli.ts finds it, so this also resolves when the suite
+// runs against src/ (which already requires a build for the exports specs).
+// Resolved on first pooled dispatch, not at import: a tree under the threshold never pools and
+// must not pay for this, let alone fail on it.
+var workerFile;
+function resolveWorkerFile() {
+    if (workerFile) return workerFile;
+    var load = (0, _nodemodule.createRequire)(require("url").pathToFileURL(__filename).toString());
+    for(var _i = 0, _iter = [
+        '..',
+        '../..',
+        '../../..'
+    ]; _i < _iter.length; _i++){
+        var rel = _iter[_i];
+        try {
+            if (load("".concat(rel, "/package.json")).name === 'sensemaking') {
+                workerFile = (0, _nodepath.join)((0, _nodepath.dirname)(load.resolve("".concat(rel, "/package.json"))), 'dist', 'cjs', 'workers', 'parse.js');
+                return workerFile;
+            }
+        } catch (unused) {}
+    }
+    throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
+}
+function featuresForFile(features, cfg, file) {
+    return features.filter(function(feature) {
+        return !feature.enabledForFile || feature.enabledForFile(cfg, file);
+    });
+}
+function reparseSerial(files, features, cfg, onParsed) {
+    var results = [];
     var done = 0;
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
     try {
-        var _loop = function() {
+        for(var _iterator = files[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var file = _step.value;
-            var _warnings;
-            // A doc only gets extract/store from features that apply to it (currently: embed, via
-            // FileStat.embed -- true iff the config names an embedding model).
-            var fileFeatures = features.filter(function(feature) {
-                return !feature.enabledForFile || feature.enabledForFile(cfg, file);
-            });
-            var _parseFile = (0, _indexts.parseFile)(file, fileFeatures, cfg), doc = _parseFile.doc, fileWarnings = _parseFile.warnings;
+            var fileFeatures = featuresForFile(features, cfg, file);
+            results.push((0, _indexts.parseFile)(file, fileFeatures, cfg));
             onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
-            (_warnings = warnings).push.apply(_warnings, _to_consumable_array(fileWarnings));
-            var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-            try {
-                for(var _iterator = Object.keys(doc.data)[Symbol.iterator](), _step1; !(_iteratorNormalCompletion = (_step1 = _iterator.next()).done); _iteratorNormalCompletion = true){
-                    var key = _step1.value;
-                    if (!seen.has(key)) {
-                        seen.add(key);
-                        newColumns.push(key);
-                    }
-                }
-            } catch (err) {
-                _didIteratorError = true;
-                _iteratorError = err;
-            } finally{
-                try {
-                    if (!_iteratorNormalCompletion && _iterator.return != null) {
-                        _iterator.return();
-                    }
-                } finally{
-                    if (_didIteratorError) {
-                        throw _iteratorError;
-                    }
-                }
-            }
-            docs.push(doc);
-        };
-        for(var _iterator = files[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true)_loop();
+        }
     } catch (err) {
         _didIteratorError = true;
         _iteratorError = err;
@@ -95,10 +241,191 @@ function reparseFiles(files, features, cfg, knownColumns, onParsed) {
             }
         }
     }
-    return {
-        docs: docs,
-        warnings: warnings,
-        newColumns: newColumns
-    };
+    return results;
+}
+// Never the tree: a worker task carries one FileStat and returns only what parseFile already
+// returns -- extracted text and feature values, never an mdast tree. The worker reads the file
+// itself. Pool created once here and destroyed once dispatch finishes; never per task.
+function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
+    return _async_to_generator(function() {
+        var Tinypool, workerData, pool, done, results, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    Tinypool = _require('tinypool').Tinypool;
+                    // cfg and the caller's feature selection are constant for the whole dispatch, so they cross
+                    // once per worker as workerData rather than once per file in the task payload. Features carry
+                    // closures and cannot cross at all; their names can, and the worker resolves them back
+                    // against the same registry, so a caller passing a subset gets that subset in both modes.
+                    workerData = {
+                        cfg: cfg,
+                        featureNames: features.map(function(feature) {
+                            return feature.name;
+                        })
+                    };
+                    pool = new Tinypool({
+                        filename: resolveWorkerFile(),
+                        minThreads: maxWorkers,
+                        maxThreads: maxWorkers,
+                        workerData: workerData
+                    });
+                    done = 0;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        4,
+                        ,
+                        6
+                    ]);
+                    return [
+                        4,
+                        Promise.all(files.map(function(file) {
+                            return _async_to_generator(function() {
+                                var result;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                pool.run(file)
+                                            ];
+                                        case 1:
+                                            result = _state.sent();
+                                            if (!result.ok) throw (0, _workererrorts.reviveError)(result.error);
+                                            onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
+                                            return [
+                                                2,
+                                                {
+                                                    doc: result.doc,
+                                                    warnings: result.warnings
+                                                }
+                                            ];
+                                    }
+                                });
+                            })();
+                        }))
+                    ];
+                case 2:
+                    results = _state.sent();
+                    return [
+                        4,
+                        pool.destroy()
+                    ];
+                case 3:
+                    _state.sent();
+                    return [
+                        2,
+                        results
+                    ];
+                case 4:
+                    err = _state.sent();
+                    // Tear the pool down before rethrowing, but never let a destroy failure replace the parse
+                    // failure that caused it: that one names the file, this one names nothing. Not a `finally`
+                    // for exactly that reason.
+                    return [
+                        4,
+                        pool.destroy().catch(function() {})
+                    ];
+                case 5:
+                    _state.sent();
+                    throw err;
+                case 6:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function reparseFiles(_0, _1, _2, _3, _4) {
+    return _async_to_generator(function(files, features, cfg, knownColumns, onParsed) {
+        var options, _options_threshold, _options_maxWorkers, threshold, maxWorkers, results, _tmp, docs, warnings, newColumns, seen, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, doc, fileWarnings, _warnings, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, key;
+        var _arguments = arguments;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    options = _arguments.length > 5 && _arguments[5] !== void 0 ? _arguments[5] : {};
+                    threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold : DEFAULT_THRESHOLD;
+                    maxWorkers = (_options_maxWorkers = options.maxWorkers) !== null && _options_maxWorkers !== void 0 ? _options_maxWorkers : Math.min(DEFAULT_MAX_WORKERS, (0, _nodeos.availableParallelism)());
+                    if (!(files.length >= threshold)) return [
+                        3,
+                        2
+                    ];
+                    return [
+                        4,
+                        reparsePooled(files, features, cfg, onParsed, maxWorkers)
+                    ];
+                case 1:
+                    _tmp = _state.sent();
+                    return [
+                        3,
+                        3
+                    ];
+                case 2:
+                    _tmp = reparseSerial(files, features, cfg, onParsed);
+                    _state.label = 3;
+                case 3:
+                    results = _tmp;
+                    docs = [];
+                    warnings = [];
+                    newColumns = [];
+                    seen = new Set(knownColumns);
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        for(_iterator = results[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                            _step_value = _step.value, doc = _step_value.doc, fileWarnings = _step_value.warnings;
+                            ;
+                            (_warnings = warnings).push.apply(_warnings, _to_consumable_array(fileWarnings));
+                            _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                            try {
+                                for(_iterator1 = Object.keys(doc.data)[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                                    key = _step1.value;
+                                    if (!seen.has(key)) {
+                                        seen.add(key);
+                                        newColumns.push(key);
+                                    }
+                                }
+                            } catch (err) {
+                                _didIteratorError1 = true;
+                                _iteratorError1 = err;
+                            } finally{
+                                try {
+                                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                                        _iterator1.return();
+                                    }
+                                } finally{
+                                    if (_didIteratorError1) {
+                                        throw _iteratorError1;
+                                    }
+                                }
+                            }
+                            docs.push(doc);
+                        }
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
+                    }
+                    return [
+                        2,
+                        {
+                            docs: docs,
+                            warnings: warnings,
+                            newColumns: newColumns
+                        }
+                    ];
+            }
+        });
+    }).apply(this, arguments);
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving by\n// construction (one pass over `files`, pushed in order) so a future concurrent dispatch can\n// replace the loop body without changing this contract or either call site.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order across `files` -- the order\n  // callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void): ReparseResult {\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  let done = 0;\n\n  for (const file of files) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures, cfg);\n    onParsed?.(++done);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["reparseFiles","files","features","cfg","knownColumns","onParsed","docs","warnings","newColumns","seen","Set","done","file","fileFeatures","filter","feature","enabledForFile","parseFile","doc","fileWarnings","push","Object","keys","data","key","has","add"],"mappings":";;;;+BAmBgBA;;;eAAAA;;;uBAhBU;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgBnB,SAASA,aAAaC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,YAAiC,EAAEC,QAAiC;IACpJ,IAAMC,OAAoB,EAAE;IAC5B,IAAMC,WAAqB,EAAE;IAC7B,IAAMC,aAAuB,EAAE;IAC/B,IAAMC,OAAO,IAAIC,IAAIN;IACrB,IAAIO,OAAO;QAEN,kCAAA,2BAAA;;;YAAA,IAAMC,OAAN;gBAMHL;YALA,sFAAsF;YACtF,mEAAmE;YACnE,IAAMM,eAAeX,SAASY,MAAM,CAAC,SAACC;uBAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACb,KAAKS;;YACzG,IAAwCK,aAAAA,IAAAA,kBAAS,EAACL,MAAMC,cAAcV,MAA9De,MAAgCD,WAAhCC,KAAKX,AAAUY,eAAiBF,WAA3BV;YACbF,qBAAAA,+BAAAA,SAAW,EAAEM;YACbJ,CAAAA,YAAAA,UAASa,IAAI,OAAbb,WAAc,qBAAGY;gBACZ,kCAAA,2BAAA;;gBAAL,QAAK,YAAaE,OAAOC,IAAI,CAACJ,IAAIK,IAAI,sBAAjC,UAAA,6BAAA,SAAA,yBAAA,iCAAoC;oBAApC,IAAMC,MAAN;oBACH,IAAI,CAACf,KAAKgB,GAAG,CAACD,MAAM;wBAClBf,KAAKiB,GAAG,CAACF;wBACThB,WAAWY,IAAI,CAACI;oBAClB;gBACF;;gBALK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAMLlB,KAAKc,IAAI,CAACF;QACZ;QAdA,QAAK,YAAcjB,0BAAd,SAAA,6BAAA,QAAA,yBAAA;;QAAA;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAgBL,OAAO;QAAEK,MAAAA;QAAMC,UAAAA;QAAUC,YAAAA;IAAW;AACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house\n// deferral reaches it. Deferred (not a top-level import) because most reconciles stay below\n// DEFAULT_THRESHOLD and should not pay for constructing pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving by\n// construction (one pass over `files`, pushed in order) so a future concurrent dispatch can\n// replace the loop body without changing this contract or either call site.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order across `files` -- the order\n  // callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-logical-core Apple M4 Pro: serial and pooled cross between 160\n// and 180 files (pool fixed cost here is ~85-100ms, not the ~300ms a heavier module graph\n// costs elsewhere), so 200 keeps a safety margin without giving up real wins. 8 workers matches\n// an interleaved worker-count sweep at 6,566 files -- 8 and 10 tie within noise, 12+ trends\n// worse (main-thread + OS contention on the 6 non-parse-worker cores).\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST always load from dist/cjs/ (house pattern: install-module-linked,\n// install-optional, node-version-install, node-exec-path all carry this same rule). A\n// worker_threads thread is a fresh realm and does not inherit the main thread's TS loader\n// hook, so a source path cannot work on any platform; dist/cjs loads from the esm and cjs\n// builds alike. Root is found the way cli.ts finds it, so this also resolves when the suite\n// runs against src/ (which already requires a build for the exports specs).\n// Resolved on first pooled dispatch, not at import: a tree under the threshold never pools and\n// must not pay for this, let alone fail on it.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (which reads `features` straight from\n// the caller, real registry or a test double) and the worker path (which can only pass `cfg`\n// across the thread boundary, so it derives `features` from activeFeatures(cfg) itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile already\n// returns -- extracted text and feature values, never an mdast tree. The worker reads the file\n// itself. Pool created once here and destroyed once dispatch finishes; never per task.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the caller's feature selection are constant for the whole dispatch, so they cross\n  // once per worker as workerData rather than once per file in the task payload. Features carry\n  // closures and cannot cross at all; their names can, and the worker resolves them back\n  // against the same registry, so a caller passing a subset gets that subset in both modes.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files`\n    // order regardless of which task finishes first, which is what first-seen column order\n    // and warning order depend on downstream. Progress ticks fire in completion order instead\n    // (only the count matters for a progress bar) -- free, since it rides the same per-task\n    // response tinypool already sends, no extra ping.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but never let a destroy failure replace the parse\n    // failure that caused it: that one names the file, this one names nothing. Not a `finally`\n    // for exactly that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["featuresForFile","reparseFiles","_require","require","createRequire","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","Tinypool","workerData","pool","err","featureNames","map","filename","minThreads","maxThreads","Promise","all","result","run","ok","reviveError","error","doc","warnings","destroy","catch","knownColumns","options","threshold","docs","newColumns","seen","fileWarnings","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QA0EgBA;eAAAA;;QAsDMC;eAAAA;;;0BAhIQ;sBACO;wBACP;uBAQJ;6BAEE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,yFAAyF;AACzF,4FAA4F;AAC5F,uFAAuF;AACvF,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAqBnF,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,4FAA4F;AAC5F,uEAAuE;AACvE,IAAME,oBAAoB;AAC1B,IAAMC,sBAAsB;AAE5B,gFAAgF;AAChF,sFAAsF;AACtF,0FAA0F;AAC1F,0FAA0F;AAC1F,4FAA4F;AAC5F,4EAA4E;AAC5E,+FAA+F;AAC/F,+CAA+C;AAC/C,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOL,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCM,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAOO,SAASf,gBAAgBgB,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAe3B,gBAAgBgB,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,uFAAuF;AACvF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB;;YAC1IC,UAKFC,YACAC,MACFR,MAOID,SAUCU;;;;oBAxBDH,WAAa9B,SAAS,YAAtB8B;oBACR,4FAA4F;oBAC5F,8FAA8F;oBAC9F,uFAAuF;oBACvF,0FAA0F;oBACpFC,aAA8B;wBAAEhB,KAAAA;wBAAKmB,cAAcpB,SAASqB,GAAG,CAAC,SAACjB;mCAAYA,QAAQT,IAAI;;oBAAE;oBAC3FuB,OAAO,IAAIF,SAAS;wBAAEM,UAAU9B;wBAAqB+B,YAAYR;wBAAYS,YAAYT;wBAAYE,YAAAA;oBAAW;oBAClHP,OAAO;;;;;;;;;oBAOO;;wBAAMe,QAAQC,GAAG,CAC/BnB,MAAMc,GAAG,CAAC,SAAOnB;;oCACTyB;;;;4CAAU;;gDAAMT,KAAKU,GAAG,CAAC1B;;;4CAAzByB,SAAU;4CAChB,IAAI,CAACA,OAAOE,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACH,OAAOI,KAAK;4CAC9CvB,qBAAAA,+BAAAA,SAAW,EAAEE;4CACb;;gDAAO;oDAAEsB,KAAKL,OAAOK,GAAG;oDAAEC,UAAUN,OAAOM,QAAQ;gDAAC;;;;4BACtD;;;;oBANIxB,UAAU;oBAQhB;;wBAAMS,KAAKgB,OAAO;;;oBAAlB;oBACA;;wBAAOzB;;;oBACAU;oBACP,0FAA0F;oBAC1F,2FAA2F;oBAC3F,2BAA2B;oBAC3B;;wBAAMD,KAAKgB,OAAO,GAAGC,KAAK,CAAC,YAAO;;;oBAAlC;oBACA,MAAMhB;;;;;;;IAEV;;AAIO,SAAelC;wCAAasB,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEmC,YAAiC,EAAE5B,QAAiC;YAAE6B,SAC1IA,oBACCA,qBADbC,WACAvB,YACAN,eAEA8B,MACAN,UACAO,YACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,oBAAQT,KAAeU,cAC1BT,WACK,4BAAA,oBAAA,iBAAA,YAAA,QAAMU;;;;;oBAX+IN,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqBhD;oBACjC0B,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBO,KAAKC,GAAG,CAACvD,qBAAqBwD,IAAAA,4BAAoB;yBAC3EvC,CAAAA,MAAMwC,MAAM,IAAIT,SAAQ,GAAxB/B;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO;;;2BAApD;;;;;;2BAAkET,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA5IC;oBAEA8B;oBACAN;oBACAO;oBACAC,OAAO,IAAIO,IAAIZ;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAyC3B,8BAAzC,6BAAA,QAAA,yBAAA,iCAAkD;0CAAlD,aAAQuB,kBAAAA,KAAeU,2BAAVT;;4BAChBA,CAAAA,YAAAA,UAASrB,IAAI,OAAbqB,WAAc,qBAAGS;4BACZ,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAAClB,IAAImB,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACF,KAAKW,GAAG,CAACT,MAAM;wCAClBF,KAAKY,GAAG,CAACV;wCACTH,WAAW5B,IAAI,CAAC+B;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLJ,KAAK3B,IAAI,CAACoB;wBACZ;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWL;;wBAAO;4BAAEO,MAAAA;4BAAMN,UAAAA;4BAAUO,YAAAA;wBAAW;;;;IACtC"}

package/dist/cjs/scan/worker-error.d.cts

@@ -0,0 +1,9 @@
+import { type SenseErrorCode } from '../errors.js';
+export interface WorkerErrorPayload {
+    name: string;
+    code?: SenseErrorCode;
+    message: string;
+    stack?: string;
+}
+export declare function serializeError(err: unknown): WorkerErrorPayload;
+export declare function reviveError(payload: WorkerErrorPayload): Error;

package/dist/cjs/scan/worker-error.d.ts

@@ -0,0 +1,9 @@
+import { type SenseErrorCode } from '../errors.js';
+export interface WorkerErrorPayload {
+    name: string;
+    code?: SenseErrorCode;
+    message: string;
+    stack?: string;
+}
+export declare function serializeError(err: unknown): WorkerErrorPayload;
+export declare function reviveError(payload: WorkerErrorPayload): Error;

package/dist/cjs/scan/worker-error.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get reviveError () {
+        return reviveError;
+    },
+    get serializeError () {
+        return serializeError;
+    }
+});
+var _errorsts = require("../errors.js");
+function _instanceof(left, right) {
+    "@swc/helpers - instanceof";
+    if (right != null && typeof Symbol !== "undefined" && right[Symbol.hasInstance]) {
+        return !!right[Symbol.hasInstance](left);
+    } else return left instanceof right;
+}
+function serializeError(err) {
+    if (_instanceof(err, Error)) return {
+        name: err.name,
+        code: _instanceof(err, _errorsts.SenseError) ? err.code : undefined,
+        message: err.message,
+        stack: err.stack
+    };
+    return {
+        name: 'Error',
+        message: String(err)
+    };
+}
+function reviveError(payload) {
+    var err = payload.code ? new _errorsts.SenseError(payload.code, payload.message) : new Error(payload.message);
+    err.name = payload.name;
+    if (payload.stack) err.stack = payload.stack;
+    return err;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/scan/worker-error.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/worker-error.ts"],"sourcesContent":["import { SenseError, type SenseErrorCode } from '../errors.ts';\n\n// Structured clone across a worker_threads boundary drops a subclass and its custom\n// properties, so a SenseError arrives as a plain Error with `code` undefined -- and `code`\n// drives caller behavior. The worker serializes explicitly (computed on its side, where\n// `instanceof SenseError` is still meaningful) and the main thread rebuilds by hand.\nexport interface WorkerErrorPayload {\n  name: string;\n  code?: SenseErrorCode;\n  message: string;\n  stack?: string;\n}\n\nexport function serializeError(err: unknown): WorkerErrorPayload {\n  if (err instanceof Error) return { name: err.name, code: err instanceof SenseError ? err.code : undefined, message: err.message, stack: err.stack };\n  return { name: 'Error', message: String(err) };\n}\n\nexport function reviveError(payload: WorkerErrorPayload): Error {\n  const err = payload.code ? new SenseError(payload.code, payload.message) : new Error(payload.message);\n  err.name = payload.name;\n  if (payload.stack) err.stack = payload.stack;\n  return err;\n}\n"],"names":["reviveError","serializeError","err","Error","name","code","SenseError","undefined","message","stack","String","payload"],"mappings":";;;;;;;;;;;QAkBgBA;eAAAA;;QALAC;eAAAA;;;wBAbgC;;;;;;;AAazC,SAASA,eAAeC,GAAY;IACzC,IAAIA,AAAG,YAAHA,KAAeC,QAAO,OAAO;QAAEC,MAAMF,IAAIE,IAAI;QAAEC,MAAMH,AAAG,YAAHA,KAAeI,oBAAU,IAAGJ,IAAIG,IAAI,GAAGE;QAAWC,SAASN,IAAIM,OAAO;QAAEC,OAAOP,IAAIO,KAAK;IAAC;IAClJ,OAAO;QAAEL,MAAM;QAASI,SAASE,OAAOR;IAAK;AAC/C;AAEO,SAASF,YAAYW,OAA2B;IACrD,IAAMT,MAAMS,QAAQN,IAAI,GAAG,IAAIC,oBAAU,CAACK,QAAQN,IAAI,EAAEM,QAAQH,OAAO,IAAI,IAAIL,MAAMQ,QAAQH,OAAO;IACpGN,IAAIE,IAAI,GAAGO,QAAQP,IAAI;IACvB,IAAIO,QAAQF,KAAK,EAAEP,IAAIO,KAAK,GAAGE,QAAQF,KAAK;IAC5C,OAAOP;AACT"}

package/dist/cjs/store/duckdb/reconcile.js

@@ -219,7 +219,7 @@ function contentRow(doc) {
 var MAX_FRONTMATTER_COLUMNS = 10000;
 function reconcile(conn, cfg, baseDir) {
     return _async_to_generator(function() {
-        var files, currentSet, existingStmt, existingRows, existing, vanished, toReparse, features, seenColumns, report, _reparseFiles, parsedDocs, warnings, newColumns, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, allColumns, writableColumns, insertSql, added, delta, addedSet, reparsedExisting;
+        var files, currentSet, existingStmt, existingRows, existing, vanished, toReparse, features, seenColumns, report, _ref, parsedDocs, warnings, newColumns, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, allColumns, writableColumns, insertSql, added, delta, addedSet, reparsedExisting;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -269,7 +269,12 @@ function reconcile(conn, cfg, baseDir) {
                 case 3:
                     seenColumns = _state.sent();
                     report = (0, _progressts.progress)('reparsing files', toReparse.length);
-                    _reparseFiles = (0, _reparsets.reparseFiles)(toReparse, features, cfg, seenColumns, report.tick), parsedDocs = _reparseFiles.docs, warnings = _reparseFiles.warnings, newColumns = _reparseFiles.newColumns;
+                    return [
+                        4,
+                        (0, _reparsets.reparseFiles)(toReparse, features, cfg, seenColumns, report.tick)
+                    ];
+                case 4:
+                    _ref = _state.sent(), parsedDocs = _ref.docs, warnings = _ref.warnings, newColumns = _ref.newColumns;
                     report.finish();
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
@@ -737,7 +742,7 @@ function reconcile(conn, cfg, baseDir) {
                             })();
                         })
                     ];
-                case 4:
+                case 5:
                     _state.sent();
                     return [
                         2,