sensemaking 0.15.4 → 0.16.0

package/dist/cjs/embed/store.js

@@ -0,0 +1,498 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get MODEL_FILENAMES () {
+        return _identityts.MODEL_FILENAMES;
+    },
+    get MODEL_FILES () {
+        return _identityts.MODEL_FILES;
+    },
+    get downloadModel () {
+        return downloadModel;
+    },
+    get downloadModelRevision () {
+        return downloadModelRevision;
+    },
+    get hasModelFiles () {
+        return _identityts.hasModelFiles;
+    },
+    get isDownloadable () {
+        return _identityts.isDownloadable;
+    },
+    get localModelMissing () {
+        return localModelMissing;
+    },
+    get modelDir () {
+        return _identityts.modelDir;
+    },
+    get modelPresent () {
+        return modelPresent;
+    },
+    get readLanguages () {
+        return _identityts.readLanguages;
+    },
+    get refsMainPath () {
+        return _identityts.refsMainPath;
+    },
+    get snapshotDir () {
+        return _identityts.snapshotDir;
+    },
+    get writeLanguages () {
+        return _identityts.writeLanguages;
+    },
+    get writeRef () {
+        return _identityts.writeRef;
+    }
+});
+var _nodefs = require("node:fs");
+var _nodepath = require("node:path");
+var _indexts = require("../config/index.js");
+var _errorsts = require("../errors.js");
+var _httpts = require("./http.js");
+var _identityts = require("./identity.js");
+var _languagests = require("./languages.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function modelPresent(cfg) {
+    var e = (0, _indexts.embedConfig)(cfg);
+    if (!e) return false;
+    return e.provider !== 'static' || (0, _identityts.hasModelFiles)(e.model);
+}
+function localModelMissing(e) {
+    return e !== null && e.provider === 'static' && !(0, _identityts.isDownloadable)(e.model) && !(0, _identityts.hasModelFiles)(e.model);
+}
+// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is
+// removed, so only the first need for an id ever costs this HEAD request.
+function resolveSha(model) {
+    return _async_to_generator(function() {
+        var existing, url, res, err, sha;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    existing = (0, _identityts.readRef)(model);
+                    if (existing) return [
+                        2,
+                        existing
+                    ];
+                    url = "https://huggingface.co/".concat(model, "/resolve/main/model.safetensors");
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        3,
+                        ,
+                        4
+                    ]);
+                    return [
+                        4,
+                        (0, _httpts.fetchWithRetry)(url, {
+                            method: 'HEAD',
+                            redirect: 'manual'
+                        })
+                    ];
+                case 2:
+                    res = _state.sent();
+                    return [
+                        3,
+                        4
+                    ];
+                case 3:
+                    err = _state.sent();
+                    throw new _errorsts.SenseError('EMBED_MODEL', "could not reach ".concat(url, ": ").concat(err.message));
+                case 4:
+                    sha = res.headers.get('x-repo-commit');
+                    if (!sha) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(url, ": response carried no x-repo-commit header"));
+                    (0, _identityts.writeRef)(model, sha);
+                    return [
+                        2,
+                        sha
+                    ];
+            }
+        });
+    })();
+}
+// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has
+// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an
+// array here). Tags are a fallback for cards that only tag languages, filtered to recognized
+// codes so an unrelated short tag (format markers etc.) is never mistaken for one.
+function languagesFromCard(info) {
+    var _info_tags;
+    var _info_cardData;
+    var lang = (_info_cardData = info.cardData) === null || _info_cardData === void 0 ? void 0 : _info_cardData.language;
+    if (typeof lang === 'string') return [
+        lang
+    ];
+    if (Array.isArray(lang) && lang.length > 0) return lang;
+    var tagLangs = ((_info_tags = info.tags) !== null && _info_tags !== void 0 ? _info_tags : []).filter(_languagests.isKnownLanguageTag);
+    return tagLangs.length > 0 ? tagLangs : undefined;
+}
+// Best-effort, cached forever once resolved (even to "none"): unlike model weights, missing or
+// unreachable metadata is not an error -- it just means the language-fit check stays off.
+function ensureLanguages(model) {
+    return _async_to_generator(function() {
+        var _languagesFromCard, measured, res, unused, info, unused1;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if ((0, _nodefs.existsSync)((0, _identityts.languagesPath)(model))) return [
+                        2
+                    ];
+                    // A measured entry needs no card and no network, and stays deterministic offline.
+                    measured = _languagests.MEASURED_MODEL_LANGUAGES[model];
+                    if (measured) {
+                        (0, _identityts.writeLanguages)(model, measured);
+                        return [
+                            2
+                        ];
+                    }
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        3,
+                        ,
+                        4
+                    ]);
+                    return [
+                        4,
+                        (0, _httpts.fetchWithRetry)("https://huggingface.co/api/models/".concat(model), {})
+                    ];
+                case 2:
+                    res = _state.sent();
+                    return [
+                        3,
+                        4
+                    ];
+                case 3:
+                    unused = _state.sent();
+                    return [
+                        2
+                    ];
+                case 4:
+                    if (!res.ok) return [
+                        2
+                    ];
+                    _state.label = 5;
+                case 5:
+                    _state.trys.push([
+                        5,
+                        7,
+                        ,
+                        8
+                    ]);
+                    return [
+                        4,
+                        res.json()
+                    ];
+                case 6:
+                    info = _state.sent();
+                    return [
+                        3,
+                        8
+                    ];
+                case 7:
+                    unused1 = _state.sent();
+                    return [
+                        2
+                    ];
+                case 8:
+                    (0, _identityts.writeLanguages)(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : []);
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function fetchToFile(url, dest) {
+    return _async_to_generator(function() {
+        var res, err, _tmp, _;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _state.trys.push([
+                        0,
+                        2,
+                        ,
+                        3
+                    ]);
+                    return [
+                        4,
+                        (0, _httpts.fetchWithRetry)(url, {})
+                    ];
+                case 1:
+                    res = _state.sent();
+                    return [
+                        3,
+                        3
+                    ];
+                case 2:
+                    err = _state.sent();
+                    throw new _errorsts.SenseError('EMBED_MODEL', "could not reach ".concat(url, ": ").concat(err.message));
+                case 3:
+                    if (!res.ok) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(url, " -> HTTP ").concat(res.status));
+                    _tmp = [
+                        "".concat(dest, ".part")
+                    ];
+                    _ = Buffer.from;
+                    return [
+                        4,
+                        res.arrayBuffer()
+                    ];
+                case 4:
+                    _nodefs.writeFileSync.apply(void 0, _tmp.concat([
+                        _.apply(Buffer, [
+                            _state.sent()
+                        ])
+                    ]));
+                    (0, _nodefs.renameSync)("".concat(dest, ".part"), dest);
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function downloadModelRevision(model, sha, onFile) {
+    return _async_to_generator(function() {
+        var dir, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    dir = (0, _identityts.snapshotDir)(model, sha);
+                    (0, _nodefs.mkdirSync)(dir, {
+                        recursive: true
+                    });
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        6,
+                        7,
+                        8
+                    ]);
+                    _iterator = _identityts.MODEL_FILES[Symbol.iterator]();
+                    _state.label = 2;
+                case 2:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        5
+                    ];
+                    file = _step.value;
+                    if ((0, _nodefs.existsSync)((0, _nodepath.join)(dir, file))) return [
+                        3,
+                        4
+                    ];
+                    onFile === null || onFile === void 0 ? void 0 : onFile(file, dir);
+                    return [
+                        4,
+                        fetchToFile("https://huggingface.co/".concat(model, "/resolve/").concat(sha, "/").concat(file), (0, _nodepath.join)(dir, file))
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        2
+                    ];
+                case 5:
+                    return [
+                        3,
+                        8
+                    ];
+                case 6:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        8
+                    ];
+                case 7:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 8:
+                    return [
+                        2,
+                        dir
+                    ];
+            }
+        });
+    })();
+}
+function downloadModel(model, onFile) {
+    return _async_to_generator(function() {
+        var sha;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (!(0, _identityts.isDownloadable)(model)) {
+                        throw new _errorsts.SenseError('EMBED_MODEL', 'embed.model "'.concat(model, '" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like "minishlab/potion-retrieval-32M"'));
+                    }
+                    return [
+                        4,
+                        resolveSha(model)
+                    ];
+                case 1:
+                    sha = _state.sent();
+                    // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first
+                    // construction and an explicit prefetch both end up with a cached language verdict.
+                    return [
+                        4,
+                        ensureLanguages(model)
+                    ];
+                case 2:
+                    _state.sent();
+                    return [
+                        2,
+                        downloadModelRevision(model, sha, onFile)
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata). Re-exported here for callers that otherwise\n// only need store.ts's network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string): Promise<string> {\n  const existing = readRef(model);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has\n// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an\n// array here). Tags are a fallback for cards that only tag languages, filtered to recognized\n// codes so an unrelated short tag (format markers etc.) is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Best-effort, cached forever once resolved (even to \"none\"): unlike model weights, missing or\n// unreachable metadata is not an error -- it just means the language-fit check stays off.\nasync function ensureLanguages(model: string): Promise<void> {\n  if (existsSync(languagesPath(model))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    return;\n  }\n  if (!res.ok) return;\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? []);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n  renameSync(`${dest}.part`, dest);\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  const dir = snapshotDir(model, sha);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model);\n  return downloadModelRevision(model, sha, onFile);\n}\n"],"names":["MODEL_FILENAMES","MODEL_FILES","downloadModel","downloadModelRevision","hasModelFiles","isDownloadable","localModelMissing","modelDir","modelPresent","readLanguages","refsMainPath","snapshotDir","writeLanguages","writeRef","cfg","e","embedConfig","provider","model","resolveSha","existing","url","res","err","sha","readRef","fetchWithRetry","method","redirect","SenseError","message","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","isKnownLanguageTag","undefined","ensureLanguages","measured","existsSync","languagesPath","MEASURED_MODEL_LANGUAGES","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","writeFileSync","renameSync","onFile","dir","file","mkdirSync","recursive","join"],"mappings":";;;;;;;;;;;QAYwCA;eAAAA,2BAAe;;QAAEC;eAAAA,uBAAW;;QA4G9CC;eAAAA;;QAbAC;eAAAA;;QA/FbC;eAAAA,yBAAa;;QAAEC;eAAAA,0BAAc;;QActBC;eAAAA;;QAdsDC;eAAAA,oBAAQ;;QAI9DC;eAAAA;;QAJgEC;eAAAA,yBAAa;;QAAEC;eAAAA,wBAAY;;QAAEC;eAAAA,uBAAW;;QAAEC;eAAAA,0BAAc;;QAAEC;eAAAA,oBAAQ;;;sBAZjF;wBAC5C;uBAEO;wBACD;sBACI;0BAC2F;2BAC7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAStD,SAASL,aAAaM,GAAW;IACtC,IAAMC,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEE,QAAQ,KAAK,YAAYb,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACzD;AAMO,SAASZ,kBAAkBS,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEE,QAAQ,KAAK,YAAY,CAACZ,IAAAA,0BAAc,EAACU,EAAEG,KAAK,KAAK,CAACd,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,SAAeC,WAAWD,KAAa;;YAC/BE,UAEAC,KAGFC,KAGKC,KAGHC;;;;oBAXAJ,WAAWK,IAAAA,mBAAO,EAACP;oBACzB,IAAIE,UAAU;;wBAAOA;;oBACfC,MAAM,AAAC,0BAA+B,OAANH,OAAM;;;;;;;;;oBAKpC;;wBAAMQ,IAAAA,sBAAc,EAACL,KAAK;4BAAEM,QAAQ;4BAAQC,UAAU;wBAAS;;;oBAArEN,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIM,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAARR,KAAI,MAA2B,OAAvB,AAACE,IAAcO,OAAO;;oBAEjFN,MAAMF,IAAIS,OAAO,CAACC,GAAG,CAAC;oBAC5B,IAAI,CAACR,KAAK,MAAM,IAAIK,oBAAU,CAAC,eAAe,AAAC,GAAM,OAAJR,KAAI;oBACrDR,IAAAA,oBAAQ,EAACK,OAAOM;oBAChB;;wBAAOA;;;;IACT;;AAOA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,mFAAmF;AACnF,SAASS,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,IAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,IAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAACC,+BAAkB;IAC5D,OAAOH,SAASD,MAAM,GAAG,IAAIC,WAAWI;AAC1C;AAEA,+FAA+F;AAC/F,0FAA0F;AAC1F,SAAeC,gBAAgB5B,KAAa;;YAqBpBe,oBAlBhBc,UAKFzB,aAOAY;;;;oBAdJ,IAAIc,IAAAA,kBAAU,EAACC,IAAAA,yBAAa,EAAC/B,SAAS;;;oBACtC,kFAAkF;oBAC5E6B,WAAWG,qCAAwB,CAAChC,MAAM;oBAChD,IAAI6B,UAAU;wBACZnC,IAAAA,0BAAc,EAACM,OAAO6B;wBACtB;;;oBACF;;;;;;;;;oBAGQ;;wBAAMrB,IAAAA,sBAAc,EAAC,AAAC,qCAA0C,OAANR,QAAS,CAAC;;;oBAA1EI,MAAM;;;;;;;oBAEN;;;;oBAEF,IAAI,CAACA,IAAI6B,EAAE,EAAE;;;;;;;;;;;oBAGH;;wBAAM7B,IAAI8B,IAAI;;;oBAAtBlB,OAAQ;;;;;;;oBAER;;;;oBAEFtB,IAAAA,0BAAc,EAACM,QAAOe,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA;;;;;;IACxB;;AAEA,SAAeoB,YAAYhC,GAAW,EAAEiC,IAAY;;YAC9ChC,KAGKC;;;;;;;;;;oBADD;;wBAAMG,IAAAA,sBAAc,EAACL,KAAK,CAAC;;;oBAAjCC,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIM,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAARR,KAAI,MAA2B,OAAvB,AAACE,IAAcO,OAAO;;oBAEvF,IAAI,CAACR,IAAI6B,EAAE,EAAE,MAAM,IAAItB,oBAAU,CAAC,eAAe,AAAC,GAAiBP,OAAfD,KAAI,aAAsB,OAAXC,IAAIiC,MAAM;;wBAC9D,GAAO,OAALD,MAAK;;wBAAQE,OAAOC,IAAI;oBAAC;;wBAAMnC,IAAIoC,WAAW;;;oBAA/DC,qBAAa;wBAAiBH,QAAAA;4BAAY;;;oBAC1CI,IAAAA,kBAAU,EAAC,AAAC,GAAO,OAALN,MAAK,UAAQA;;;;;;IAC7B;;AAIO,SAAenD,sBAAsBe,KAAa,EAAEM,GAAW,EAAEqC,MAA4C;;YAC5GC,KAED,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAFLD,MAAMnD,IAAAA,uBAAW,EAACO,OAAOM;oBAC/BwC,IAAAA,iBAAS,EAACF,KAAK;wBAAEG,WAAW;oBAAK;oBAC5B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAchE,uBAAW;;;2BAAzB,6BAAA,QAAA;;;;oBAAM8D,OAAN;oBACH,IAAIf,IAAAA,kBAAU,EAACkB,IAAAA,cAAI,EAACJ,KAAKC,QAAQ;;;;oBACjCF,mBAAAA,6BAAAA,OAASE,MAAMD;oBACf;;wBAAMT,YAAY,AAAC,0BAA0C7B,OAAjBN,OAAM,aAAkB6C,OAAPvC,KAAI,KAAQ,OAALuC,OAAQG,IAAAA,cAAI,EAACJ,KAAKC;;;oBAAtF;;;oBAHG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAKL;;wBAAOD;;;;IACT;;AAIO,SAAe5D,cAAcgB,KAAa,EAAE2C,MAA4C;;YAIvFrC;;;;oBAHN,IAAI,CAACnB,IAAAA,0BAAc,EAACa,QAAQ;wBAC1B,MAAM,IAAIW,oBAAU,CAAC,eAAe,AAAC,gBAAqB,OAANX,OAAM;oBAC5D;oBACY;;wBAAMC,WAAWD;;;oBAAvBM,MAAM;oBACZ,sFAAsF;oBACtF,oFAAoF;oBACpF;;wBAAMsB,gBAAgB5B;;;oBAAtB;oBACA;;wBAAOf,sBAAsBe,OAAOM,KAAKqC;;;;IAC3C"}

package/dist/cjs/embed/types.d.cts

@@ -0,0 +1,8 @@
+export interface EmbedProvider {
+    id: string;
+    dims: number;
+    batchCap: number;
+    languages?: string[];
+    embedDocuments(texts: string[]): Promise<Float32Array[]>;
+    embedQuery(text: string): Promise<Float32Array>;
+}

package/dist/cjs/embed/types.d.ts

@@ -0,0 +1,8 @@
+export interface EmbedProvider {
+    id: string;
+    dims: number;
+    batchCap: number;
+    languages?: string[];
+    embedDocuments(texts: string[]): Promise<Float32Array[]>;
+    embedQuery(text: string): Promise<Float32Array>;
+}

package/dist/cjs/embed/types.js

@@ -0,0 +1,7 @@
+// The provider contract every wire protocol implements: a Liskov-substitutable
+// surface so query/search code never branches on provider identity.
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,oEAAoE"}

package/dist/cjs/errors.d.cts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.d.ts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAkBxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAmBxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}

package/dist/cjs/features/embed.d.cts

@@ -1,30 +1,7 @@
-import type { DatabaseSync } from 'node:sqlite';
-import type { Config } from '../config/index.js';
 import type { Feature } from './types.js';
-export interface EmbedProvider {
-    id: string;
-    dims: number;
-    embed(texts: string[]): Promise<Float32Array[]>;
+export interface Chunk {
+    startLine: number;
+    endLine: number;
+    text: string;
 }
 export declare const embed: Feature;
-export declare const MODEL_FILENAMES: string;
-export declare function isDownloadable(model: string): boolean;
-export declare function modelDir(model: string): string;
-export declare function hasModelFiles(model: string): boolean;
-export declare function modelPresent(cfg: Config): boolean;
-export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;
-export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
-export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
-    path: string;
-    lines: string;
-    similarity: number;
-}>>;
-export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
-export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
-    exclude: Set<string>;
-    allowed?: Set<string>;
-    k: number;
-}): Array<{
-    path: string;
-    similarity: number;
-}>;

package/dist/cjs/features/embed.d.ts

@@ -1,30 +1,7 @@
-import type { DatabaseSync } from 'node:sqlite';
-import type { Config } from '../config/index.js';
 import type { Feature } from './types.js';
-export interface EmbedProvider {
-    id: string;
-    dims: number;
-    embed(texts: string[]): Promise<Float32Array[]>;
+export interface Chunk {
+    startLine: number;
+    endLine: number;
+    text: string;
 }
 export declare const embed: Feature;
-export declare const MODEL_FILENAMES: string;
-export declare function isDownloadable(model: string): boolean;
-export declare function modelDir(model: string): string;
-export declare function hasModelFiles(model: string): boolean;
-export declare function modelPresent(cfg: Config): boolean;
-export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;
-export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
-export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
-    path: string;
-    lines: string;
-    similarity: number;
-}>>;
-export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
-export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
-    exclude: Set<string>;
-    allowed?: Set<string>;
-    k: number;
-}): Array<{
-    path: string;
-    similarity: number;
-}>;