sensemaking 0.15.4 → 0.16.0

package/dist/cjs/embed/identity.js

@@ -0,0 +1,194 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get MODEL_FILENAMES () {
+        return MODEL_FILENAMES;
+    },
+    get MODEL_FILES () {
+        return MODEL_FILES;
+    },
+    get hasModelFiles () {
+        return hasModelFiles;
+    },
+    get isDownloadable () {
+        return isDownloadable;
+    },
+    get languagesPath () {
+        return languagesPath;
+    },
+    get modelDir () {
+        return modelDir;
+    },
+    get modelIdentity () {
+        return modelIdentity;
+    },
+    get readLanguages () {
+        return readLanguages;
+    },
+    get readRef () {
+        return readRef;
+    },
+    get refsMainPath () {
+        return refsMainPath;
+    },
+    get snapshotDir () {
+        return snapshotDir;
+    },
+    get writeLanguages () {
+        return writeLanguages;
+    },
+    get writeRef () {
+        return writeRef;
+    }
+});
+var _nodefs = require("node:fs");
+var _nodeos = require("node:os");
+var _nodepath = require("node:path");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var MODEL_FILES = [
+    'model.safetensors',
+    'tokenizer.json'
+];
+var MODEL_FILENAMES = MODEL_FILES.join(' and ');
+// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather
+// than resolved against the HF cache.
+var HF_MODEL_ID = /^[A-Za-z0-9][\w.-]*\/[A-Za-z0-9][\w.-]*$/;
+function isDownloadable(model) {
+    return HF_MODEL_ID.test(model);
+}
+function repoDir(model) {
+    var _model_split = _sliced_to_array(model.split('/'), 2), org = _model_split[0], name = _model_split[1];
+    return (0, _nodepath.join)((0, _nodeos.homedir)(), '.sense', 'models', "models--".concat(org, "--").concat(name));
+}
+function refsMainPath(model) {
+    return (0, _nodepath.join)(repoDir(model), 'refs', 'main');
+}
+function readRef(model) {
+    var p = refsMainPath(model);
+    if (!(0, _nodefs.existsSync)(p)) return undefined;
+    var sha = (0, _nodefs.readFileSync)(p, 'utf8').trim();
+    return sha.length > 0 ? sha : undefined;
+}
+function writeRef(model, sha) {
+    (0, _nodefs.mkdirSync)((0, _nodepath.join)(repoDir(model), 'refs'), {
+        recursive: true
+    });
+    (0, _nodefs.writeFileSync)(refsMainPath(model), sha);
+}
+function snapshotDir(model, sha) {
+    return (0, _nodepath.join)(repoDir(model), 'snapshots', sha);
+}
+function languagesPath(model) {
+    return (0, _nodepath.join)(repoDir(model), 'languages.json');
+}
+function readLanguages(model) {
+    var p = languagesPath(model);
+    if (!(0, _nodefs.existsSync)(p)) return undefined;
+    try {
+        return JSON.parse((0, _nodefs.readFileSync)(p, 'utf8'));
+    } catch (unused) {
+        return undefined;
+    }
+}
+function writeLanguages(model, languages) {
+    (0, _nodefs.mkdirSync)(repoDir(model), {
+        recursive: true
+    });
+    (0, _nodefs.writeFileSync)(languagesPath(model), JSON.stringify(languages));
+}
+function modelDir(model) {
+    if (!isDownloadable(model)) return model;
+    var sha = readRef(model);
+    return sha ? snapshotDir(model, sha) : repoDir(model);
+}
+function hasModelFiles(model) {
+    var dir = modelDir(model);
+    return MODEL_FILES.every(function(file) {
+        return (0, _nodefs.existsSync)((0, _nodepath.join)(dir, file));
+    });
+}
+function modelIdentity(model) {
+    if (isDownloadable(model)) return readRef(model);
+    var dir = modelDir(model);
+    var parts = [];
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = MODEL_FILES[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var file = _step.value;
+            try {
+                var st = (0, _nodefs.statSync)((0, _nodepath.join)(dir, file));
+                parts.push("".concat(st.size, ":").concat(Math.trunc(st.mtimeMs)));
+            } catch (unused) {
+                return undefined;
+            }
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return parts.join(',');
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/identity.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/identity.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\n\n// HF cache layout, no network: ~/.sense/models/models--<org>--<name>/{refs/main,\n// snapshots/<sha>/<files>}, files stored directly (hf_hub's blobs/symlinks layer skipped).\n\nexport const MODEL_FILES = ['model.safetensors', 'tokenizer.json'];\n// The pair, for messages that name what a local model directory must contain.\nexport const MODEL_FILENAMES = MODEL_FILES.join(' and ');\n\n// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather\n// than resolved against the HF cache.\nconst HF_MODEL_ID = /^[A-Za-z0-9][\\w.-]*\\/[A-Za-z0-9][\\w.-]*$/;\n\nexport function isDownloadable(model: string): boolean {\n  return HF_MODEL_ID.test(model);\n}\n\nfunction repoDir(model: string): string {\n  const [org, name] = model.split('/');\n  return join(homedir(), '.sense', 'models', `models--${org}--${name}`);\n}\n\nexport function refsMainPath(model: string): string {\n  return join(repoDir(model), 'refs', 'main');\n}\n\n// What `main` last resolved to, read from disk -- no network. Undefined until some resolution\n// (a download or an explicit prefetch) has written it once.\nexport function readRef(model: string): string | undefined {\n  const p = refsMainPath(model);\n  if (!existsSync(p)) return undefined;\n  const sha = readFileSync(p, 'utf8').trim();\n  return sha.length > 0 ? sha : undefined;\n}\n\n// A recorded ref pins until its snapshot directory is removed; nothing here re-resolves it.\nexport function writeRef(model: string, sha: string): void {\n  mkdirSync(join(repoDir(model), 'refs'), { recursive: true });\n  writeFileSync(refsMainPath(model), sha);\n}\n\nexport function snapshotDir(model: string, sha: string): string {\n  return join(repoDir(model), 'snapshots', sha);\n}\n\n// Languages describe the repo, not one weight snapshot, so this lives beside refs/, not\n// inside a snapshot dir.\nexport function languagesPath(model: string): string {\n  return join(repoDir(model), 'languages.json');\n}\n\n// undefined means \"never resolved yet\"; an empty array is a resolved fact (checked, none\n// declared) and is cached the same way, so a languageless model is not re-fetched every run.\nexport function readLanguages(model: string): string[] | undefined {\n  const p = languagesPath(model);\n  if (!existsSync(p)) return undefined;\n  try {\n    return JSON.parse(readFileSync(p, 'utf8'));\n  } catch {\n    return undefined;\n  }\n}\n\nexport function writeLanguages(model: string, languages: string[]): void {\n  mkdirSync(repoDir(model), { recursive: true });\n  writeFileSync(languagesPath(model), JSON.stringify(languages));\n}\n\n// A local directory the caller manages, or the resolved snapshot once `main` is pinned; an HF\n// id with no recorded ref yet has no known snapshot, so this names the (fileless) repo dir.\nexport function modelDir(model: string): string {\n  if (!isDownloadable(model)) return model;\n  const sha = readRef(model);\n  return sha ? snapshotDir(model, sha) : repoDir(model);\n}\n\n// Both files, so an interrupted download reads as absent and the next one resumes it.\nexport function hasModelFiles(model: string): boolean {\n  const dir = modelDir(model);\n  return MODEL_FILES.every((file) => existsSync(join(dir, file)));\n}\n\n// Weight identity for the signature, no network: a HF id's resolved sha, or the two\n// files' size+mtime for a local path. Undefined when nothing is known yet.\nexport function modelIdentity(model: string): string | undefined {\n  if (isDownloadable(model)) return readRef(model);\n  const dir = modelDir(model);\n  const parts: string[] = [];\n  for (const file of MODEL_FILES) {\n    try {\n      const st = statSync(join(dir, file));\n      parts.push(`${st.size}:${Math.trunc(st.mtimeMs)}`);\n    } catch {\n      return undefined;\n    }\n  }\n  return parts.join(',');\n}\n"],"names":["MODEL_FILENAMES","MODEL_FILES","hasModelFiles","isDownloadable","languagesPath","modelDir","modelIdentity","readLanguages","readRef","refsMainPath","snapshotDir","writeLanguages","writeRef","join","HF_MODEL_ID","model","test","repoDir","split","org","name","homedir","p","existsSync","undefined","sha","readFileSync","trim","length","mkdirSync","recursive","writeFileSync","JSON","parse","languages","stringify","dir","every","file","parts","st","statSync","push","Math","size","trunc","mtimeMs"],"mappings":";;;;;;;;;;;QASaA;eAAAA;;QAFAC;eAAAA;;QAwEGC;eAAAA;;QAhEAC;eAAAA;;QAkCAC;eAAAA;;QAuBAC;eAAAA;;QAcAC;eAAAA;;QA/BAC;eAAAA;;QAzBAC;eAAAA;;QANAC;eAAAA;;QAmBAC;eAAAA;;QAsBAC;eAAAA;;QA3BAC;eAAAA;;;sBAtC6D;sBACrD;wBACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKd,IAAMX,cAAc;IAAC;IAAqB;CAAiB;AAE3D,IAAMD,kBAAkBC,YAAYY,IAAI,CAAC;AAEhD,+FAA+F;AAC/F,sCAAsC;AACtC,IAAMC,cAAc;AAEb,SAASX,eAAeY,KAAa;IAC1C,OAAOD,YAAYE,IAAI,CAACD;AAC1B;AAEA,SAASE,QAAQF,KAAa;IAC5B,IAAoBA,gCAAAA,MAAMG,KAAK,CAAC,UAAzBC,MAAaJ,iBAARK,OAAQL;IACpB,OAAOF,IAAAA,cAAI,EAACQ,IAAAA,eAAO,KAAI,UAAU,UAAU,AAAC,WAAkBD,OAARD,KAAI,MAAS,OAALC;AAChE;AAEO,SAASX,aAAaM,KAAa;IACxC,OAAOF,IAAAA,cAAI,EAACI,QAAQF,QAAQ,QAAQ;AACtC;AAIO,SAASP,QAAQO,KAAa;IACnC,IAAMO,IAAIb,aAAaM;IACvB,IAAI,CAACQ,IAAAA,kBAAU,EAACD,IAAI,OAAOE;IAC3B,IAAMC,MAAMC,IAAAA,oBAAY,EAACJ,GAAG,QAAQK,IAAI;IACxC,OAAOF,IAAIG,MAAM,GAAG,IAAIH,MAAMD;AAChC;AAGO,SAASZ,SAASG,KAAa,EAAEU,GAAW;IACjDI,IAAAA,iBAAS,EAAChB,IAAAA,cAAI,EAACI,QAAQF,QAAQ,SAAS;QAAEe,WAAW;IAAK;IAC1DC,IAAAA,qBAAa,EAACtB,aAAaM,QAAQU;AACrC;AAEO,SAASf,YAAYK,KAAa,EAAEU,GAAW;IACpD,OAAOZ,IAAAA,cAAI,EAACI,QAAQF,QAAQ,aAAaU;AAC3C;AAIO,SAASrB,cAAcW,KAAa;IACzC,OAAOF,IAAAA,cAAI,EAACI,QAAQF,QAAQ;AAC9B;AAIO,SAASR,cAAcQ,KAAa;IACzC,IAAMO,IAAIlB,cAAcW;IACxB,IAAI,CAACQ,IAAAA,kBAAU,EAACD,IAAI,OAAOE;IAC3B,IAAI;QACF,OAAOQ,KAAKC,KAAK,CAACP,IAAAA,oBAAY,EAACJ,GAAG;IACpC,EAAE,eAAM;QACN,OAAOE;IACT;AACF;AAEO,SAASb,eAAeI,KAAa,EAAEmB,SAAmB;IAC/DL,IAAAA,iBAAS,EAACZ,QAAQF,QAAQ;QAAEe,WAAW;IAAK;IAC5CC,IAAAA,qBAAa,EAAC3B,cAAcW,QAAQiB,KAAKG,SAAS,CAACD;AACrD;AAIO,SAAS7B,SAASU,KAAa;IACpC,IAAI,CAACZ,eAAeY,QAAQ,OAAOA;IACnC,IAAMU,MAAMjB,QAAQO;IACpB,OAAOU,MAAMf,YAAYK,OAAOU,OAAOR,QAAQF;AACjD;AAGO,SAASb,cAAca,KAAa;IACzC,IAAMqB,MAAM/B,SAASU;IACrB,OAAOd,YAAYoC,KAAK,CAAC,SAACC;eAASf,IAAAA,kBAAU,EAACV,IAAAA,cAAI,EAACuB,KAAKE;;AAC1D;AAIO,SAAShC,cAAcS,KAAa;IACzC,IAAIZ,eAAeY,QAAQ,OAAOP,QAAQO;IAC1C,IAAMqB,MAAM/B,SAASU;IACrB,IAAMwB,QAAkB,EAAE;QACrB,kCAAA,2BAAA;;QAAL,QAAK,YAActC,gCAAd,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMqC,OAAN;YACH,IAAI;gBACF,IAAME,KAAKC,IAAAA,gBAAQ,EAAC5B,IAAAA,cAAI,EAACuB,KAAKE;gBAC9BC,MAAMG,IAAI,CAAC,AAAC,GAAaC,OAAXH,GAAGI,IAAI,EAAC,KAA0B,OAAvBD,KAAKE,KAAK,CAACL,GAAGM,OAAO;YAChD,EAAE,eAAM;gBACN,OAAOtB;YACT;QACF;;QAPK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,OAAOe,MAAM1B,IAAI,CAAC;AACpB"}

package/dist/cjs/embed/langfit.d.cts

@@ -0,0 +1,6 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { EmbedProvider } from './types.js';
+type LangCounts = Record<string, number>;
+export declare function languageDistribution(db: DatabaseSync): LangCounts | undefined;
+export declare function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void>;
+export {};

package/dist/cjs/embed/langfit.d.ts

@@ -0,0 +1,6 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { EmbedProvider } from './types.js';
+type LangCounts = Record<string, number>;
+export declare function languageDistribution(db: DatabaseSync): LangCounts | undefined;
+export declare function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void>;
+export {};

package/dist/cjs/embed/langfit.js

@@ -0,0 +1,326 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get checkLanguageFit () {
+        return checkLanguageFit;
+    },
+    get languageDistribution () {
+        return languageDistribution;
+    }
+});
+var _sharedts = require("../db/shared.js");
+var _errorsts = require("../errors.js");
+var _languagests = require("./languages.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
+function _getRequireWildcardCache(nodeInterop) {
+    if (typeof WeakMap !== "function") return null;
+    var cacheBabelInterop = new WeakMap();
+    var cacheNodeInterop = new WeakMap();
+    return (_getRequireWildcardCache = function(nodeInterop) {
+        return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
+    })(nodeInterop);
+}
+function _interop_require_wildcard(obj, nodeInterop) {
+    if (!nodeInterop && obj && obj.__esModule) return obj;
+    if (obj === null || typeof obj !== "object" && typeof obj !== "function") return {
+        default: obj
+    };
+    var cache = _getRequireWildcardCache(nodeInterop);
+    if (cache && cache.has(obj)) return cache.get(obj);
+    var newObj = {
+        __proto__: null
+    };
+    var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
+    for(var key in obj){
+        if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
+            var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
+            if (desc && (desc.get || desc.set)) Object.defineProperty(newObj, key, desc);
+            else newObj[key] = obj[key];
+        }
+    }
+    newObj.default = obj;
+    if (cache) cache.set(obj, newObj);
+    return newObj;
+}
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var META_KEY = 'embed_languages';
+// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a
+// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).
+var SAMPLE_CHARS = 300;
+// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule
+// never fires below this many classified chunks, declared languages or not.
+var MIN_CLASSIFIED = 10;
+function languageDistribution(db) {
+    var raw = (0, _sharedts.getMeta)(db, META_KEY);
+    return raw ? JSON.parse(raw) : undefined;
+}
+function classify(franc, text) {
+    var code = franc(text.slice(0, SAMPLE_CHARS));
+    return code === 'und' ? undefined : code;
+}
+function checkLanguageFit(db, provider, texts) {
+    return _async_to_generator(function() {
+        var _languageDistribution, franc, persisted, merged, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, text, _merged_code, code, total, declared, _Object_entries_sort_, majorityLang, majorityCount, pct;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (texts.length === 0) return [
+                        2
+                    ];
+                    return [
+                        4,
+                        Promise.resolve().then(function() {
+                            return /*#__PURE__*/ _interop_require_wildcard(require("franc-min"));
+                        })
+                    ];
+                case 1:
+                    franc = _state.sent().franc;
+                    persisted = (_languageDistribution = languageDistribution(db)) !== null && _languageDistribution !== void 0 ? _languageDistribution : {};
+                    merged = _object_spread({}, persisted);
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        for(_iterator = texts[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                            text = _step.value;
+                            ;
+                            code = classify(franc, text);
+                            if (code) merged[code] = ((_merged_code = merged[code]) !== null && _merged_code !== void 0 ? _merged_code : 0) + 1;
+                        }
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
+                    }
+                    total = Object.values(merged).reduce(function(a, b) {
+                        return a + b;
+                    }, 0);
+                    if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {
+                        declared = new Set(provider.languages.map(_languagests.toIso3));
+                        _Object_entries_sort_ = _sliced_to_array(Object.entries(merged).sort(function(a, b) {
+                            return b[1] - a[1];
+                        })[0], 2), majorityLang = _Object_entries_sort_[0], majorityCount = _Object_entries_sort_[1];
+                        if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {
+                            pct = Math.round(majorityCount / total * 100);
+                            throw new _errorsts.SenseError('EMBED_MODEL_MISMATCH', "embed model ".concat(provider.id, " declares languages [").concat(provider.languages.join(', '), "], but ").concat(pct, "% of this tree's classified text is \"").concat(majorityLang, "\" (ISO 639-3), which is not among them; choose a model for this tree's languages: `sense init --model ...`, see the sense-setup skill or INTEGRATIONS.md"));
+                        }
+                    }
+                    (0, _sharedts.setMeta)(db, META_KEY, JSON.stringify(merged));
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/langfit.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getMeta, setMeta } from '../db/shared.ts';\nimport { SenseError } from '../errors.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst META_KEY = 'embed_languages';\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\ntype LangCounts = Record<string, number>;\n\nexport function languageDistribution(db: DatabaseSync): LangCounts | undefined {\n  const raw = getMeta(db, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nfunction classify(franc: (text: string) => string, text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const { franc } = await import('franc-min');\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(franc, text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  setMeta(db, META_KEY, JSON.stringify(merged));\n}\n"],"names":["checkLanguageFit","languageDistribution","META_KEY","SAMPLE_CHARS","MIN_CLASSIFIED","db","raw","getMeta","JSON","parse","undefined","classify","franc","text","code","slice","provider","texts","persisted","merged","total","declared","Object","majorityLang","majorityCount","pct","length","values","reduce","a","b","languages","Set","map","toIso3","entries","sort","has","Math","round","SenseError","id","join","setMeta","stringify"],"mappings":";;;;;;;;;;;QA4BsBA;eAAAA;;QAZNC;eAAAA;;;wBAfiB;wBACN;2BACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvB,IAAMC,WAAW;AACjB,+FAA+F;AAC/F,8FAA8F;AAC9F,IAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,IAAMC,iBAAiB;AAIhB,SAASH,qBAAqBI,EAAgB;IACnD,IAAMC,MAAMC,IAAAA,iBAAO,EAACF,IAAIH;IACxB,OAAOI,MAAME,KAAKC,KAAK,CAACH,OAAOI;AACjC;AAEA,SAASC,SAASC,KAA+B,EAAEC,IAAY;IAC7D,IAAMC,OAAOF,MAAMC,KAAKE,KAAK,CAAC,GAAGZ;IACjC,OAAOW,SAAS,QAAQJ,YAAYI;AACtC;AAIO,SAAed,iBAAiBK,EAAgB,EAAEW,QAAuB,EAAEC,KAAe;;YAG7EhB,uBADVW,OACFM,WACAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMN,MAEiBM,cADpBL,MAIFM,OAEEC,UACgCC,uBAA/BC,cAAcC,eAEbC;;;;oBAdV,IAAIR,MAAMS,MAAM,KAAK,GAAG;;;oBACN;;wBAAM;2EAAA,QAAO;;;;oBAAvBd,QAAU,cAAVA;oBACFM,aAAYjB,wBAAAA,qBAAqBI,iBAArBJ,mCAAAA,wBAA4B,CAAC;oBACzCkB,SAAS,mBAAKD;oBACf,kCAAA,2BAAA;;wBAAL,IAAK,YAAcD,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAfJ,OAAN;;4BACGC,OAAOH,SAASC,OAAOC;4BAC7B,IAAIC,MAAMK,MAAM,CAACL,KAAK,GAAG,EAACK,eAAAA,MAAM,CAACL,KAAK,cAAZK,0BAAAA,eAAgB,KAAK;wBACjD;;wBAHK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKCC,QAAQE,OAAOK,MAAM,CAACR,QAAQS,MAAM,CAAC,SAACC,GAAGC;+BAAMD,IAAIC;uBAAG;oBAC5D,IAAId,SAASe,SAAS,IAAIf,SAASe,SAAS,CAACL,MAAM,GAAG,KAAKN,SAAShB,gBAAgB;wBAC5EiB,WAAW,IAAIW,IAAIhB,SAASe,SAAS,CAACE,GAAG,CAACC,mBAAM;wBAChBZ,yCAAAA,OAAOa,OAAO,CAAChB,QAAQiB,IAAI,CAAC,SAACP,GAAGC;mCAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;0BAAC,CAAC,EAAE,MAApFN,eAA+BD,0BAAjBE,gBAAiBF;wBACtC,IAAIE,gBAAgBJ,SAAS,OAAO,CAACC,SAASgB,GAAG,CAACd,eAAe;4BACzDE,MAAMa,KAAKC,KAAK,CAAC,AAACf,gBAAgBJ,QAAS;4BACjD,MAAM,IAAIoB,oBAAU,CAClB,wBACA,AAAC,eAAiDxB,OAAnCA,SAASyB,EAAE,EAAC,yBAA8DhB,OAAvCT,SAASe,SAAS,CAACW,IAAI,CAAC,OAAM,WAAoDnB,OAA3CE,KAAI,0CAAoD,OAAbF,cAAa;wBAErJ;oBACF;oBACAoB,IAAAA,iBAAO,EAACtC,IAAIH,UAAUM,KAAKoC,SAAS,CAACzB;;;;;;IACvC"}

package/dist/cjs/embed/languages.d.cts

@@ -0,0 +1,3 @@
+export declare function isKnownLanguageTag(tag: string): boolean;
+export declare function toIso3(code: string): string;
+export declare const MEASURED_MODEL_LANGUAGES: Record<string, string[]>;

package/dist/cjs/embed/languages.d.ts

@@ -0,0 +1,3 @@
+export declare function isKnownLanguageTag(tag: string): boolean;
+export declare function toIso3(code: string): string;
+export declare const MEASURED_MODEL_LANGUAGES: Record<string, string[]>;

package/dist/cjs/embed/languages.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get MEASURED_MODEL_LANGUAGES () {
+        return MEASURED_MODEL_LANGUAGES;
+    },
+    get isKnownLanguageTag () {
+        return isKnownLanguageTag;
+    },
+    get toIso3 () {
+        return toIso3;
+    }
+});
+// HF model cards declare languages as ISO 639-1 tags (e.g. "zh"); franc-min reports ISO 639-3
+// (e.g. "cmn"). This bridges the two for comparison. Limited to franc-min's own 82-language
+// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min
+// can actually emit; a declared tag with no entry here is compared as-is.
+var ISO_639_1_TO_3 = {
+    ar: 'arb',
+    am: 'amh',
+    az: 'azj',
+    be: 'bel',
+    bg: 'bul',
+    bn: 'ben',
+    bs: 'bos',
+    cs: 'ces',
+    de: 'deu',
+    el: 'ell',
+    en: 'eng',
+    es: 'spa',
+    fa: 'pes',
+    fr: 'fra',
+    gu: 'guj',
+    ha: 'hau',
+    hi: 'hin',
+    hr: 'hrv',
+    hu: 'hun',
+    id: 'ind',
+    ig: 'ibo',
+    it: 'ita',
+    ja: 'jpn',
+    jv: 'jav',
+    kk: 'kaz',
+    kn: 'kan',
+    ko: 'kor',
+    ml: 'mal',
+    mr: 'mar',
+    ms: 'zlm',
+    my: 'mya',
+    ne: 'npi',
+    nl: 'nld',
+    pa: 'pan',
+    pl: 'pol',
+    pt: 'por',
+    ro: 'ron',
+    ru: 'rus',
+    rw: 'kin',
+    si: 'sin',
+    so: 'som',
+    sr: 'srp',
+    su: 'sun',
+    sv: 'swe',
+    sw: 'swh',
+    ta: 'tam',
+    te: 'tel',
+    th: 'tha',
+    tl: 'tgl',
+    tr: 'tur',
+    uk: 'ukr',
+    ur: 'urd',
+    uz: 'uzn',
+    vi: 'vie',
+    yo: 'yor',
+    zh: 'cmn',
+    zu: 'zul'
+};
+function isKnownLanguageTag(tag) {
+    return tag in ISO_639_1_TO_3;
+}
+function toIso3(code) {
+    var _ISO_639_1_TO_3_base;
+    var base = code.toLowerCase().split('-')[0];
+    return (_ISO_639_1_TO_3_base = ISO_639_1_TO_3[base]) !== null && _ISO_639_1_TO_3_base !== void 0 ? _ISO_639_1_TO_3_base : base;
+}
+var MEASURED_MODEL_LANGUAGES = {
+    'minishlab/potion-retrieval-32M': [
+        'en'
+    ]
+};
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }