sensemaking 0.15.4 → 0.16.0

package/dist/cjs/embed/languages.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/languages.ts"],"sourcesContent":["// HF model cards declare languages as ISO 639-1 tags (e.g. \"zh\"); franc-min reports ISO 639-3\n// (e.g. \"cmn\"). This bridges the two for comparison. Limited to franc-min's own 82-language\n// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min\n// can actually emit; a declared tag with no entry here is compared as-is.\nconst ISO_639_1_TO_3: Record<string, string> = {\n  ar: 'arb',\n  am: 'amh',\n  az: 'azj',\n  be: 'bel',\n  bg: 'bul',\n  bn: 'ben',\n  bs: 'bos',\n  cs: 'ces',\n  de: 'deu',\n  el: 'ell',\n  en: 'eng',\n  es: 'spa',\n  fa: 'pes',\n  fr: 'fra',\n  gu: 'guj',\n  ha: 'hau',\n  hi: 'hin',\n  hr: 'hrv',\n  hu: 'hun',\n  id: 'ind',\n  ig: 'ibo',\n  it: 'ita',\n  ja: 'jpn',\n  jv: 'jav',\n  kk: 'kaz',\n  kn: 'kan',\n  ko: 'kor',\n  ml: 'mal',\n  mr: 'mar',\n  ms: 'zlm',\n  my: 'mya',\n  ne: 'npi',\n  nl: 'nld',\n  pa: 'pan',\n  pl: 'pol',\n  pt: 'por',\n  ro: 'ron',\n  ru: 'rus',\n  rw: 'kin',\n  si: 'sin',\n  so: 'som',\n  sr: 'srp',\n  su: 'sun',\n  sv: 'swe',\n  sw: 'swh',\n  ta: 'tam',\n  te: 'tel',\n  th: 'tha',\n  tl: 'tgl',\n  tr: 'tur',\n  uk: 'ukr',\n  ur: 'urd',\n  uz: 'uzn',\n  vi: 'vie',\n  yo: 'yor',\n  zh: 'cmn',\n  zu: 'zul',\n};\n\nexport function isKnownLanguageTag(tag: string): boolean {\n  return tag in ISO_639_1_TO_3;\n}\n\n// Normalizes a declared or detected language code to ISO 639-3 for comparison; strips a\n// region/script subtag (\"zh-cn\" -> \"zh\") first. Codes already in 639-3 form pass through.\nexport function toIso3(code: string): string {\n  const base = code.toLowerCase().split('-')[0];\n  return ISO_639_1_TO_3[base] ?? base;\n}\n\n// Languages this project measured for models whose cards declare none, so the fit check is\n// not silently off for the shipped default; asserted here because it was measured, not attested.\nexport const MEASURED_MODEL_LANGUAGES: Record<string, string[]> = {\n  'minishlab/potion-retrieval-32M': ['en'], // BENCHMARKING.md bake-off; token-loss measurements\n};\n"],"names":["MEASURED_MODEL_LANGUAGES","isKnownLanguageTag","toIso3","ISO_639_1_TO_3","ar","am","az","be","bg","bn","bs","cs","de","el","en","es","fa","fr","gu","ha","hi","hr","hu","id","ig","it","ja","jv","kk","kn","ko","ml","mr","ms","my","ne","nl","pa","pl","pt","ro","ru","rw","si","so","sr","su","sv","sw","ta","te","th","tl","tr","uk","ur","uz","vi","yo","zh","zu","tag","code","base","toLowerCase","split"],"mappings":";;;;;;;;;;;QA6EaA;eAAAA;;QAbGC;eAAAA;;QAMAC;eAAAA;;;AAtEhB,8FAA8F;AAC9F,4FAA4F;AAC5F,gGAAgG;AAChG,0EAA0E;AAC1E,IAAMC,iBAAyC;IAC7CC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;AACN;AAEO,SAAS3D,mBAAmB4D,GAAW;IAC5C,OAAOA,OAAO1D;AAChB;AAIO,SAASD,OAAO4D,IAAY;QAE1B3D;IADP,IAAM4D,OAAOD,KAAKE,WAAW,GAAGC,KAAK,CAAC,IAAI,CAAC,EAAE;IAC7C,QAAO9D,uBAAAA,cAAc,CAAC4D,KAAK,cAApB5D,kCAAAA,uBAAwB4D;AACjC;AAIO,IAAM/D,2BAAqD;IAChE,kCAAkC;QAAC;KAAK;AAC1C"}

package/dist/cjs/embed/openai.d.cts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider>;

package/dist/cjs/embed/openai.d.ts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider>;

package/dist/cjs/embed/openai.js

@@ -0,0 +1,268 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "openaiProvider", {
+    enumerable: true,
+    get: function() {
+        return openaiProvider;
+    }
+});
+var _errorsts = require("../errors.js");
+var _httpts = require("./http.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var BATCH_CAP = 64;
+function openaiProvider(model, url, keyEnv) {
+    return _async_to_generator(function() {
+        var base, headers, key, dims;
+        function post(texts) {
+            return _async_to_generator(function() {
+                var res, body, data;
+                return _ts_generator(this, function(_state) {
+                    switch(_state.label){
+                        case 0:
+                            return [
+                                4,
+                                (0, _httpts.fetchWithRetry)("".concat(base, "/embeddings"), {
+                                    method: 'POST',
+                                    headers: headers,
+                                    body: JSON.stringify({
+                                        model: model,
+                                        input: texts
+                                    })
+                                })
+                            ];
+                        case 1:
+                            res = _state.sent();
+                            if (!res.ok) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(base, "/embeddings -> HTTP ").concat(res.status));
+                            return [
+                                4,
+                                res.json()
+                            ];
+                        case 2:
+                            body = _state.sent();
+                            // The schema carries `index` because response order is not promised, and the caller maps
+                            // vectors back to chunks positionally: sort by it whenever every element has one.
+                            data = body.data.every(function(d) {
+                                return typeof d.index === 'number';
+                            }) ? _to_consumable_array(body.data).sort(function(a, b) {
+                                return a.index - b.index;
+                            }) : body.data;
+                            // A server that drops an input (empty or over-length strings are the usual cause) would
+                            // otherwise shift every later vector onto the wrong chunk, silently.
+                            if (data.length !== texts.length) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(base, "/embeddings returned ").concat(data.length, " embeddings for ").concat(texts.length, " inputs"));
+                            return [
+                                2,
+                                data.map(function(d) {
+                                    return Float32Array.from(d.embedding);
+                                })
+                            ];
+                    }
+                });
+            })();
+        }
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (!url) throw new _errorsts.SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
+                    base = url.replace(/\/+$/, '');
+                    headers = {
+                        'content-type': 'application/json'
+                    };
+                    key = keyEnv ? process.env[keyEnv] : undefined;
+                    if (key) headers.authorization = "Bearer ".concat(key);
+                    return [
+                        4,
+                        post([
+                            'dimension probe'
+                        ])
+                    ];
+                case 1:
+                    dims = _state.sent()[0].length;
+                    // Symmetric for now: per-model prefixes (nomic's search_document/search_query) are a
+                    // later workstream.
+                    return [
+                        2,
+                        {
+                            id: "openai:".concat(base, ":").concat(model),
+                            dims: dims,
+                            batchCap: BATCH_CAP,
+                            embedDocuments: post,
+                            embedQuery: function embedQuery(text) {
+                                return _async_to_generator(function() {
+                                    return _ts_generator(this, function(_state) {
+                                        switch(_state.label){
+                                            case 0:
+                                                return [
+                                                    4,
+                                                    post([
+                                                        text
+                                                    ])
+                                                ];
+                                            case 1:
+                                                return [
+                                                    2,
+                                                    _state.sent()[0]
+                                                ];
+                                        }
+                                    });
+                                })();
+                            }
+                        }
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/openai.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // Symmetric for now: per-model prefixes (nomic's search_document/search_query) are a\n  // later workstream.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["openaiProvider","BATCH_CAP","model","url","keyEnv","base","headers","key","dims","post","texts","res","body","data","fetchWithRetry","method","JSON","stringify","input","ok","SenseError","status","json","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","replace","process","env","undefined","authorization","id","batchCap","embedDocuments","embedQuery","text"],"mappings":";;;;+BAQsBA;;;eAAAA;;;wBARK;sBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;;YAE/FC,MACAC,SACAC,KAgBAC;QAbN,SAAeC,KAAKC,KAAe;;oBAC3BC,KAEAC,MAGAC;;;;4BALM;;gCAAMC,IAAAA,sBAAc,EAAC,AAAC,GAAO,OAALT,MAAK,gBAAc;oCAAEU,QAAQ;oCAAQT,SAAAA;oCAASM,MAAMI,KAAKC,SAAS,CAAC;wCAAEf,OAAAA;wCAAOgB,OAAOR;oCAAM;gCAAG;;;4BAA1HC,MAAM;4BACZ,IAAI,CAACA,IAAIQ,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BT,OAA3BN,MAAK,wBAAiC,OAAXM,IAAIU,MAAM;4BAC3E;;gCAAMV,IAAIW,IAAI;;;4BAAtBV,OAAQ;4BACd,yFAAyF;4BACzF,kFAAkF;4BAC5EC,OAAOD,KAAKC,IAAI,CAACU,KAAK,CAAC,SAACC;uCAAM,OAAOA,EAAEC,KAAK,KAAK;iCAAY,AAAC,qBAAGb,KAAKC,IAAI,EAAEa,IAAI,CAAC,SAACC,GAAGC;uCAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK;iCAAeb,KAAKC,IAAI;4BACvJ,wFAAwF;4BACxF,qEAAqE;4BACrE,IAAIA,KAAKgB,MAAM,KAAKnB,MAAMmB,MAAM,EAAE,MAAM,IAAIT,oBAAU,CAAC,eAAe,AAAC,GAA8BP,OAA5BR,MAAK,yBAAqDK,OAA9BG,KAAKgB,MAAM,EAAC,oBAA+B,OAAbnB,MAAMmB,MAAM,EAAC;4BAChJ;;gCAAOhB,KAAKiB,GAAG,CAAC,SAACN;2CAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;;;;;YACtD;;;;;oBAjBA,IAAI,CAAC9B,KAAK,MAAM,IAAIiB,oBAAU,CAAC,eAAe;oBACxCf,OAAOF,IAAI+B,OAAO,CAAC,QAAQ;oBAC3B5B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAAS+B,QAAQC,GAAG,CAAChC,OAAO,GAAGiC;oBAC3C,IAAI9B,KAAKD,QAAQgC,aAAa,GAAG,AAAC,UAAa,OAAJ/B;oBAe7B;;wBAAME;4BAAM;;;;oBAApBD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACqB,MAAM;oBACxD,qFAAqF;oBACrF,oBAAoB;oBACpB;;wBAAO;4BAAEU,IAAI,AAAC,UAAiBrC,OAARG,MAAK,KAAS,OAANH;4BAASM,MAAAA;4BAAMgC,UAAUvC;4BAAWwC,gBAAgBhC;4BAAMiC,YAAY,SAAZA,WAAmBC;;;;;gDAAU;;oDAAMlC;wDAAMkC;;;;;;oDAAZ,aAAmB,CAAC,EAAE;;;;;;wBAAC;;;;IAC/I"}

package/dist/cjs/embed/query.d.cts

@@ -0,0 +1,21 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Config } from '../config/index.js';
+export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
+    v: Float32Array;
+    scale: number;
+};
+export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
+    path: string;
+    lines: string;
+    similarity: number;
+}>>;
+export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
+export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+    exclude: Set<string>;
+    allowed?: Set<string>;
+    k: number;
+}): Array<{
+    path: string;
+    similarity: number;
+}>;

package/dist/cjs/embed/query.d.ts

@@ -0,0 +1,21 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Config } from '../config/index.js';
+export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
+    v: Float32Array;
+    scale: number;
+};
+export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
+    path: string;
+    lines: string;
+    similarity: number;
+}>>;
+export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
+export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+    exclude: Set<string>;
+    allowed?: Set<string>;
+    k: number;
+}): Array<{
+    path: string;
+    similarity: number;
+}>;