sensemaking 0.17.1 → 0.18.0

package/dist/cjs/store/duckdb/vectors.js

@@ -0,0 +1,502 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get scanCandidates () {
+        return scanCandidates;
+    },
+    get scanSimilar () {
+        return scanSimilar;
+    },
+    get writeVectorBatch () {
+        return writeVectorBatch;
+    }
+});
+var _transactionts = require("../transaction.js");
+var _batchts = require("./batch.js");
+var _nativets = require("./native.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by
+// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with
+// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --
+// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own
+// representation and stays as it is. Every function here takes `dims` as a parameter rather
+// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real
+// constant at the call site).
+//
+// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only
+// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's
+// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever
+// open.ts already resolved, including via an on-demand install (see native.ts).
+// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
+// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
+var untyped = undefined;
+// Seed chunks that participate in a `related` scan; must match sqlite/vectors.ts's
+// TARGET_CHUNK_CAP for identical sampling behavior (see that file for the measurement it's
+// based on -- 12.7s at 201 chunks/note without sampling).
+var TARGET_CHUNK_CAP = 16;
+// array_cosine_similarity is already a true cosine (no int8 accumulation to push slightly
+// outside [-1, 1]), but rounded the same way sqlite's scores are so both stores print the
+// same bounded number rather than one having more decimal noise than the other.
+function asCosine(score) {
+    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
+}
+// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced
+// under the column's width); zero-padding leaves cosine scores unchanged since the added
+// dimensions contribute nothing to either vector's dot product or norm.
+function padded(values, dims) {
+    var out = new Array(dims).fill(0);
+    for(var d = 0; d < Math.min(values.length, dims); d++)out[d] = values[d];
+    return out;
+}
+function inClause(column, count) {
+    return "".concat(column, " IN (").concat(Array.from({
+        length: count
+    }, function() {
+        return '?';
+    }).join(', '), ")");
+}
+// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces
+// for every store, see toStore); dequantizing into a plain float array is this store's own
+// representation choice, so both stores are handed the same vectors and diverge only here.
+function dequantize(row, dims) {
+    var q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+    return padded(Array.from(q, function(v) {
+        return v * row.scale;
+    }), dims);
+}
+function writeVectorBatch(duckdb, conn, dims, rows) {
+    return _async_to_generator(function() {
+        var _ref, arrayValue, ARRAY, DOUBLE, rewritten;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (rows.length === 0) return [
+                        2
+                    ];
+                    return [
+                        4,
+                        (0, _nativets.duckdbApi)()
+                    ];
+                case 1:
+                    _ref = _state.sent(), arrayValue = _ref.arrayValue, ARRAY = _ref.ARRAY, DOUBLE = _ref.DOUBLE;
+                    rewritten = (0, _batchts.rewriteUpdate)('UPDATE embeddings SET vector = ? WHERE "path" = ? AND chunk = ?', rows.length);
+                    if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                var stmt, values, types, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row;
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                duckdb.prepare(rewritten.sql)
+                                            ];
+                                        case 1:
+                                            stmt = _state.sent();
+                                            _state.label = 2;
+                                        case 2:
+                                            _state.trys.push([
+                                                2,
+                                                ,
+                                                4,
+                                                5
+                                            ]);
+                                            values = [];
+                                            types = [];
+                                            _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                            try {
+                                                for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                                    row = _step.value;
+                                                    values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);
+                                                    types.push(ARRAY(DOUBLE, dims), untyped, untyped);
+                                                }
+                                            } catch (err) {
+                                                _didIteratorError = true;
+                                                _iteratorError = err;
+                                            } finally{
+                                                try {
+                                                    if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                                        _iterator.return();
+                                                    }
+                                                } finally{
+                                                    if (_didIteratorError) {
+                                                        throw _iteratorError;
+                                                    }
+                                                }
+                                            }
+                                            stmt.bind(values, types);
+                                            return [
+                                                4,
+                                                stmt.run()
+                                            ];
+                                        case 3:
+                                            _state.sent();
+                                            return [
+                                                3,
+                                                5
+                                            ];
+                                        case 4:
+                                            stmt.destroySync();
+                                            return [
+                                                7
+                                            ];
+                                        case 5:
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        })
+                    ];
+                case 2:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function scanCandidates(duckdb, qv, dims, fetch, allowed) {
+    return _async_to_generator(function() {
+        var _ref, arrayValue, ARRAY, DOUBLE, allowedList, sql, stmt, values, types, reader, rows;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (allowed && allowed.size === 0) return [
+                        2,
+                        []
+                    ];
+                    return [
+                        4,
+                        (0, _nativets.duckdbApi)()
+                    ];
+                case 1:
+                    _ref = _state.sent(), arrayValue = _ref.arrayValue, ARRAY = _ref.ARRAY, DOUBLE = _ref.DOUBLE;
+                    allowedList = allowed ? _to_consumable_array(allowed) : [];
+                    sql = 'SELECT "path", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT "path", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL '.concat(allowed ? "AND ".concat(inClause('"path"', allowedList.length)) : '', '\n    ) sub\n    GROUP BY "path"\n    ORDER BY score DESC\n    LIMIT ?');
+                    return [
+                        4,
+                        duckdb.prepare(sql)
+                    ];
+                case 2:
+                    stmt = _state.sent();
+                    _state.label = 3;
+                case 3:
+                    _state.trys.push([
+                        3,
+                        ,
+                        5,
+                        6
+                    ]);
+                    values = [
+                        arrayValue(padded(qv, dims))
+                    ].concat(_to_consumable_array(allowedList), [
+                        fetch
+                    ]);
+                    types = [
+                        ARRAY(DOUBLE, dims)
+                    ].concat(_to_consumable_array(allowedList.map(function() {
+                        return untyped;
+                    })), [
+                        untyped
+                    ]);
+                    stmt.bind(values, types);
+                    return [
+                        4,
+                        stmt.runAndReadAll()
+                    ];
+                case 4:
+                    reader = _state.sent();
+                    rows = reader.getRowObjectsJS();
+                    return [
+                        2,
+                        rows.map(function(r) {
+                            return {
+                                path: r.path,
+                                lines: "L".concat(r.start_line, "-").concat(r.end_line),
+                                similarity: asCosine(r.score)
+                            };
+                        })
+                    ];
+                case 5:
+                    stmt.destroySync();
+                    return [
+                        7
+                    ];
+                case 6:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function scanSimilar(duckdb, conn, dims, path, opts) {
+    return _async_to_generator(function() {
+        var targetStmt, targetRows, step, targets, _ref, arrayValue, ARRAY, DOUBLE, excludeList, allowedList, sql, stmt, values, types, reader, rows;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (opts.allowed && opts.allowed.size === 0) return [
+                        2,
+                        []
+                    ];
+                    return [
+                        4,
+                        conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk')
+                    ];
+                case 1:
+                    targetStmt = _state.sent();
+                    return [
+                        4,
+                        targetStmt.all(path)
+                    ];
+                case 2:
+                    targetRows = _state.sent();
+                    if (targetRows.length === 0) return [
+                        2,
+                        []
+                    ];
+                    step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
+                    targets = targetRows.filter(function(_, i) {
+                        return i % step === 0;
+                    }).map(function(row) {
+                        return row.vector;
+                    });
+                    return [
+                        4,
+                        (0, _nativets.duckdbApi)()
+                    ];
+                case 3:
+                    _ref = _state.sent(), arrayValue = _ref.arrayValue, ARRAY = _ref.ARRAY, DOUBLE = _ref.DOUBLE;
+                    excludeList = _to_consumable_array(opts.exclude);
+                    allowedList = opts.allowed ? _to_consumable_array(opts.allowed) : [];
+                    sql = "WITH targets(tv) AS (VALUES ".concat(targets.map(function() {
+                        return '(?)';
+                    }).join(', '), ')\n    SELECT e."path" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e."path" != ?\n      ').concat(excludeList.length > 0 ? "AND NOT ".concat(inClause('e."path"', excludeList.length)) : '', "\n      ").concat(opts.allowed ? "AND ".concat(inClause('e."path"', allowedList.length)) : '', '\n    GROUP BY e."path"\n    ORDER BY score DESC\n    LIMIT ?');
+                    return [
+                        4,
+                        duckdb.prepare(sql)
+                    ];
+                case 4:
+                    stmt = _state.sent();
+                    _state.label = 5;
+                case 5:
+                    _state.trys.push([
+                        5,
+                        ,
+                        7,
+                        8
+                    ]);
+                    values = _to_consumable_array(targets.map(function(t) {
+                        return arrayValue(padded(t, dims));
+                    })).concat([
+                        path
+                    ], _to_consumable_array(excludeList), _to_consumable_array(allowedList), [
+                        opts.k
+                    ]);
+                    types = _to_consumable_array(targets.map(function() {
+                        return ARRAY(DOUBLE, dims);
+                    })).concat([
+                        untyped
+                    ], _to_consumable_array(excludeList.map(function() {
+                        return untyped;
+                    })), _to_consumable_array(allowedList.map(function() {
+                        return untyped;
+                    })), [
+                        untyped
+                    ]);
+                    stmt.bind(values, types);
+                    return [
+                        4,
+                        stmt.runAndReadAll()
+                    ];
+                case 6:
+                    reader = _state.sent();
+                    rows = reader.getRowObjectsJS();
+                    return [
+                        2,
+                        rows.map(function(r) {
+                            return {
+                                path: r.path,
+                                similarity: asCosine(r.score)
+                            };
+                        })
+                    ];
+                case 7:
+                    stmt.destroySync();
+                    return [
+                        7
+                    ];
+                case 8:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/duckdb/vectors.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by\n// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with\n// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --\n// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own\n// representation and stays as it is. Every function here takes `dims` as a parameter rather\n// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real\n// constant at the call site).\n//\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only\n// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's\n// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever\n// open.ts already resolved, including via an on-demand install (see native.ts).\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// Seed chunks that participate in a `related` scan; must match sqlite/vectors.ts's\n// TARGET_CHUNK_CAP for identical sampling behavior (see that file for the measurement it's\n// based on -- 12.7s at 201 chunks/note without sampling).\nconst TARGET_CHUNK_CAP = 16;\n\n// array_cosine_similarity is already a true cosine (no int8 accumulation to push slightly\n// outside [-1, 1]), but rounded the same way sqlite's scores are so both stores print the\n// same bounded number rather than one having more decimal noise than the other.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced\n// under the column's width); zero-padding leaves cosine scores unchanged since the added\n// dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces\n// for every store, see toStore); dequantizing into a plain float array is this store's own\n// representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's\n// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native\n// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --\n// automatic type inference on the raw values misreads an all-integer first component (a real\n// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row\n// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks\n// the winning chunk's line range from the same row max(score) came from, per path. An empty\n// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed\n// into one native scan: the sampled target vectors become a small VALUES list, cross-joined\n// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost\n// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const targets = targetRows.filter((_, i) => i % step === 0).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","TARGET_CHUNK_CAP","asCosine","score","Math","round","min","max","padded","values","dims","out","Array","fill","d","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","duckdbApi","rewriteUpdate","Error","withTransaction","stmt","types","prepare","sql","push","path","chunk","bind","run","destroySync","qv","fetch","allowed","allowedList","reader","size","map","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","opts","targetStmt","targetRows","step","targets","excludeList","all","ceil","filter","_","i","exclude","t","k"],"mappings":";;;;;;;;;;;QA2FsBA;eAAAA;;QA8BAC;eAAAA;;QAxDAC;eAAAA;;;6BAhEU;uBAEF;wBACJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,mFAAmF;AACnF,uFAAuF;AACvF,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAC5F,8FAA8F;AAC9F,8BAA8B;AAC9B,EAAE;AACF,4FAA4F;AAC5F,8FAA8F;AAC9F,gGAAgG;AAChG,gFAAgF;AAEhF,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAEhB,mFAAmF;AACnF,2FAA2F;AAC3F,0DAA0D;AAC1D,IAAMC,mBAAmB;AAEzB,0FAA0F;AAC1F,0FAA0F;AAC1F,gFAAgF;AAChF,SAASC,SAASC,KAAa;IAC7B,OAAOC,KAAKC,KAAK,CAACD,KAAKE,GAAG,CAAC,GAAGF,KAAKG,GAAG,CAAC,CAAC,GAAGJ,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,wEAAwE;AACxE,SAASK,OAAOC,MAAyB,EAAEC,IAAY;IACrD,IAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIV,KAAKE,GAAG,CAACG,OAAOM,MAAM,EAAEL,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASK,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBN,OAAdK,QAAO,SAA2D,OAApDL,MAAMO,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,2FAA2F;AAC3F,SAASC,WAAWC,GAAmB,EAAEZ,IAAY;IACnD,IAAMa,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOpB,OACLI,MAAMO,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCpB;AAEJ;AAQO,SAAeZ,iBAAiBiC,MAAwB,EAAEC,IAAgB,EAAEtB,IAAY,EAAEuB,IAAsB;;YAE/E,MAA9BC,YAAYC,OAAOC,QACrBC;;;;oBAFN,IAAIJ,KAAKlB,MAAM,KAAK,GAAG;;;oBACe;;wBAAMuB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBC,YAAYE,IAAAA,sBAAa,EAAC,mEAAmEN,KAAKlB,MAAM;oBAC9G,IAAI,CAACsB,WAAW,MAAM,IAAIG,MAAM;oBAEhC;;wBAAMC,IAAAA,8BAAe,EAACT,MAAM;;oCACpBU,MAEEjC,QACAkC,OACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMrB;;;;4CAJA;;gDAAMS,OAAOa,OAAO,CAACP,UAAUQ,GAAG;;;4CAAzCH,OAAO;;;;;;;;;4CAELjC;4CACAkC;4CACD,kCAAA,2BAAA;;gDAAL,IAAK,YAAaV,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;oDAAbX,MAAN;oDACHb,OAAOqC,IAAI,CAACZ,WAAWb,WAAWC,KAAKZ,QAAQY,IAAIyB,IAAI,EAAEzB,IAAI0B,KAAK;oDAClEL,MAAMG,IAAI,CAACX,MAAMC,QAAQ1B,OAAOX,SAASA;gDAC3C;;gDAHK;gDAAA;;;yDAAA,6BAAA;wDAAA;;;wDAAA;8DAAA;;;;4CAIL2C,KAAKO,IAAI,CAACxC,QAAQkC;4CAClB;;gDAAMD,KAAKQ,GAAG;;;4CAAd;;;;;;4CAEAR,KAAKS,WAAW;;;;;;;;;;4BAEpB;;;;oBAdA;;;;;;IAeF;;AAKO,SAAevD,eAAemC,MAAwB,EAAEqB,EAAgB,EAAE1C,IAAY,EAAE2C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BpB,YAAYC,OAAOC,QACrBmB,aACAV,KASAH,MAEEjC,QACAkC,OAEAa,QACAvB;;;;oBAlBR,IAAIqB,WAAWA,QAAQG,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMnB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBmB,cAAcD,UAAW,qBAAGA;oBAC5BT,MAAM,AAAC,iRAIiF,OAA/DS,UAAU,AAAC,OAA6C,OAAvCtC,SAAS,UAAUuC,YAAYxC,MAAM,KAAM,IAAG;oBAKjF;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELjC,SAAwB;wBAACyB,WAAW1B,OAAO4C,IAAI1C;sBAAvB,OAA+B,qBAAG6C;wBAAaF;;oBACvEV,QAAsB;wBAACR,MAAMC,QAAQ1B;sBAAf,OAAsB,qBAAG6C,YAAYG,GAAG,CAAC;+BAAM3D;;wBAAUA;;oBACrF2C,KAAKO,IAAI,CAACxC,QAAQkC;oBACH;;wBAAMD,KAAKiB,aAAa;;;oBAAjCH,SAAS;oBACTvB,OAAOuB,OAAOI,eAAe;oBACnC;;wBAAO3B,KAAKyB,GAAG,CAAC,SAACG;mCAAO;gCAAEd,MAAMc,EAAEd,IAAI;gCAAEe,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAY/D,SAAS2D,EAAE1D,KAAK;4BAAE;;;;oBAE/GuC,KAAKS,WAAW;;;;;;;;;;IAEpB;;AAMO,SAAetD,YAAYkC,MAAwB,EAAEC,IAAgB,EAAEtB,IAAY,EAAEqC,IAAY,EAAEmB,IAAgE;;YAElKC,YACAC,YAEAC,MACAC,SAEgC,MAA9BpC,YAAYC,OAAOC,QACrBmC,aACAhB,aACAV,KASAH,MAEEjC,QACAkC,OAEAa,QACAvB;;;;oBAzBR,IAAIiC,KAAKZ,OAAO,IAAIY,KAAKZ,OAAO,CAACG,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAMzB,KAAKY,OAAO,CAAC;;;oBAAhCuB,aAAa;oBACC;;wBAAMA,WAAWK,GAAG,CAACzB;;;oBAAnCqB,aAAc;oBACpB,IAAIA,WAAWrD,MAAM,KAAK,GAAG;;;;oBACvBsD,OAAOjE,KAAKG,GAAG,CAAC,GAAGH,KAAKqE,IAAI,CAACL,WAAWrD,MAAM,GAAGd;oBACjDqE,UAAUF,WAAWM,MAAM,CAAC,SAACC,GAAGC;+BAAMA,IAAIP,SAAS;uBAAGX,GAAG,CAAC,SAACpC;+BAAQA,IAAIG,MAAM;;oBAE7C;;wBAAMa,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9BJ,aAA8B,KAA9BA,YAAYC,QAAkB,KAAlBA,OAAOC,SAAW,KAAXA;oBACrBmC,cAAe,qBAAGL,KAAKW,OAAO;oBAC9BtB,cAAcW,KAAKZ,OAAO,GAAI,qBAAGY,KAAKZ,OAAO;oBAC7CT,MAAM,AAAC,+BAIP0B,OAJqCD,QAAQZ,GAAG,CAAC;+BAAM;uBAAOtC,IAAI,CAAC,OAAM,qLAKzE8C,OADAK,YAAYxD,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAYuD,YAAYxD,MAAM,KAAM,IAAG,YACd,OAAtEmD,KAAKZ,OAAO,GAAG,AAAC,OAA+C,OAAzCtC,SAAS,YAAYuC,YAAYxC,MAAM,KAAM,IAAG;oBAI/D;;wBAAMgB,OAAOa,OAAO,CAACC;;;oBAA5BH,OAAO;;;;;;;;;oBAELjC,SAAwB,AAAC,qBAAG6D,QAAQZ,GAAG,CAAC,SAACoB;+BAAM5C,WAAW1B,OAAOsE,GAAGpE;;wBAASqC;uBAAM,qBAAGwB,cAAa,qBAAGhB;wBAAaW,KAAKa,CAAC;;oBACzHpC,QAAsB,AAAC,qBAAG2B,QAAQZ,GAAG,CAAC;+BAAMvB,MAAMC,QAAQ1B;;wBAAQX;uBAAS,qBAAGwE,YAAYb,GAAG,CAAC;+BAAM3D;yBAAU,qBAAGwD,YAAYG,GAAG,CAAC;+BAAM3D;;wBAAUA;;oBACvJ2C,KAAKO,IAAI,CAACxC,QAAQkC;oBACH;;wBAAMD,KAAKiB,aAAa;;;oBAAjCH,SAAS;oBACTvB,OAAOuB,OAAOI,eAAe;oBACnC;;wBAAO3B,KAAKyB,GAAG,CAAC,SAACG;mCAAO;gCAAEd,MAAMc,EAAEd,IAAI;gCAAEkB,YAAY/D,SAAS2D,EAAE1D,KAAK;4BAAE;;;;oBAEtEuC,KAAKS,WAAW;;;;;;;;;;IAEpB"}

package/dist/cjs/store/index.d.cts

@@ -0,0 +1,7 @@
+import type { ResolvedConfig } from '../config/index.js';
+import type { OpenResult } from './sqlite/open.js';
+export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
+export { getMeta, setMeta } from './meta.js';
+export type { OpenResult } from './sqlite/open.js';
+export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';

package/dist/cjs/store/index.d.ts

@@ -0,0 +1,7 @@
+import type { ResolvedConfig } from '../config/index.js';
+import type { OpenResult } from './sqlite/open.js';
+export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
+export { getMeta, setMeta } from './meta.js';
+export type { OpenResult } from './sqlite/open.js';
+export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';