sensemaking 0.12.2 → 0.13.0

package/dist/esm/{db.js → db/reconcile.js}

@@ -1,131 +1,21 @@
-// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with
-// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.
-import { mkdirSync, rmSync } from 'node:fs';
-import { join } from 'node:path';
-import { DatabaseSync } from 'node:sqlite';
-import { contentTokenize, featureSignature, STATE_DIR } from './config.js';
-import { SenseError } from './errors.js';
-import { activeFeatures } from './features/index.js';
-import { progress } from './progress.js';
-import { listFiles, parseFile, RESERVED_COLUMNS } from './scan.js';
-import { segmentField, segmentMatch } from './segment.js';
+import { contentTokenize } from '../config/index.js';
+import { SenseError } from '../errors.js';
+import { activeFeatures } from '../features/index.js';
+import { progress } from '../progress.js';
+import { listFiles, parseFile, RESERVED_COLUMNS } from '../scan.js';
+import { segmentField } from '../segment.js';
+import { getColumns, getMeta, quoteIdent, setMeta } from './shared.js';
 // Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last
 // computed value on every touch, not just the reconciles that recompute it.
 const CORE_FRONTMATTER_COLUMNS = new Set([
     'path',
     '_mtime',
+    '_ctime',
     '_size',
     '_parse_error'
 ]);
-// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;
-// everything beyond frontmatter + content lives in src/features/.
-export const DB_FILENAME = 'cache.db';
-// Cache shape version, independent of the config's own `version`. Bumping it rebuilds
-// existing trees on first query.
-export const SCHEMA_VERSION = '12';
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 const MAX_FRONTMATTER_COLUMNS = 2000;
-function quoteIdent(name) {
-    return `"${name.split('"').join('""')}"`;
-}
-// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.
-function registerFunctions(db, segmenting) {
-    db.function('has', {
-        deterministic: true,
-        varargs: false
-    }, (field, value)=>{
-        if (field === null || field === undefined) return 0;
-        const needle = String(value);
-        if (typeof field === 'string') {
-            if (field.startsWith('[')) {
-                try {
-                    const parsed = JSON.parse(field);
-                    if (Array.isArray(parsed)) {
-                        return parsed.some((item)=>String(item) === needle) ? 1 : 0;
-                    }
-                } catch  {}
-            }
-            return field.includes(needle) ? 1 : 0;
-        }
-        return String(field).includes(needle) ? 1 : 0;
-    });
-    // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back, and
-    // matches nothing, so the same transform is available to hand-written SQL by name. `segmenting`
-    // is the same config predicate commands.ts's search() gates on (contentTokenize(cfg) ===
-    // undefined), passed in by open() rather than read back from persisted state.
-    db.function('segment', {
-        deterministic: true,
-        varargs: false
-    }, (terms)=>{
-        const text = terms === null || terms === undefined ? '' : String(terms);
-        return segmenting ? segmentMatch(text) : text;
-    });
-}
-function getColumns(db) {
-    const rows = db.prepare('PRAGMA table_info(frontmatter)').all();
-    return new Set(rows.map((r)=>r.name));
-}
-// Stemming is English-only, but the segmentation underneath it is what decides coverage:
-// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)
-// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how
-// such a tree picks trigram instead.
-const DEFAULT_TOKENIZE = 'porter unicode61';
-// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a
-// configured value has to be concatenated. Probing a throwaway table is what makes that safe
-// and is also the whole validation: anything the linked SQLite accepts passes, anything else
-// fails here with SQLite's own message rather than against the real table. It means no table
-// of which version added which tokenizer has to be maintained.
-function resolveTokenize(db, cfg) {
-    const configured = contentTokenize(cfg);
-    if (configured === undefined) return DEFAULT_TOKENIZE;
-    const literal = configured.replace(/'/g, "''");
-    try {
-        db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');
-        db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);
-        db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');
-    } catch (err) {
-        throw new SenseError('CONFIG_INVALID', `content.tokenize "${configured}" is not a tokenizer this SQLite accepts (${err.message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);
-    }
-    return literal;
-}
-// The tokenizer the content table was actually built with, from its own DDL -- the one
-// record that cannot desynchronize from the table. NULL when the table does not exist yet.
-function storedTokenize(db) {
-    const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get();
-    if (!row) return null;
-    const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);
-    return m ? m[1] : null;
-}
-// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
-// can't dump file text into context. Features add their own tables after the core ones.
-function ensureSchema(db, cfg, tokenize) {
-    db.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
-    // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL
-    // against the resolved tokenizer before this runs, so a stale table is already gone by now.
-    // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
-    // snippet(content, 2, ...) are documented against the first three columns and keep working
-    // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
-    // runs for text that needs it, and an empty string for text that does not.
-    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);
-    // Coverage, not ownership: a path can appear under several presets. path leads the PK so the
-    // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.
-    db.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
-    db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
-    for (const feature of activeFeatures(cfg))feature.schema(db);
-    if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);
-    if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));
-}
-export function getMeta(db, key) {
-    const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key);
-    return row ? row.value : null;
-}
-export function setMeta(db, key, value) {
-    if (value === null) {
-        db.prepare('DELETE FROM meta WHERE key = ?').run(key);
-        return;
-    }
-    db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);
-}
 // Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same
 // literal instead of two copies drifting apart.
 const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, "path", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?, ?, ?, ?)`;
@@ -134,10 +24,6 @@ const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, "p
 function insertContentRow(insertBody, doc, segmenting) {
     insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');
 }
-export function docCount(db) {
-    const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get();
-    return row.n;
-}
 export function reconcile(db, cfg, baseDir) {
     const files = listFiles(cfg, baseDir);
     const currentSet = new Set(files.map((f)=>f.relPath));
@@ -236,6 +122,7 @@ export function reconcile(db, cfg, baseDir) {
                     var _doc_data_col;
                     if (col === 'path') return doc.relPath;
                     if (col === '_mtime') return doc.mtimeMs;
+                    if (col === '_ctime') return doc.ctimeMs;
                     if (col === '_size') return doc.size;
                     // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
                     if (col === '_parse_error') return doc.parseError;
@@ -278,7 +165,7 @@ export function reconcile(db, cfg, baseDir) {
 }
 // Segment keys that moved between two feature signatures (see config.featureSignature's
 // format: global features, embed provider, tokenize, then one segment per preset).
-function changedSignatureKeys(before, after) {
+export function changedSignatureKeys(before, after) {
     const keyOf = (part)=>part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0];
     const parse = (sig)=>new Map(sig.split('|').map((part)=>[
                 keyOf(part),
@@ -292,7 +179,7 @@ function changedSignatureKeys(before, after) {
     return changed;
 }
 // Names what moved, for the rebuild notice.
-function signatureDiff(before, after) {
+export function signatureDiff(before, after) {
     const changed = changedSignatureKeys(before, after);
     const label = (key)=>key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset "${key.slice(7)}"` : 'features';
     return changed.size === 0 ? 'features' : [
@@ -305,7 +192,7 @@ function signatureDiff(before, after) {
 // run here -- doc.search (title/summary/text) is all content population needs. Returns
 // parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are
 // untouched, so reconcile() never reparses these files and would otherwise never emit them again.
-function rebuildContentTable(db, cfg, baseDir) {
+export function rebuildContentTable(db, cfg, baseDir) {
     const known = new Set(db.prepare('SELECT "path" FROM frontmatter').all().map((r)=>r.path));
     const files = listFiles(cfg, baseDir).filter((f)=>known.has(f.relPath));
     const segmenting = contentTokenize(cfg) === undefined;
@@ -325,94 +212,3 @@ function rebuildContentTable(db, cfg, baseDir) {
     }
     return warnings;
 }
-export function open(cfg) {
-    var _getMeta;
-    const stateDir = join(cfg.baseDir, STATE_DIR);
-    mkdirSync(stateDir, {
-        recursive: true
-    });
-    const dbPath = join(stateDir, DB_FILENAME);
-    const db = new DatabaseSync(dbPath);
-    db.exec('PRAGMA journal_mode = WAL');
-    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
-    // that outwaits it still fails loudly.
-    db.exec('PRAGMA busy_timeout = 30000');
-    registerFunctions(db, contentTokenize(cfg) === undefined);
-    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
-    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd
-    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.
-    const tokenize = resolveTokenize(db, cfg);
-    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an
-    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).
-    const version = getMeta(db, 'schema_version');
-    const features = getMeta(db, 'features');
-    const wantFeatures = featureSignature(cfg);
-    let tokenizeOnlyRebuild = false;
-    if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
-        // Indexing derives from presets, so a config edit rebuilding the cache must say so and
-        // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
-        if (version !== null && version !== SCHEMA_VERSION) {
-            console.error('sense: cache format changed (new sensemaking version); rebuilding the index');
-            db.close();
-            clearCache(cfg);
-            return open(cfg);
-        }
-        const changedKeys = changedSignatureKeys(features !== null && features !== void 0 ? features : '', wantFeatures);
-        // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived
-        // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset
-        // edit, an embed model change -- still takes the full clear/reopen below.
-        if (changedKeys.size === 1 && changedKeys.has('tokenize')) {
-            console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
-            db.exec('DROP TABLE content');
-            tokenizeOnlyRebuild = true;
-        } else {
-            const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
-            console.error(`sense: config change (${changed}) rebuilds the index`);
-            db.close();
-            clearCache(cfg);
-            return open(cfg);
-        }
-    }
-    // Meta can lie after a crash between table creation and the signature write; the table's
-    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild
-    // just dropped content above, so storedTokenize sees no table and this guard is skipped
-    // naturally rather than needing its own case.
-    const stored = storedTokenize(db);
-    if (stored !== null && stored !== tokenize) {
-        console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
-        db.close();
-        clearCache(cfg);
-        return open(cfg);
-    }
-    ensureSchema(db, cfg, tokenize);
-    let rebuildWarnings = [];
-    if (tokenizeOnlyRebuild) {
-        rebuildWarnings = rebuildContentTable(db, cfg, cfg.baseDir);
-        setMeta(db, 'features', wantFeatures);
-    }
-    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
-    // Installed before reconcile() -- that call is the one that races a watcher's transaction.
-    const recordedMaxMs = Number((_getMeta = getMeta(db, 'reconcile_max_ms')) !== null && _getMeta !== void 0 ? _getMeta : '0');
-    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
-    const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);
-    return {
-        db,
-        cfg,
-        dbPath,
-        parsed,
-        warnings: [
-            ...rebuildWarnings,
-            ...warnings
-        ]
-    };
-}
-// Deletes the cache directory, and only that. The rebuild is not this function's job: the next
-// open() reconciles, which is what running any command already does, so a verb that bundled the
-// two described the half that was not its own. Manual reset for a doubted cache; the schema and
-// config-signature mismatches below reset themselves.
-export function clearCache(cfg) {
-    rmSync(join(cfg.baseDir, STATE_DIR), {
-        recursive: true,
-        force: true
-    });
-}

package/dist/esm/db/reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/reconcile.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport type { Config } from '../config/index.ts';\nimport { contentTokenize } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../progress.ts';\nimport type { ParsedDoc } from '../scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../scan.ts';\nimport { segmentField } from '../segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from './shared.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so content population is\n// defined once. Assumes the frontmatter row for doc.relPath already exists (rowid lookup).\nfunction insertContentRow(insertBody: StatementSync, doc: ParsedDoc, segmenting: boolean): void {\n  insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(INSERT_CONTENT_SQL);\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertContentRow(insertBody, doc, segmenting);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport function rebuildContentTable(db: DatabaseSync, cfg: Config, baseDir: string): string[] {\n  const known = new Set((db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const insertBody = db.prepare(INSERT_CONTENT_SQL);\n  const warnings: string[] = [];\n  db.exec('BEGIN');\n  try {\n    for (const file of files) {\n      const { doc, warnings: fileWarnings } = parseFile(file);\n      warnings.push(...fileWarnings);\n      insertContentRow(insertBody, doc, segmenting);\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","segmentField","getColumns","getMeta","quoteIdent","setMeta","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","insertContentRow","insertBody","doc","segmenting","run","relPath","search","title","summary","text","reconcile","db","cfg","baseDir","files","currentSet","map","f","existingRows","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","feature","enabledForFile","fileWarnings","tick","push","key","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","txStart","Date","now","exec","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","undefined","values","ctimeMs","parseError","presetName","presets","store","extracted","name","afterReconcile","err","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","val","signatureDiff","label","rebuildContentTable","known"],"mappings":"AAEA,SAASA,eAAe,QAAQ,qBAAqB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,iBAAiB;AAE1C,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,aAAa;AACpE,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,cAAc;AAEvE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,0FAA0F;AAC1F,2FAA2F;AAC3F,SAASC,iBAAiBC,UAAyB,EAAEC,GAAc,EAAEC,UAAmB;IACtFF,WAAWG,GAAG,CAACF,IAAIG,OAAO,EAAEH,IAAII,MAAM,CAACC,KAAK,EAAEL,IAAII,MAAM,CAACE,OAAO,EAAEN,IAAII,MAAM,CAACG,IAAI,EAAEP,IAAIG,OAAO,EAAEF,aAAaZ,aAAaW,IAAII,MAAM,CAACC,KAAK,IAAI,IAAIJ,aAAaZ,aAAaW,IAAII,MAAM,CAACE,OAAO,IAAI,IAAIL,aAAaZ,aAAaW,IAAII,MAAM,CAACG,IAAI,IAAI;AACrP;AAEA,OAAO,SAASC,UAAUC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtE,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIlB,IAAIiB,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAeP,GAAGQ,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMC,WAAW,IAAIC,IAAIJ,aAAaF,GAAG,CAAC,CAACO,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWP,aAAaQ,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGR,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYd,MAAMY,MAAM,CAAC,CAACT;QAC9B,MAAMY,MAAMR,SAASS,GAAG,CAACb,EAAEZ,OAAO;QAClC,OAAO,CAACwB,OAAOA,IAAIE,MAAM,KAAKd,EAAEe,OAAO,IAAIH,IAAII,KAAK,KAAKhB,EAAEiB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWpD,eAAe0B;IAChC,MAAM2B,cAAc/C,WAAWmB;IAC/B,MAAM6B,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAASvD,SAAS,mBAAmByC,UAAUO,MAAM;IAC3D,IAAIQ,cAAc;IAClB,KAAK,MAAMC,QAAQhB,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMiB,eAAeP,SAASZ,MAAM,CAAC,CAACoB,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACnC,KAAKgC;QACzG,MAAM,EAAE1C,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMG,OAAOC,OAAOC,IAAI,CAACnD,IAAIoD,IAAI,EAAG;YACvC,IAAI,CAACf,YAAYZ,GAAG,CAACwB,MAAM;gBACzBZ,YAAYgB,GAAG,CAACJ;gBAChBX,WAAWU,IAAI,CAACC;YAClB;QACF;QACAV,WAAWS,IAAI,CAAChD;IAClB;IACAwC,OAAOc,MAAM;IAEb,MAAMC,aAAa;WAAIlB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIkB,WAAWtB,MAAM,GAAGrC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAEwE,WAAWtB,MAAM,CAAC,0EAA0E,EAAErC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAM4D,kBAAkBD,WAAW/B,MAAM,CAAC,CAACiC,IAAM/D,yBAAyB+B,GAAG,CAACgC,MAAM,CAACrE,iBAAiBqC,GAAG,CAACgC;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB1C,GAAG,CAACtB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB1C,GAAG,CAAC,IAAM,KAAK6C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLhC,MAAM,CAAC,CAACiC,IAAMA,MAAM,QACpB3C,GAAG,CAAC,CAAC2C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQlC,UAAUF,MAAM,CAAC,CAACT,IAAM,CAACI,SAASM,GAAG,CAACV,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAM0D,QAAwB;QAAEjD;QAAOkD,UAAUvB,WAAWzB,GAAG,CAAC,CAACiD,IAAMA,EAAE5D,OAAO;QAAGyD;QAAOrC;IAAS;IAEnG,MAAMyC,UAAUC,KAAKC,GAAG;IACxBzD,GAAG0D,IAAI,CAAC;IACR,IAAI;YAkD8BvB;QAjDhC,KAAK,MAAMwB,OAAO9B,WAAY7B,GAAG0D,IAAI,CAAC,CAAC,mCAAmC,EAAE3E,WAAW4E,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAU5D,GAAGQ,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMqD,iBAAiB7D,GAAGQ,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMsD,mBAAmB9D,GAAGQ,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAIM,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMuC,MAAM/D,GAAGQ,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAMK,QAAQC,SAAU;oBAMKqB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrEyB,QAAQnE,GAAG,CAACoB;gBACZkD,IAAItE,GAAG,CAACoB;gBACRgD,eAAepE,GAAG,CAACoB;gBACnB,KAAK,MAAMsB,WAAWR,UAAUQ,kBAAAA,QAAQ6B,MAAM,cAAd7B,sCAAAA,qBAAAA,SAAiBnC,IAAIa,MAAMuC;YAC7D;QACF;QACA,IAAItB,WAAWN,MAAM,GAAG,GAAG;YACzB,MAAMyC,SAASjE,GAAGQ,OAAO,CAACyC;YAC1B,MAAM3D,aAAaU,GAAGQ,OAAO,CAACpB;YAC9B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMI,aAAanB,gBAAgB4B,SAASiE;YAC5C,KAAK,MAAM3E,OAAOuC,WAAY;oBAsBIK;gBArBhC,MAAMgC,SAASpB,gBAAgB1C,GAAG,CAAC,CAACsD;wBAO3BpE;oBANP,IAAIoE,QAAQ,QAAQ,OAAOpE,IAAIG,OAAO;oBACtC,IAAIiE,QAAQ,UAAU,OAAOpE,IAAI8B,OAAO;oBACxC,IAAIsC,QAAQ,UAAU,OAAOpE,IAAI6E,OAAO;oBACxC,IAAIT,QAAQ,SAAS,OAAOpE,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIoC,QAAQ,gBAAgB,OAAOpE,IAAI8E,UAAU;oBACjD,QAAO9E,gBAAAA,IAAIoD,IAAI,CAACgB,IAAI,cAAbpE,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0E,OAAOxE,GAAG,IAAI0E;gBACd,oFAAoF;gBACpF,IAAIzD,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAG;wBAEGyC;oBADhCyB,QAAQnE,GAAG,CAACF,IAAIG,OAAO;oBACvB,KAAK,MAAMyC,WAAWR,UAAUQ,mBAAAA,QAAQ6B,MAAM,cAAd7B,uCAAAA,sBAAAA,SAAiBnC,IAAIT,IAAIG,OAAO,EAAE0D;gBACpE;gBACA/D,iBAAiBC,YAAYC,KAAKC;gBAClC,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIkB,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAGmE,eAAepE,GAAG,CAACF,IAAIG,OAAO;gBAC7D,KAAK,MAAM4E,cAAc/E,IAAIgF,OAAO,CAAET,iBAAiBrE,GAAG,CAACF,IAAIG,OAAO,EAAE4E;gBACxE,KAAK,MAAMnC,WAAWR,UAAUQ,iBAAAA,QAAQqC,KAAK,cAAbrC,qCAAAA,oBAAAA,SAAgBnC,IAAIT,IAAIG,OAAO,EAAEH,IAAIkF,SAAS,CAACtC,QAAQuC,IAAI,CAAC,EAAEtB;YAChG;QACF;QACA,KAAK,MAAMjB,WAAWR,UAAUQ,0BAAAA,QAAQwC,cAAc,cAAtBxC,8CAAAA,6BAAAA,SAAyBnC,IAAIoD;QAC7DpD,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAarB,KAAKC,GAAG,KAAKF;IAChC,MAAMuB,UAAUhG,QAAQkB,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM+E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS/F,QAAQgB,IAAI,oBAAoBiF,OAAOJ;IAEjE,OAAO;QAAEpD,QAAQK,WAAWN,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASwD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGvC,IAAI,CAAC,OAAOoC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAIhF,IAAIgF,IAAIH,KAAK,CAAC,KAAKnF,GAAG,CAAC,CAACiF,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAI5G;IACpB,KAAK,MAAM,CAACsD,KAAKuD,IAAI,IAAIF,EAAG,IAAID,EAAEzE,GAAG,CAACqB,SAASuD,KAAKD,QAAQlD,GAAG,CAACJ;IAChE,KAAK,MAAMA,OAAOoD,EAAElD,IAAI,GAAI,IAAI,CAACmD,EAAE7E,GAAG,CAACwB,MAAMsD,QAAQlD,GAAG,CAACJ;IACzD,OAAOsD;AACT;AAEA,4CAA4C;AAC5C,OAAO,SAASE,cAAcb,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMa,QAAQ,CAACzD,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAI+C,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAE/C,IAAIiD,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQvE,IAAI,KAAK,IAAI,aAAa;WAAIuE;KAAQ,CAACzF,GAAG,CAAC4F,OAAO/C,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,SAASgD,oBAAoBlG,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAChF,MAAMiG,QAAQ,IAAIjH,IAAI,AAACc,GAAGQ,OAAO,CAAC,kCAAkCC,GAAG,GAA+BJ,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IACvH,MAAMV,QAAQ1B,UAAUwB,KAAKC,SAASa,MAAM,CAAC,CAACT,IAAM6F,MAAMnF,GAAG,CAACV,EAAEZ,OAAO;IACvE,MAAMF,aAAanB,gBAAgB4B,SAASiE;IAC5C,MAAM5E,aAAaU,GAAGQ,OAAO,CAACpB;IAC9B,MAAMsC,WAAqB,EAAE;IAC7B1B,GAAG0D,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAMzB,QAAQ9B,MAAO;YACxB,MAAM,EAAEZ,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD;YAClDP,SAASa,IAAI,IAAIF;YACjBhD,iBAAiBC,YAAYC,KAAKC;QACpC;QACAQ,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IACA,OAAOlD;AACT"}

package/dist/esm/db/shared.d.ts

@@ -0,0 +1,5 @@
+import type { DatabaseSync } from 'node:sqlite';
+export declare function quoteIdent(name: string): string;
+export declare function getColumns(db: DatabaseSync): Set<string>;
+export declare function getMeta(db: DatabaseSync, key: string): string | null;
+export declare function setMeta(db: DatabaseSync, key: string, value: string | null): void;

package/dist/esm/db/shared.js

@@ -0,0 +1,20 @@
+// SQL and meta-table primitives both open() and reconcile() need, kept below both so
+// neither imports the other's module.
+export function quoteIdent(name) {
+    return `"${name.split('"').join('""')}"`;
+}
+export function getColumns(db) {
+    const rows = db.prepare('PRAGMA table_info(frontmatter)').all();
+    return new Set(rows.map((r)=>r.name));
+}
+export function getMeta(db, key) {
+    const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key);
+    return row ? row.value : null;
+}
+export function setMeta(db, key, value) {
+    if (value === null) {
+        db.prepare('DELETE FROM meta WHERE key = ?').run(key);
+        return;
+    }
+    db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);
+}

package/dist/esm/db/shared.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/shared.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\n\n// SQL and meta-table primitives both open() and reconcile() need, kept below both so\n// neither imports the other's module.\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport function getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n"],"names":["quoteIdent","name","split","join","getColumns","db","rows","prepare","all","Set","map","r","getMeta","key","row","get","value","setMeta","run"],"mappings":"AAEA,qFAAqF;AACrF,sCAAsC;AAEtC,OAAO,SAASA,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,SAASC,WAAWC,EAAgB;IACzC,MAAMC,OAAOD,GAAGE,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAIC,IAAIH,KAAKI,GAAG,CAAC,CAACC,IAAMA,EAAEV,IAAI;AACvC;AAEA,OAAO,SAASW,QAAQP,EAAgB,EAAEQ,GAAW;IACnD,MAAMC,MAAMT,GAAGE,OAAO,CAAC,wCAAwCQ,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASC,QAAQZ,EAAgB,EAAEQ,GAAW,EAAEG,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBX,GAAGE,OAAO,CAAC,kCAAkCW,GAAG,CAACL;QACjD;IACF;IACAR,GAAGE,OAAO,CAAC,qGAAqGW,GAAG,CAACL,KAAKG;AAC3H"}

package/dist/esm/features/embed.d.ts

@@ -1,5 +1,5 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { Config } from '../config.js';
+import type { Config } from '../config/index.js';
 import type { Feature } from './types.js';
 export interface EmbedProvider {
     id: string;

package/dist/esm/features/embed.js

@@ -1,7 +1,7 @@
 import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
-import { embedConfig } from '../config.js';
+import { embedConfig } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { progress } from '../progress.js';
 import { parseFile } from '../scan.js';
@@ -285,6 +285,7 @@ export async function embedPending(db, cfg, baseDir) {
                 relPath: path,
                 absPath: join(baseDir, path),
                 mtimeMs: 0,
+                ctimeMs: 0,
                 size: 0,
                 presets: [],
                 embed: true

package/dist/esm/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// Heading-based chunks, int8 vectors with a per-vector scale, NULL vector = not yet embedded.\n// Reconcile writes dirty rows; embedding tops up on the next search, so staleness costs recall.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,\n// preamble kept, whole body when no headings; the title/summary prefix mirrors bm25 weighting.\n//\n// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw\n// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw\n// put the frontmatter block in its own leading chunk on every note with a heading, which made\n// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also\n// disagreed with FTS, which indexes the body alone.\nfunction chunksOf(body: string, search?: { title: string; summary: string }, offset = 0): Chunk[] {\n  const lines = body.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    // A body with nothing in it yields no chunks at all, so a frontmatter-only note has no\n    // vectors rather than a vector of its own YAML.\n    if (text.length > 0) chunks.push({ startLine: start + offset, endLine: end + offset, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    return chunksOf(body, search, raw.split('\\n').length - body.split('\\n').length);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  // A tree with no embedding model never had extract() run for the doc (db.ts's per-file\n  // filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.\n  store(db, path, extracted) {\n    if (!extracted) return;\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nconst MODEL_FILES = ['model.safetensors', 'tokenizer.json'];\n// The pair, for messages that name what a local model directory must contain.\nexport const MODEL_FILENAMES = MODEL_FILES.join(' and ');\n\n// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather\n// than flattened into a cache key Windows would reject.\nconst HF_MODEL_ID = /^[A-Za-z0-9][\\w.-]*\\/[A-Za-z0-9][\\w.-]*$/;\n\n// Downloadable iff it names a Hugging Face repo; a local path is the caller's to populate.\nexport function isDownloadable(model: string): boolean {\n  return HF_MODEL_ID.test(model);\n}\n\n// Machine-wide, not per-tree: `.sense/` is the index a rebuild throws away, while a 124 MB\n// model is shared by every tree on the machine. Honors XDG_CACHE_HOME where it is set;\n// elsewhere ~/.cache, which is also where Hugging Face's own libraries keep theirs.\nfunction cacheRoot(): string {\n  const xdg = process.env.XDG_CACHE_HOME;\n  return join(xdg && xdg.length > 0 ? xdg : join(homedir(), '.cache'), 'sensemaking', 'models');\n}\n\n// A model is either a local directory the caller pointed at, or the cache `sense download`\n// fills. Nothing here fetches: an absent model degrades search to its other signals rather\n// than pulling 124 MB out of a command that reads like a query.\nexport function modelDir(model: string): string {\n  if (!HF_MODEL_ID.test(model)) return model;\n  return join(cacheRoot(), model.replace(/\\//g, '--'));\n}\n\n// Both files, so an interrupted download reads as absent and the next one resumes it.\nexport function hasModelFiles(model: string): boolean {\n  const dir = modelDir(model);\n  return MODEL_FILES.every((file) => existsSync(join(dir, file)));\n}\n\n// api providers have nothing to download, so they are never \"missing\" here; an unreachable\n// endpoint surfaces as an EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.type === 'api' || hasModelFiles(e.model);\n}\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\n// The only code path that touches the network for weights. `sense download` calls it; no\n// query ever does. Idempotent: a file already on disk is left alone.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const dir = modelDir(model);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  const dir = modelDir(model);\n  for (const file of MODEL_FILES) {\n    if (!existsSync(join(dir, file))) {\n      // A local path the caller controls gets told what is missing where; only a repo id can\n      // be fixed by downloading.\n      const fix = isDownloadable(model) ? 'run `sense download`' : `expected ${MODEL_FILENAMES} in that directory`;\n      throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); ${fix}`);\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json, then run `sense download`');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + BATCH, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["existsSync","mkdirSync","readFileSync","renameSync","writeFileSync","homedir","join","embedConfig","SenseError","progress","parseFile","STORE_DIMS","TARGET_CHUNK_CAP","BATCH","chunksOf","body","search","offset","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","chunks","forEach","start","end","text","slice","trim","startLine","endLine","embed","name","schema","db","exec","extract","raw","remove","path","prepare","run","store","extracted","insert","c","idx","enabledForFile","_cfg","file","MODEL_FILES","MODEL_FILENAMES","HF_MODEL_ID","isDownloadable","model","cacheRoot","xdg","process","env","XDG_CACHE_HOME","modelDir","replace","hasModelFiles","dir","every","modelPresent","cfg","e","type","fetchToFile","url","dest","fetch","then","res","ok","status","Buffer","from","arrayBuffer","downloadModel","onFile","recursive","staticProvider","tokenizerJson","fix","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","Tokenizer","tok","unkId","vocab","unk_token","one","ids","encode","add_special_tokens","id","isInteger","v","off","d","norm","Math","sqrt","texts","map","apiProvider","keyEnv","base","headers","key","undefined","authorization","post","method","stringify","input","json","data","embedding","providers","Map","getProvider","sig","p","get","set","toStore","full","int8","scale","max","abs","embedPending","baseDir","provider","dirty","all","storeDims","min","byPath","row","list","chunk","jobs","chunkIdxs","relPath","absPath","mtimeMs","size","presets","doc","update","report","batch","vectors","j","job","q","Int8Array","round","err","tick","finish","asCosine","score","semanticCandidates","terms","allowed","match","t","includes","qv","rows","best","has","vector","byteLength","dot","existing","start_line","end_line","sort","a","b","similarity","hasEmbedding","similarNotes","opts","targetRows","step","ceil","target","_","exclude","other","len"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,EAAEC,YAAY,EAAEC,UAAU,EAAEC,aAAa,QAAQ,UAAU;AACzF,SAASC,OAAO,QAAQ,UAAU;AAClC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,WAAW,QAAQ,eAAe;AAC3C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,iBAAiB;AAC1C,SAASC,SAAS,QAAQ,aAAa;AAGvC,8FAA8F;AAC9F,gGAAgG;AAEhG,6EAA6E;AAC7E,yEAAyE;AACzE,MAAMC,aAAa;AACnB,0FAA0F;AAC1F,MAAMC,mBAAmB;AACzB,MAAMC,QAAQ;AAcd,8FAA8F;AAC9F,+FAA+F;AAC/F,EAAE;AACF,2FAA2F;AAC3F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,oDAAoD;AACpD,SAASC,SAASC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC;IACrF,MAAMC,QAAQH,KAAKI,KAAK,CAAC;IACzB,MAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,MAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;WAAMA;KAAO,GAAGA;IAC5E,MAAMO,SAAS;QAACX,mBAAAA,6BAAAA,OAAQY,KAAK;QAAEZ,mBAAAA,6BAAAA,OAAQa,OAAO;KAAC,CAACC,MAAM,CAACC,SAASzB,IAAI,CAAC;IACrE,MAAM0B,SAAkB,EAAE;IAC1BN,OAAOO,OAAO,CAAC,CAACC,OAAOZ;QACrB,MAAMa,MAAMb,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,MAAMa,OAAOlB,MACVmB,KAAK,CAACH,QAAQ,GAAGC,KACjB7B,IAAI,CAAC,MACLgC,IAAI;QACP,uFAAuF;QACvF,gDAAgD;QAChD,IAAIF,KAAKb,MAAM,GAAG,GAAGS,OAAOP,IAAI,CAAC;YAAEc,WAAWL,QAAQjB;YAAQuB,SAASL,MAAMlB;YAAQmB,MAAMT,SAAS,GAAGA,OAAO,EAAE,EAAES,MAAM,GAAGA;QAAK;IAClI;IACA,OAAOJ;AACT;AAEA,OAAO,MAAMS,QAAiB;IAC5BC,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAC1K;IACAC,SAAQC,GAAG,EAAEhC,IAAI,EAAEC,MAAM;QACvB,mFAAmF;QACnF,OAAOF,SAASC,MAAMC,QAAQ+B,IAAI5B,KAAK,CAAC,MAAMI,MAAM,GAAGR,KAAKI,KAAK,CAAC,MAAMI,MAAM;IAChF;IACAyB,QAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACA,uFAAuF;IACvF,mFAAmF;IACnFG,OAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAI,CAACA,WAAW;QAChB,MAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBpB,OAAO,CAAC,CAACsB,GAAGC,MAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEhB,SAAS,EAAEgB,EAAEf,OAAO;IACzF;IACAiB,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKlB,KAAK;IACnB;AACF,EAAE;AAEF,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,MAAMmB,cAAc;IAAC;IAAqB;CAAiB;AAC3D,8EAA8E;AAC9E,OAAO,MAAMC,kBAAkBD,YAAYtD,IAAI,CAAC,SAAS;AAEzD,+FAA+F;AAC/F,wDAAwD;AACxD,MAAMwD,cAAc;AAEpB,2FAA2F;AAC3F,OAAO,SAASC,eAAeC,KAAa;IAC1C,OAAOF,YAAYtC,IAAI,CAACwC;AAC1B;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,oFAAoF;AACpF,SAASC;IACP,MAAMC,MAAMC,QAAQC,GAAG,CAACC,cAAc;IACtC,OAAO/D,KAAK4D,OAAOA,IAAI3C,MAAM,GAAG,IAAI2C,MAAM5D,KAAKD,WAAW,WAAW,eAAe;AACtF;AAEA,2FAA2F;AAC3F,2FAA2F;AAC3F,gEAAgE;AAChE,OAAO,SAASiE,SAASN,KAAa;IACpC,IAAI,CAACF,YAAYtC,IAAI,CAACwC,QAAQ,OAAOA;IACrC,OAAO1D,KAAK2D,aAAaD,MAAMO,OAAO,CAAC,OAAO;AAChD;AAEA,sFAAsF;AACtF,OAAO,SAASC,cAAcR,KAAa;IACzC,MAAMS,MAAMH,SAASN;IACrB,OAAOJ,YAAYc,KAAK,CAAC,CAACf,OAAS3D,WAAWM,KAAKmE,KAAKd;AAC1D;AAEA,2FAA2F;AAC3F,kEAAkE;AAClE,OAAO,SAASgB,aAAaC,GAAW;IACtC,MAAMC,IAAItE,YAAYqE;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,IAAI,KAAK,SAASN,cAAcK,EAAEb,KAAK;AAClD;AAEA,SAASe,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,OAAOC;QAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAI7E,WAAW,eAAe,CAAC,uBAAuB,EAAEwE,IAAI,SAAS,EAAEI,IAAIE,MAAM,EAAE;QACtGlF,cAAc,GAAG6E,KAAK,KAAK,CAAC,EAAEM,OAAOC,IAAI,CAAC,MAAMJ,IAAIK,WAAW;QAC/DtF,WAAW,GAAG8E,KAAK,KAAK,CAAC,EAAEA;IAC7B;AACF;AAEA,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,eAAeS,cAAc1B,KAAa,EAAE2B,MAA4C;IAC7F,IAAI,CAAC5B,eAAeC,QAAQ;QAC1B,MAAM,IAAIxD,WAAW,eAAe,CAAC,aAAa,EAAEwD,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMS,MAAMH,SAASN;IACrB/D,UAAUwE,KAAK;QAAEmB,WAAW;IAAK;IACjC,KAAK,MAAMjC,QAAQC,YAAa;QAC9B,IAAI5D,WAAWM,KAAKmE,KAAKd,QAAQ;QACjCgC,mBAAAA,6BAAAA,OAAShC,MAAMc;QACf,MAAMM,YAAY,CAAC,uBAAuB,EAAEf,MAAM,cAAc,EAAEL,MAAM,EAAErD,KAAKmE,KAAKd;IACtF;IACA,OAAOc;AACT;AAEA,eAAeoB,eAAe7B,KAAa;;QAyBE8B,sBAA7BA,4BAAAA;IAxBd,MAAMrB,MAAMH,SAASN;IACrB,KAAK,MAAML,QAAQC,YAAa;QAC9B,IAAI,CAAC5D,WAAWM,KAAKmE,KAAKd,QAAQ;YAChC,uFAAuF;YACvF,2BAA2B;YAC3B,MAAMoC,MAAMhC,eAAeC,SAAS,yBAAyB,CAAC,SAAS,EAAEH,gBAAgB,kBAAkB,CAAC;YAC5G,MAAM,IAAIrD,WAAW,uBAAuB,CAAC,YAAY,EAAEwD,MAAM,6BAA6B,EAAES,IAAI,GAAG,EAAEsB,KAAK;QAChH;IACF;IAEA,MAAMhD,MAAM7C,aAAaI,KAAKmE,KAAK;IACnC,MAAMuB,YAAYC,OAAOlD,IAAImD,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACtD,IAAIuD,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIrG,WAAW,eAAe,GAAGwD,MAAM,oCAAoC,CAAC;IAC1H,MAAM8C,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlE,IAAImE,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatE,IAAIuE,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatE,IAAIuE,MAAM,CAACjF,KAAK,CAAC4E,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAMjB,gBAAgBM,KAAKC,KAAK,CAACnG,aAAaI,KAAKmE,KAAK,mBAAmB;IAC3E,oFAAoF;IACpF,MAAM,EAAE8C,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC;IACnC,MAAMC,MAAM,IAAID,UAAUzB,eAAe,CAAC;IAC1C,MAAM2B,iBAAQ3B,wBAAAA,cAAc9B,KAAK,cAAnB8B,6CAAAA,6BAAAA,sBAAqB4B,KAAK,cAA1B5B,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAc9B,KAAK,cAAnB8B,2CAAAA,qBAAqB6B,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIxF,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMyF,MAAM,AAACL,IAAIM,MAAM,CAAC1F,MAAM;YAAE2F,oBAAoB;QAAM,GAAGF,GAAG,CAAc/F,MAAM,CAAC,CAACkG,KAAO/B,OAAOgC,SAAS,CAACD,OAAOA,OAAOP;QAC5H,MAAMS,IAAI,IAAIb,aAAaN;QAC3B,IAAIc,IAAItG,MAAM,KAAK,GAAG,OAAO2G;QAC7B,KAAK,MAAMF,MAAMH,IAAK;YACpB,MAAMM,MAAMH,KAAKjB;YACjB,IAAK,IAAIqB,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIhB,MAAM,CAACe,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIP,IAAItG,MAAM;YAClB8G,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAEhE,OAAO;QAAE+C;QAAMtE,OAAO,OAAO+F,QAAUA,MAAMC,GAAG,CAACb;IAAK;AAC/E;AAEA,gFAAgF;AAEhF,eAAec,YAAY1E,KAAa,EAAEgB,GAAuB,EAAE2D,MAA0B;IAC3F,IAAI,CAAC3D,KAAK,MAAM,IAAIxE,WAAW,eAAe;IAC9C,MAAMoI,OAAO5D,IAAIT,OAAO,CAAC,QAAQ;IACjC,MAAMsE,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASxE,QAAQC,GAAG,CAACuE,OAAO,GAAGI;IAC3C,IAAID,KAAKD,QAAQG,aAAa,GAAG,CAAC,OAAO,EAAEF,KAAK;IAEhD,eAAeG,KAAKT,KAAe;QACjC,MAAMpD,MAAM,MAAMF,MAAM,GAAG0D,KAAK,WAAW,CAAC,EAAE;YAAEM,QAAQ;YAAQL;YAAS9H,MAAMqF,KAAK+C,SAAS,CAAC;gBAAEnF;gBAAOoF,OAAOZ;YAAM;QAAG;QACvH,IAAI,CAACpD,IAAIC,EAAE,EAAE,MAAM,IAAI7E,WAAW,eAAe,GAAGoI,KAAK,oBAAoB,EAAExD,IAAIE,MAAM,EAAE;QAC3F,MAAMvE,OAAQ,MAAMqE,IAAIiE,IAAI;QAC5B,OAAOtI,KAAKuI,IAAI,CAACb,GAAG,CAAC,CAACL,IAAMf,aAAa7B,IAAI,CAAC4C,EAAEmB,SAAS;IAC3D;IAEA,MAAMxC,OAAO,AAAC,CAAA,MAAMkC,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAAC1H,MAAM;IACxD,OAAO;QAAEyG,IAAI,CAAC,IAAI,EAAEY,KAAK,CAAC,EAAE5E,OAAO;QAAE+C;QAAMtE,OAAOwG;IAAK;AACzD;AAEA,MAAMO,YAAY,IAAIC;AAEtB,SAASC,YAAY9E,GAAW;QAGMC;IAFpC,MAAMA,IAAItE,YAAYqE;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIrE,WAAW,kBAAkB;IAC/C,MAAMmJ,MAAM,GAAG9E,EAAEC,IAAI,CAAC,CAAC,EAAED,EAAEb,KAAK,CAAC,CAAC,GAAEa,SAAAA,EAAEG,GAAG,cAALH,oBAAAA,SAAS,IAAI;IACjD,IAAI+E,IAAIJ,UAAUK,GAAG,CAACF;IACtB,IAAI,CAACC,GAAG;QACNA,IAAI/E,EAAEC,IAAI,KAAK,QAAQ4D,YAAY7D,EAAEb,KAAK,EAAEa,EAAEG,GAAG,EAAEH,EAAEiE,GAAG,IAAIjD,eAAehB,EAAEb,KAAK;QAClFwF,UAAUM,GAAG,CAACH,KAAKC;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAEjD,IAAY,EAAEkD,IAAa;IAC9D,MAAM/B,IAAI,IAAIb,aAAaN;IAC3B,IAAIsB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKC,QAAQ2B,IAAI,CAAC5B,EAAE,GAAG4B,IAAI,CAAC5B,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,GAAG4B,IAAI,CAAC5B,EAAE,GAAGC;IAChD,IAAI,CAAC4B,MAAM,OAAO;QAAE/B;QAAGgC,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAI/B,IAAI,GAAGA,IAAIrB,MAAMqB,IAAK+B,MAAM7B,KAAK6B,GAAG,CAACA,KAAK7B,KAAK8B,GAAG,CAAClC,CAAC,CAACE,EAAE;IAChE,OAAO;QAAEF;QAAGgC,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAazH,EAAgB,EAAEgC,GAAW,EAAE0F,OAAe;IAC/E,MAAMC,WAAW,MAAMb,YAAY9E,MAAM,kDAAkD;IAC3F,MAAM4F,QAAQ5H,GAAGM,OAAO,CAAC,oFAAoFuH,GAAG;IAChH,IAAID,MAAMjJ,MAAM,KAAK,GAAG;IACxB,MAAMmJ,YAAYpC,KAAKqC,GAAG,CAAChK,YAAY4J,SAASxD,IAAI;IAEpD,MAAM6D,SAAS,IAAInB;IACnB,KAAK,MAAMoB,OAAOL,MAAO;YACVI;QAAb,MAAME,QAAOF,cAAAA,OAAOf,GAAG,CAACgB,IAAI5H,IAAI,eAAnB2H,yBAAAA,cAAwB,EAAE;QACvCE,KAAKrJ,IAAI,CAACoJ,IAAIE,KAAK;QACnBH,OAAOd,GAAG,CAACe,IAAI5H,IAAI,EAAE6H;IACvB;IAEA,MAAME,OAA6D,EAAE;IACrE,KAAK,MAAM,CAAC/H,MAAMgI,UAAU,IAAIL,OAAQ;QACtC,IAAI5I;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAAStB,UAAU;gBAAEwK,SAASjI;gBAAMkI,SAAS7K,KAAKgK,SAASrH;gBAAOmI,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAE7I,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAE8I,GAAG,CAAClI,SAAS,CAACZ,KAAK;QACjJ,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMe,OAAOyH,UAAW,IAAIjJ,MAAM,CAACwB,IAAI,EAAEwH,KAAKvJ,IAAI,CAAC;YAAEwB;YAAM8H,OAAOvH;YAAKpB,MAAMJ,MAAM,CAACwB,IAAI,CAACpB,IAAI;QAAC;IACrG;IAEA,MAAMoJ,SAAS5I,GAAGM,OAAO,CAAC;IAC1B,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAMuI,SAAShL,SAAS,oBAAoBuK,KAAKzJ,MAAM;IACvD,IAAK,IAAID,IAAI,GAAGA,IAAI0J,KAAKzJ,MAAM,EAAED,KAAKT,MAAO;QAC3C,MAAM6K,QAAQV,KAAK3I,KAAK,CAACf,GAAGA,IAAIT;QAChC,MAAM8K,UAAU,MAAMpB,SAAS9H,KAAK,CAACiJ,MAAMjD,GAAG,CAAC,CAACmD,IAAMA,EAAExJ,IAAI;QAC5DQ,GAAGC,IAAI,CAAC;QACR,IAAI;YACF6I,MAAMzJ,OAAO,CAAC,CAAC4J,KAAKD;gBAClB,MAAM,EAAE1D,CAAC,EAAEgC,KAAK,EAAE,GAAGH,QAAQ4B,OAAO,CAACC,EAAE,EAAElB,WAAW;gBACpD,MAAMoB,IAAI,IAAIC,UAAUrB;gBACxB,IAAK,IAAItC,IAAI,GAAGA,IAAIsC,WAAWtC,IAAK0D,CAAC,CAAC1D,EAAE,GAAGE,KAAK0D,KAAK,CAAC9D,CAAC,CAACE,EAAE,GAAG8B;gBAC7DsB,OAAOrI,GAAG,CAAC+G,OAAO3E,OAAOC,IAAI,CAACsG,EAAExE,MAAM,GAAGuE,IAAI5I,IAAI,EAAE4I,IAAId,KAAK;YAC9D;YACAnI,GAAGC,IAAI,CAAC;QACV,EAAE,OAAOoJ,KAAK;YACZrJ,GAAGC,IAAI,CAAC;YACR,MAAMoJ;QACR;QACAR,OAAOS,IAAI,CAAC5D,KAAKqC,GAAG,CAACrJ,IAAIT,OAAOmK,KAAKzJ,MAAM;IAC7C;IACAkK,OAAOU,MAAM;AACf;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAO/D,KAAK0D,KAAK,CAAC1D,KAAKqC,GAAG,CAAC,GAAGrC,KAAK6B,GAAG,CAAC,CAAC,GAAGkC,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmB1J,EAAgB,EAAEgC,GAAW,EAAE2H,KAAa,EAAErH,MAAa,EAAEsH,OAAqB;QAO3GD;IANd,MAAMjC,UAAU,AAAC1F,IAAgC0F,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAI9J,WAAW,eAAe;IAClD,MAAM6J,aAAazH,IAAIgC,KAAK0F;IAE5B,MAAMC,WAAW,MAAMb,YAAY9E;IACnC,MAAM8F,YAAYpC,KAAKqC,GAAG,CAAChK,YAAY4J,SAASxD,IAAI;IACpD,MAAM3E,OAAO,EAACmK,eAAAA,MAAME,KAAK,CAAC,gCAAZF,0BAAAA,eAAkC,EAAE,EAAEzK,MAAM,CAAC,CAAC4K,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIpM,IAAI,CAAC;IAClH,MAAM,EAAE4H,GAAG0E,EAAE,EAAE,GAAG7C,QAAQ,AAAC,CAAA,MAAMQ,SAAS9H,KAAK,CAAC;QAACL;KAAK,CAAA,CAAE,CAAC,EAAE,EAAEsI,WAAW;IAExE,MAAMmC,OAAOjK,GAAGM,OAAO,CAAC,+FAA+FuH,GAAG;IAQ1H,MAAMqC,OAAO,IAAIrD;IACjB,KAAK,MAAMoB,OAAOgC,KAAM;QACtB,IAAIL,WAAW,CAACA,QAAQO,GAAG,CAAClC,IAAI5H,IAAI,GAAG;QACvC,MAAM6I,IAAI,IAAIC,UAAUlB,IAAImC,MAAM,CAAC1F,MAAM,EAAEuD,IAAImC,MAAM,CAAC9F,UAAU,EAAEoB,KAAKqC,GAAG,CAACD,WAAWG,IAAImC,MAAM,CAACC,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAI9E,IAAI,GAAGA,IAAI0D,EAAEvK,MAAM,EAAE6G,IAAK8E,OAAOpB,CAAC,CAAC1D,EAAE,GAAGwE,EAAE,CAACxE,EAAE;QACtD,MAAMiE,QAAQa,MAAMrC,IAAIX,KAAK;QAC7B,MAAMiD,WAAWL,KAAKjD,GAAG,CAACgB,IAAI5H,IAAI;QAClC,IAAI,CAACkK,YAAYd,QAAQc,SAASd,KAAK,EAAES,KAAKhD,GAAG,CAACe,IAAI5H,IAAI,EAAE;YAAEoJ;YAAOnL,OAAO,CAAC,CAAC,EAAE2J,IAAIuC,UAAU,CAAC,CAAC,EAAEvC,IAAIwC,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIP,KAAKpG,OAAO;KAAG,CACvB4G,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACnB,KAAK,GAAGkB,CAAC,CAAC,EAAE,CAAClB,KAAK,EACtChK,KAAK,CAAC,GAAG6C,QACTuD,GAAG,CAAC,CAAC,CAACxF,MAAMuK,EAAE,GAAM,CAAA;YAAEvK;YAAM/B,OAAOsM,EAAEtM,KAAK;YAAEuM,YAAYrB,SAASoB,EAAEnB,KAAK;QAAE,CAAA;AAC/E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,SAASqB,aAAa9K,EAAgB,EAAEK,IAAY;IACzD,MAAM4H,MAAMjI,GAAGM,OAAO,CAAC,kFAAkF2G,GAAG,CAAC5G;IAC7G,OAAO4H,QAAQ9B;AACjB;AAEA,+FAA+F;AAC/F,uEAAuE;AACvE,OAAO,SAAS4E,aAAa/K,EAAgB,EAAEc,IAAY,EAAET,IAAY,EAAE2K,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,MAAMC,aAAajL,GAAGM,OAAO,CAAC,+FAA+FuH,GAAG,CAACxH;IACjI,IAAI4K,WAAWtM,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMuM,OAAOxF,KAAK6B,GAAG,CAAC,GAAG7B,KAAKyF,IAAI,CAACF,WAAWtM,MAAM,GAAGX;IACvD,MAAMoN,SAASH,WAAW/L,MAAM,CAAC,CAACmM,GAAG3M,IAAMA,IAAIwM,SAAS,GAAGrF,GAAG,CAAC,CAACoC,MAAS,CAAA;YAAE3C,GAAG,IAAI6D,UAAUlB,IAAImC,MAAM,CAAC1F,MAAM,EAAEuD,IAAImC,MAAM,CAAC9F,UAAU,EAAE2D,IAAImC,MAAM,CAACC,UAAU;YAAG/C,OAAOW,IAAIX,KAAK;QAAC,CAAA;IAE/K,MAAM2C,OAAOjK,GAAGM,OAAO,CAAC,yEAAyEuH,GAAG;IACpG,MAAMqC,OAAO,IAAIrD;IACjB,KAAK,MAAMoB,OAAOgC,KAAM;QACtB,IAAIhC,IAAI5H,IAAI,KAAKA,QAAQ2K,KAAKM,OAAO,CAACnB,GAAG,CAAClC,IAAI5H,IAAI,KAAM2K,KAAKpB,OAAO,IAAI,CAACoB,KAAKpB,OAAO,CAACO,GAAG,CAAClC,IAAI5H,IAAI,GAAI;QACtG,MAAMkL,QAAQ,IAAIpC,UAAUlB,IAAImC,MAAM,CAAC1F,MAAM,EAAEuD,IAAImC,MAAM,CAAC9F,UAAU,EAAE2D,IAAImC,MAAM,CAACC,UAAU;QAC3F,KAAK,MAAMP,KAAKsB,OAAQ;YACtB,IAAId,MAAM;YACV,MAAMkB,MAAM9F,KAAKqC,GAAG,CAAC+B,EAAExE,CAAC,CAAC3G,MAAM,EAAE4M,MAAM5M,MAAM;YAC7C,IAAK,IAAI6G,IAAI,GAAGA,IAAIgG,KAAKhG,IAAK8E,OAAOR,EAAExE,CAAC,CAACE,EAAE,GAAG+F,KAAK,CAAC/F,EAAE;YACtD,MAAMiE,QAAQa,MAAMR,EAAExC,KAAK,GAAGW,IAAIX,KAAK;YACvC,MAAMiD,WAAWL,KAAKjD,GAAG,CAACgB,IAAI5H,IAAI;YAClC,IAAIkK,aAAapE,aAAasD,QAAQc,UAAUL,KAAKhD,GAAG,CAACe,IAAI5H,IAAI,EAAEoJ;QACrE;IACF;IAEA,OAAO;WAAIS,KAAKpG,OAAO;KAAG,CACvB4G,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BlL,KAAK,CAAC,GAAGuL,KAAKhH,CAAC,EACf6B,GAAG,CAAC,CAAC,CAACmB,GAAGyC,MAAM,GAAM,CAAA;YAAEpJ,MAAM2G;YAAG6D,YAAYrB,SAASC;QAAO,CAAA;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// Heading-based chunks, int8 vectors with a per-vector scale, NULL vector = not yet embedded.\n// Reconcile writes dirty rows; embedding tops up on the next search, so staleness costs recall.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,\n// preamble kept, whole body when no headings; the title/summary prefix mirrors bm25 weighting.\n//\n// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw\n// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw\n// put the frontmatter block in its own leading chunk on every note with a heading, which made\n// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also\n// disagreed with FTS, which indexes the body alone.\nfunction chunksOf(body: string, search?: { title: string; summary: string }, offset = 0): Chunk[] {\n  const lines = body.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    // A body with nothing in it yields no chunks at all, so a frontmatter-only note has no\n    // vectors rather than a vector of its own YAML.\n    if (text.length > 0) chunks.push({ startLine: start + offset, endLine: end + offset, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    return chunksOf(body, search, raw.split('\\n').length - body.split('\\n').length);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  // A tree with no embedding model never had extract() run for the doc (db.ts's per-file\n  // filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.\n  store(db, path, extracted) {\n    if (!extracted) return;\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nconst MODEL_FILES = ['model.safetensors', 'tokenizer.json'];\n// The pair, for messages that name what a local model directory must contain.\nexport const MODEL_FILENAMES = MODEL_FILES.join(' and ');\n\n// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather\n// than flattened into a cache key Windows would reject.\nconst HF_MODEL_ID = /^[A-Za-z0-9][\\w.-]*\\/[A-Za-z0-9][\\w.-]*$/;\n\n// Downloadable iff it names a Hugging Face repo; a local path is the caller's to populate.\nexport function isDownloadable(model: string): boolean {\n  return HF_MODEL_ID.test(model);\n}\n\n// Machine-wide, not per-tree: `.sense/` is the index a rebuild throws away, while a 124 MB\n// model is shared by every tree on the machine. Honors XDG_CACHE_HOME where it is set;\n// elsewhere ~/.cache, which is also where Hugging Face's own libraries keep theirs.\nfunction cacheRoot(): string {\n  const xdg = process.env.XDG_CACHE_HOME;\n  return join(xdg && xdg.length > 0 ? xdg : join(homedir(), '.cache'), 'sensemaking', 'models');\n}\n\n// A model is either a local directory the caller pointed at, or the cache `sense download`\n// fills. Nothing here fetches: an absent model degrades search to its other signals rather\n// than pulling 124 MB out of a command that reads like a query.\nexport function modelDir(model: string): string {\n  if (!HF_MODEL_ID.test(model)) return model;\n  return join(cacheRoot(), model.replace(/\\//g, '--'));\n}\n\n// Both files, so an interrupted download reads as absent and the next one resumes it.\nexport function hasModelFiles(model: string): boolean {\n  const dir = modelDir(model);\n  return MODEL_FILES.every((file) => existsSync(join(dir, file)));\n}\n\n// api providers have nothing to download, so they are never \"missing\" here; an unreachable\n// endpoint surfaces as an EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.type === 'api' || hasModelFiles(e.model);\n}\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\n// The only code path that touches the network for weights. `sense download` calls it; no\n// query ever does. Idempotent: a file already on disk is left alone.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const dir = modelDir(model);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  const dir = modelDir(model);\n  for (const file of MODEL_FILES) {\n    if (!existsSync(join(dir, file))) {\n      // A local path the caller controls gets told what is missing where; only a repo id can\n      // be fixed by downloading.\n      const fix = isDownloadable(model) ? 'run `sense download`' : `expected ${MODEL_FILENAMES} in that directory`;\n      throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); ${fix}`);\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json, then run `sense download`');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + BATCH, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["existsSync","mkdirSync","readFileSync","renameSync","writeFileSync","homedir","join","embedConfig","SenseError","progress","parseFile","STORE_DIMS","TARGET_CHUNK_CAP","BATCH","chunksOf","body","search","offset","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","chunks","forEach","start","end","text","slice","trim","startLine","endLine","embed","name","schema","db","exec","extract","raw","remove","path","prepare","run","store","extracted","insert","c","idx","enabledForFile","_cfg","file","MODEL_FILES","MODEL_FILENAMES","HF_MODEL_ID","isDownloadable","model","cacheRoot","xdg","process","env","XDG_CACHE_HOME","modelDir","replace","hasModelFiles","dir","every","modelPresent","cfg","e","type","fetchToFile","url","dest","fetch","then","res","ok","status","Buffer","from","arrayBuffer","downloadModel","onFile","recursive","staticProvider","tokenizerJson","fix","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","Tokenizer","tok","unkId","vocab","unk_token","one","ids","encode","add_special_tokens","id","isInteger","v","off","d","norm","Math","sqrt","texts","map","apiProvider","keyEnv","base","headers","key","undefined","authorization","post","method","stringify","input","json","data","embedding","providers","Map","getProvider","sig","p","get","set","toStore","full","int8","scale","max","abs","embedPending","baseDir","provider","dirty","all","storeDims","min","byPath","row","list","chunk","jobs","chunkIdxs","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","update","report","batch","vectors","j","job","q","Int8Array","round","err","tick","finish","asCosine","score","semanticCandidates","terms","allowed","match","t","includes","qv","rows","best","has","vector","byteLength","dot","existing","start_line","end_line","sort","a","b","similarity","hasEmbedding","similarNotes","opts","targetRows","step","ceil","target","_","exclude","other","len"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,EAAEC,YAAY,EAAEC,UAAU,EAAEC,aAAa,QAAQ,UAAU;AACzF,SAASC,OAAO,QAAQ,UAAU;AAClC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,iBAAiB;AAC1C,SAASC,SAAS,QAAQ,aAAa;AAGvC,8FAA8F;AAC9F,gGAAgG;AAEhG,6EAA6E;AAC7E,yEAAyE;AACzE,MAAMC,aAAa;AACnB,0FAA0F;AAC1F,MAAMC,mBAAmB;AACzB,MAAMC,QAAQ;AAcd,8FAA8F;AAC9F,+FAA+F;AAC/F,EAAE;AACF,2FAA2F;AAC3F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,oDAAoD;AACpD,SAASC,SAASC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC;IACrF,MAAMC,QAAQH,KAAKI,KAAK,CAAC;IACzB,MAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,MAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;WAAMA;KAAO,GAAGA;IAC5E,MAAMO,SAAS;QAACX,mBAAAA,6BAAAA,OAAQY,KAAK;QAAEZ,mBAAAA,6BAAAA,OAAQa,OAAO;KAAC,CAACC,MAAM,CAACC,SAASzB,IAAI,CAAC;IACrE,MAAM0B,SAAkB,EAAE;IAC1BN,OAAOO,OAAO,CAAC,CAACC,OAAOZ;QACrB,MAAMa,MAAMb,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,MAAMa,OAAOlB,MACVmB,KAAK,CAACH,QAAQ,GAAGC,KACjB7B,IAAI,CAAC,MACLgC,IAAI;QACP,uFAAuF;QACvF,gDAAgD;QAChD,IAAIF,KAAKb,MAAM,GAAG,GAAGS,OAAOP,IAAI,CAAC;YAAEc,WAAWL,QAAQjB;YAAQuB,SAASL,MAAMlB;YAAQmB,MAAMT,SAAS,GAAGA,OAAO,EAAE,EAAES,MAAM,GAAGA;QAAK;IAClI;IACA,OAAOJ;AACT;AAEA,OAAO,MAAMS,QAAiB;IAC5BC,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAC1K;IACAC,SAAQC,GAAG,EAAEhC,IAAI,EAAEC,MAAM;QACvB,mFAAmF;QACnF,OAAOF,SAASC,MAAMC,QAAQ+B,IAAI5B,KAAK,CAAC,MAAMI,MAAM,GAAGR,KAAKI,KAAK,CAAC,MAAMI,MAAM;IAChF;IACAyB,QAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACA,uFAAuF;IACvF,mFAAmF;IACnFG,OAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAI,CAACA,WAAW;QAChB,MAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBpB,OAAO,CAAC,CAACsB,GAAGC,MAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEhB,SAAS,EAAEgB,EAAEf,OAAO;IACzF;IACAiB,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKlB,KAAK;IACnB;AACF,EAAE;AAEF,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,MAAMmB,cAAc;IAAC;IAAqB;CAAiB;AAC3D,8EAA8E;AAC9E,OAAO,MAAMC,kBAAkBD,YAAYtD,IAAI,CAAC,SAAS;AAEzD,+FAA+F;AAC/F,wDAAwD;AACxD,MAAMwD,cAAc;AAEpB,2FAA2F;AAC3F,OAAO,SAASC,eAAeC,KAAa;IAC1C,OAAOF,YAAYtC,IAAI,CAACwC;AAC1B;AAEA,2FAA2F;AAC3F,uFAAuF;AACvF,oFAAoF;AACpF,SAASC;IACP,MAAMC,MAAMC,QAAQC,GAAG,CAACC,cAAc;IACtC,OAAO/D,KAAK4D,OAAOA,IAAI3C,MAAM,GAAG,IAAI2C,MAAM5D,KAAKD,WAAW,WAAW,eAAe;AACtF;AAEA,2FAA2F;AAC3F,2FAA2F;AAC3F,gEAAgE;AAChE,OAAO,SAASiE,SAASN,KAAa;IACpC,IAAI,CAACF,YAAYtC,IAAI,CAACwC,QAAQ,OAAOA;IACrC,OAAO1D,KAAK2D,aAAaD,MAAMO,OAAO,CAAC,OAAO;AAChD;AAEA,sFAAsF;AACtF,OAAO,SAASC,cAAcR,KAAa;IACzC,MAAMS,MAAMH,SAASN;IACrB,OAAOJ,YAAYc,KAAK,CAAC,CAACf,OAAS3D,WAAWM,KAAKmE,KAAKd;AAC1D;AAEA,2FAA2F;AAC3F,kEAAkE;AAClE,OAAO,SAASgB,aAAaC,GAAW;IACtC,MAAMC,IAAItE,YAAYqE;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,IAAI,KAAK,SAASN,cAAcK,EAAEb,KAAK;AAClD;AAEA,SAASe,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,OAAOC;QAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAI7E,WAAW,eAAe,CAAC,uBAAuB,EAAEwE,IAAI,SAAS,EAAEI,IAAIE,MAAM,EAAE;QACtGlF,cAAc,GAAG6E,KAAK,KAAK,CAAC,EAAEM,OAAOC,IAAI,CAAC,MAAMJ,IAAIK,WAAW;QAC/DtF,WAAW,GAAG8E,KAAK,KAAK,CAAC,EAAEA;IAC7B;AACF;AAEA,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,eAAeS,cAAc1B,KAAa,EAAE2B,MAA4C;IAC7F,IAAI,CAAC5B,eAAeC,QAAQ;QAC1B,MAAM,IAAIxD,WAAW,eAAe,CAAC,aAAa,EAAEwD,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMS,MAAMH,SAASN;IACrB/D,UAAUwE,KAAK;QAAEmB,WAAW;IAAK;IACjC,KAAK,MAAMjC,QAAQC,YAAa;QAC9B,IAAI5D,WAAWM,KAAKmE,KAAKd,QAAQ;QACjCgC,mBAAAA,6BAAAA,OAAShC,MAAMc;QACf,MAAMM,YAAY,CAAC,uBAAuB,EAAEf,MAAM,cAAc,EAAEL,MAAM,EAAErD,KAAKmE,KAAKd;IACtF;IACA,OAAOc;AACT;AAEA,eAAeoB,eAAe7B,KAAa;;QAyBE8B,sBAA7BA,4BAAAA;IAxBd,MAAMrB,MAAMH,SAASN;IACrB,KAAK,MAAML,QAAQC,YAAa;QAC9B,IAAI,CAAC5D,WAAWM,KAAKmE,KAAKd,QAAQ;YAChC,uFAAuF;YACvF,2BAA2B;YAC3B,MAAMoC,MAAMhC,eAAeC,SAAS,yBAAyB,CAAC,SAAS,EAAEH,gBAAgB,kBAAkB,CAAC;YAC5G,MAAM,IAAIrD,WAAW,uBAAuB,CAAC,YAAY,EAAEwD,MAAM,6BAA6B,EAAES,IAAI,GAAG,EAAEsB,KAAK;QAChH;IACF;IAEA,MAAMhD,MAAM7C,aAAaI,KAAKmE,KAAK;IACnC,MAAMuB,YAAYC,OAAOlD,IAAImD,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACtD,IAAIuD,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIrG,WAAW,eAAe,GAAGwD,MAAM,oCAAoC,CAAC;IAC1H,MAAM8C,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlE,IAAImE,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatE,IAAIuE,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatE,IAAIuE,MAAM,CAACjF,KAAK,CAAC4E,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAMjB,gBAAgBM,KAAKC,KAAK,CAACnG,aAAaI,KAAKmE,KAAK,mBAAmB;IAC3E,oFAAoF;IACpF,MAAM,EAAE8C,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC;IACnC,MAAMC,MAAM,IAAID,UAAUzB,eAAe,CAAC;IAC1C,MAAM2B,iBAAQ3B,wBAAAA,cAAc9B,KAAK,cAAnB8B,6CAAAA,6BAAAA,sBAAqB4B,KAAK,cAA1B5B,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAc9B,KAAK,cAAnB8B,2CAAAA,qBAAqB6B,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIxF,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMyF,MAAM,AAACL,IAAIM,MAAM,CAAC1F,MAAM;YAAE2F,oBAAoB;QAAM,GAAGF,GAAG,CAAc/F,MAAM,CAAC,CAACkG,KAAO/B,OAAOgC,SAAS,CAACD,OAAOA,OAAOP;QAC5H,MAAMS,IAAI,IAAIb,aAAaN;QAC3B,IAAIc,IAAItG,MAAM,KAAK,GAAG,OAAO2G;QAC7B,KAAK,MAAMF,MAAMH,IAAK;YACpB,MAAMM,MAAMH,KAAKjB;YACjB,IAAK,IAAIqB,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIhB,MAAM,CAACe,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIP,IAAItG,MAAM;YAClB8G,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAEhE,OAAO;QAAE+C;QAAMtE,OAAO,OAAO+F,QAAUA,MAAMC,GAAG,CAACb;IAAK;AAC/E;AAEA,gFAAgF;AAEhF,eAAec,YAAY1E,KAAa,EAAEgB,GAAuB,EAAE2D,MAA0B;IAC3F,IAAI,CAAC3D,KAAK,MAAM,IAAIxE,WAAW,eAAe;IAC9C,MAAMoI,OAAO5D,IAAIT,OAAO,CAAC,QAAQ;IACjC,MAAMsE,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASxE,QAAQC,GAAG,CAACuE,OAAO,GAAGI;IAC3C,IAAID,KAAKD,QAAQG,aAAa,GAAG,CAAC,OAAO,EAAEF,KAAK;IAEhD,eAAeG,KAAKT,KAAe;QACjC,MAAMpD,MAAM,MAAMF,MAAM,GAAG0D,KAAK,WAAW,CAAC,EAAE;YAAEM,QAAQ;YAAQL;YAAS9H,MAAMqF,KAAK+C,SAAS,CAAC;gBAAEnF;gBAAOoF,OAAOZ;YAAM;QAAG;QACvH,IAAI,CAACpD,IAAIC,EAAE,EAAE,MAAM,IAAI7E,WAAW,eAAe,GAAGoI,KAAK,oBAAoB,EAAExD,IAAIE,MAAM,EAAE;QAC3F,MAAMvE,OAAQ,MAAMqE,IAAIiE,IAAI;QAC5B,OAAOtI,KAAKuI,IAAI,CAACb,GAAG,CAAC,CAACL,IAAMf,aAAa7B,IAAI,CAAC4C,EAAEmB,SAAS;IAC3D;IAEA,MAAMxC,OAAO,AAAC,CAAA,MAAMkC,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAAC1H,MAAM;IACxD,OAAO;QAAEyG,IAAI,CAAC,IAAI,EAAEY,KAAK,CAAC,EAAE5E,OAAO;QAAE+C;QAAMtE,OAAOwG;IAAK;AACzD;AAEA,MAAMO,YAAY,IAAIC;AAEtB,SAASC,YAAY9E,GAAW;QAGMC;IAFpC,MAAMA,IAAItE,YAAYqE;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIrE,WAAW,kBAAkB;IAC/C,MAAMmJ,MAAM,GAAG9E,EAAEC,IAAI,CAAC,CAAC,EAAED,EAAEb,KAAK,CAAC,CAAC,GAAEa,SAAAA,EAAEG,GAAG,cAALH,oBAAAA,SAAS,IAAI;IACjD,IAAI+E,IAAIJ,UAAUK,GAAG,CAACF;IACtB,IAAI,CAACC,GAAG;QACNA,IAAI/E,EAAEC,IAAI,KAAK,QAAQ4D,YAAY7D,EAAEb,KAAK,EAAEa,EAAEG,GAAG,EAAEH,EAAEiE,GAAG,IAAIjD,eAAehB,EAAEb,KAAK;QAClFwF,UAAUM,GAAG,CAACH,KAAKC;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAEjD,IAAY,EAAEkD,IAAa;IAC9D,MAAM/B,IAAI,IAAIb,aAAaN;IAC3B,IAAIsB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKC,QAAQ2B,IAAI,CAAC5B,EAAE,GAAG4B,IAAI,CAAC5B,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,GAAG4B,IAAI,CAAC5B,EAAE,GAAGC;IAChD,IAAI,CAAC4B,MAAM,OAAO;QAAE/B;QAAGgC,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAI/B,IAAI,GAAGA,IAAIrB,MAAMqB,IAAK+B,MAAM7B,KAAK6B,GAAG,CAACA,KAAK7B,KAAK8B,GAAG,CAAClC,CAAC,CAACE,EAAE;IAChE,OAAO;QAAEF;QAAGgC,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAazH,EAAgB,EAAEgC,GAAW,EAAE0F,OAAe;IAC/E,MAAMC,WAAW,MAAMb,YAAY9E,MAAM,kDAAkD;IAC3F,MAAM4F,QAAQ5H,GAAGM,OAAO,CAAC,oFAAoFuH,GAAG;IAChH,IAAID,MAAMjJ,MAAM,KAAK,GAAG;IACxB,MAAMmJ,YAAYpC,KAAKqC,GAAG,CAAChK,YAAY4J,SAASxD,IAAI;IAEpD,MAAM6D,SAAS,IAAInB;IACnB,KAAK,MAAMoB,OAAOL,MAAO;YACVI;QAAb,MAAME,QAAOF,cAAAA,OAAOf,GAAG,CAACgB,IAAI5H,IAAI,eAAnB2H,yBAAAA,cAAwB,EAAE;QACvCE,KAAKrJ,IAAI,CAACoJ,IAAIE,KAAK;QACnBH,OAAOd,GAAG,CAACe,IAAI5H,IAAI,EAAE6H;IACvB;IAEA,MAAME,OAA6D,EAAE;IACrE,KAAK,MAAM,CAAC/H,MAAMgI,UAAU,IAAIL,OAAQ;QACtC,IAAI5I;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAAStB,UAAU;gBAAEwK,SAASjI;gBAAMkI,SAAS7K,KAAKgK,SAASrH;gBAAOmI,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAE9I,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAE+I,GAAG,CAACnI,SAAS,CAACZ,KAAK;QAC7J,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMe,OAAOyH,UAAW,IAAIjJ,MAAM,CAACwB,IAAI,EAAEwH,KAAKvJ,IAAI,CAAC;YAAEwB;YAAM8H,OAAOvH;YAAKpB,MAAMJ,MAAM,CAACwB,IAAI,CAACpB,IAAI;QAAC;IACrG;IAEA,MAAMqJ,SAAS7I,GAAGM,OAAO,CAAC;IAC1B,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAMwI,SAASjL,SAAS,oBAAoBuK,KAAKzJ,MAAM;IACvD,IAAK,IAAID,IAAI,GAAGA,IAAI0J,KAAKzJ,MAAM,EAAED,KAAKT,MAAO;QAC3C,MAAM8K,QAAQX,KAAK3I,KAAK,CAACf,GAAGA,IAAIT;QAChC,MAAM+K,UAAU,MAAMrB,SAAS9H,KAAK,CAACkJ,MAAMlD,GAAG,CAAC,CAACoD,IAAMA,EAAEzJ,IAAI;QAC5DQ,GAAGC,IAAI,CAAC;QACR,IAAI;YACF8I,MAAM1J,OAAO,CAAC,CAAC6J,KAAKD;gBAClB,MAAM,EAAE3D,CAAC,EAAEgC,KAAK,EAAE,GAAGH,QAAQ6B,OAAO,CAACC,EAAE,EAAEnB,WAAW;gBACpD,MAAMqB,IAAI,IAAIC,UAAUtB;gBACxB,IAAK,IAAItC,IAAI,GAAGA,IAAIsC,WAAWtC,IAAK2D,CAAC,CAAC3D,EAAE,GAAGE,KAAK2D,KAAK,CAAC/D,CAAC,CAACE,EAAE,GAAG8B;gBAC7DuB,OAAOtI,GAAG,CAAC+G,OAAO3E,OAAOC,IAAI,CAACuG,EAAEzE,MAAM,GAAGwE,IAAI7I,IAAI,EAAE6I,IAAIf,KAAK;YAC9D;YACAnI,GAAGC,IAAI,CAAC;QACV,EAAE,OAAOqJ,KAAK;YACZtJ,GAAGC,IAAI,CAAC;YACR,MAAMqJ;QACR;QACAR,OAAOS,IAAI,CAAC7D,KAAKqC,GAAG,CAACrJ,IAAIT,OAAOmK,KAAKzJ,MAAM;IAC7C;IACAmK,OAAOU,MAAM;AACf;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAOhE,KAAK2D,KAAK,CAAC3D,KAAKqC,GAAG,CAAC,GAAGrC,KAAK6B,GAAG,CAAC,CAAC,GAAGmC,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmB3J,EAAgB,EAAEgC,GAAW,EAAE4H,KAAa,EAAEtH,MAAa,EAAEuH,OAAqB;QAO3GD;IANd,MAAMlC,UAAU,AAAC1F,IAAgC0F,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAI9J,WAAW,eAAe;IAClD,MAAM6J,aAAazH,IAAIgC,KAAK0F;IAE5B,MAAMC,WAAW,MAAMb,YAAY9E;IACnC,MAAM8F,YAAYpC,KAAKqC,GAAG,CAAChK,YAAY4J,SAASxD,IAAI;IACpD,MAAM3E,OAAO,EAACoK,eAAAA,MAAME,KAAK,CAAC,gCAAZF,0BAAAA,eAAkC,EAAE,EAAE1K,MAAM,CAAC,CAAC6K,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIrM,IAAI,CAAC;IAClH,MAAM,EAAE4H,GAAG2E,EAAE,EAAE,GAAG9C,QAAQ,AAAC,CAAA,MAAMQ,SAAS9H,KAAK,CAAC;QAACL;KAAK,CAAA,CAAE,CAAC,EAAE,EAAEsI,WAAW;IAExE,MAAMoC,OAAOlK,GAAGM,OAAO,CAAC,+FAA+FuH,GAAG;IAQ1H,MAAMsC,OAAO,IAAItD;IACjB,KAAK,MAAMoB,OAAOiC,KAAM;QACtB,IAAIL,WAAW,CAACA,QAAQO,GAAG,CAACnC,IAAI5H,IAAI,GAAG;QACvC,MAAM8I,IAAI,IAAIC,UAAUnB,IAAIoC,MAAM,CAAC3F,MAAM,EAAEuD,IAAIoC,MAAM,CAAC/F,UAAU,EAAEoB,KAAKqC,GAAG,CAACD,WAAWG,IAAIoC,MAAM,CAACC,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAI/E,IAAI,GAAGA,IAAI2D,EAAExK,MAAM,EAAE6G,IAAK+E,OAAOpB,CAAC,CAAC3D,EAAE,GAAGyE,EAAE,CAACzE,EAAE;QACtD,MAAMkE,QAAQa,MAAMtC,IAAIX,KAAK;QAC7B,MAAMkD,WAAWL,KAAKlD,GAAG,CAACgB,IAAI5H,IAAI;QAClC,IAAI,CAACmK,YAAYd,QAAQc,SAASd,KAAK,EAAES,KAAKjD,GAAG,CAACe,IAAI5H,IAAI,EAAE;YAAEqJ;YAAOpL,OAAO,CAAC,CAAC,EAAE2J,IAAIwC,UAAU,CAAC,CAAC,EAAExC,IAAIyC,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIP,KAAKrG,OAAO;KAAG,CACvB6G,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACnB,KAAK,GAAGkB,CAAC,CAAC,EAAE,CAAClB,KAAK,EACtCjK,KAAK,CAAC,GAAG6C,QACTuD,GAAG,CAAC,CAAC,CAACxF,MAAMwK,EAAE,GAAM,CAAA;YAAExK;YAAM/B,OAAOuM,EAAEvM,KAAK;YAAEwM,YAAYrB,SAASoB,EAAEnB,KAAK;QAAE,CAAA;AAC/E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,SAASqB,aAAa/K,EAAgB,EAAEK,IAAY;IACzD,MAAM4H,MAAMjI,GAAGM,OAAO,CAAC,kFAAkF2G,GAAG,CAAC5G;IAC7G,OAAO4H,QAAQ9B;AACjB;AAEA,+FAA+F;AAC/F,uEAAuE;AACvE,OAAO,SAAS6E,aAAahL,EAAgB,EAAEc,IAAY,EAAET,IAAY,EAAE4K,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,MAAMC,aAAalL,GAAGM,OAAO,CAAC,+FAA+FuH,GAAG,CAACxH;IACjI,IAAI6K,WAAWvM,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMwM,OAAOzF,KAAK6B,GAAG,CAAC,GAAG7B,KAAK0F,IAAI,CAACF,WAAWvM,MAAM,GAAGX;IACvD,MAAMqN,SAASH,WAAWhM,MAAM,CAAC,CAACoM,GAAG5M,IAAMA,IAAIyM,SAAS,GAAGtF,GAAG,CAAC,CAACoC,MAAS,CAAA;YAAE3C,GAAG,IAAI8D,UAAUnB,IAAIoC,MAAM,CAAC3F,MAAM,EAAEuD,IAAIoC,MAAM,CAAC/F,UAAU,EAAE2D,IAAIoC,MAAM,CAACC,UAAU;YAAGhD,OAAOW,IAAIX,KAAK;QAAC,CAAA;IAE/K,MAAM4C,OAAOlK,GAAGM,OAAO,CAAC,yEAAyEuH,GAAG;IACpG,MAAMsC,OAAO,IAAItD;IACjB,KAAK,MAAMoB,OAAOiC,KAAM;QACtB,IAAIjC,IAAI5H,IAAI,KAAKA,QAAQ4K,KAAKM,OAAO,CAACnB,GAAG,CAACnC,IAAI5H,IAAI,KAAM4K,KAAKpB,OAAO,IAAI,CAACoB,KAAKpB,OAAO,CAACO,GAAG,CAACnC,IAAI5H,IAAI,GAAI;QACtG,MAAMmL,QAAQ,IAAIpC,UAAUnB,IAAIoC,MAAM,CAAC3F,MAAM,EAAEuD,IAAIoC,MAAM,CAAC/F,UAAU,EAAE2D,IAAIoC,MAAM,CAACC,UAAU;QAC3F,KAAK,MAAMP,KAAKsB,OAAQ;YACtB,IAAId,MAAM;YACV,MAAMkB,MAAM/F,KAAKqC,GAAG,CAACgC,EAAEzE,CAAC,CAAC3G,MAAM,EAAE6M,MAAM7M,MAAM;YAC7C,IAAK,IAAI6G,IAAI,GAAGA,IAAIiG,KAAKjG,IAAK+E,OAAOR,EAAEzE,CAAC,CAACE,EAAE,GAAGgG,KAAK,CAAChG,EAAE;YACtD,MAAMkE,QAAQa,MAAMR,EAAEzC,KAAK,GAAGW,IAAIX,KAAK;YACvC,MAAMkD,WAAWL,KAAKlD,GAAG,CAACgB,IAAI5H,IAAI;YAClC,IAAImK,aAAarE,aAAauD,QAAQc,UAAUL,KAAKjD,GAAG,CAACe,IAAI5H,IAAI,EAAEqJ;QACrE;IACF;IAEA,OAAO;WAAIS,KAAKrG,OAAO;KAAG,CACvB6G,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BnL,KAAK,CAAC,GAAGwL,KAAKjH,CAAC,EACf6B,GAAG,CAAC,CAAC,CAACmB,GAAG0C,MAAM,GAAM,CAAA;YAAErJ,MAAM2G;YAAG8D,YAAYrB,SAASC;QAAO,CAAA;AACjE"}

package/dist/esm/features/index.d.ts

@@ -1,4 +1,4 @@
-import type { Config } from '../config.js';
+import type { Config } from '../config/index.js';
 import type { Feature } from './types.js';
 export declare const FEATURES: Feature[];
 export declare function activeFeatures(cfg: Config): Feature[];

package/dist/esm/features/index.js

@@ -1,12 +1,14 @@
-import { featureEnabled } from '../config.js';
+import { featureEnabled } from '../config/index.js';
 import { embed } from './embed.js';
 import { links } from './links.js';
 import { rank } from './rank.js';
 import { sections } from './sections.js';
+import { tags } from './tags.js';
 // Registry order matters: rank reads the links table in afterReconcile.
 export const FEATURES = [
     links,
     sections,
+    tags,
     rank,
     embed
 ];

package/dist/esm/features/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/index.ts"],"sourcesContent":["import type { Config } from '../config.ts';\nimport { featureEnabled } from '../config.ts';\nimport { embed } from './embed.ts';\nimport { links } from './links.ts';\nimport { rank } from './rank.ts';\nimport { sections } from './sections.ts';\nimport type { Feature } from './types.ts';\n\n// Registry order matters: rank reads the links table in afterReconcile.\nexport const FEATURES: Feature[] = [links, sections, rank, embed];\n\nexport function activeFeatures(cfg: Config): Feature[] {\n  return FEATURES.filter((feature) => featureEnabled(cfg, feature.name));\n}\n\nexport { linkEdges } from './links.ts';\nexport type { Section } from './sections.ts';\nexport type { Feature } from './types.ts';\n"],"names":["featureEnabled","embed","links","rank","sections","FEATURES","activeFeatures","cfg","filter","feature","name","linkEdges"],"mappings":"AACA,SAASA,cAAc,QAAQ,eAAe;AAC9C,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,QAAQ,QAAQ,gBAAgB;AAGzC,wEAAwE;AACxE,OAAO,MAAMC,WAAsB;IAACH;IAAOE;IAAUD;IAAMF;CAAM,CAAC;AAElE,OAAO,SAASK,eAAeC,GAAW;IACxC,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAYT,eAAeO,KAAKE,QAAQC,IAAI;AACtE;AAEA,SAASC,SAAS,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/index.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { featureEnabled } from '../config/index.ts';\nimport { embed } from './embed.ts';\nimport { links } from './links.ts';\nimport { rank } from './rank.ts';\nimport { sections } from './sections.ts';\nimport { tags } from './tags.ts';\nimport type { Feature } from './types.ts';\n\n// Registry order matters: rank reads the links table in afterReconcile.\nexport const FEATURES: Feature[] = [links, sections, tags, rank, embed];\n\nexport function activeFeatures(cfg: Config): Feature[] {\n  return FEATURES.filter((feature) => featureEnabled(cfg, feature.name));\n}\n\nexport { linkEdges } from './links.ts';\nexport type { Section } from './sections.ts';\nexport type { Feature } from './types.ts';\n"],"names":["featureEnabled","embed","links","rank","sections","tags","FEATURES","activeFeatures","cfg","filter","feature","name","linkEdges"],"mappings":"AACA,SAASA,cAAc,QAAQ,qBAAqB;AACpD,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,QAAQ,QAAQ,gBAAgB;AACzC,SAASC,IAAI,QAAQ,YAAY;AAGjC,wEAAwE;AACxE,OAAO,MAAMC,WAAsB;IAACJ;IAAOE;IAAUC;IAAMF;IAAMF;CAAM,CAAC;AAExE,OAAO,SAASM,eAAeC,GAAW;IACxC,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAYV,eAAeQ,KAAKE,QAAQC,IAAI;AACtE;AAEA,SAASC,SAAS,QAAQ,aAAa"}