sensemaking 0.4.0 → 0.6.0

package/dist/esm/features/embed.js

@@ -0,0 +1,290 @@
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { embedConfig } from '../config.js';
+import { SenseError } from '../errors.js';
+import { parseFile } from '../scan.js';
+// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,
+// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.
+// Reconcile stores dirty rows inside its transaction; embedding tops up on the next
+// semantic query (embedPending), so staleness costs recall, never correctness.
+// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is
+// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
+const STORE_DIMS = 256;
+const BATCH = 64;
+// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text
+// re-derived from the file -- chunk text is never stored). Heading-delimited with the
+// preamble kept; whole file when no headings. The title/summary prefix mirrors the
+// bm25 column weighting.
+function chunksOf(raw, search) {
+    const lines = raw.split('\n');
+    const starts = [];
+    let inFence = false;
+    for(let i = 0; i < lines.length; i++){
+        if (/^(```|~~~)/.test(lines[i])) {
+            inFence = !inFence;
+            continue;
+        }
+        if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);
+    }
+    const bounds = starts.length === 0 ? [
+        1
+    ] : starts[0] > 1 ? [
+        1,
+        ...starts
+    ] : starts;
+    const prefix = [
+        search === null || search === void 0 ? void 0 : search.title,
+        search === null || search === void 0 ? void 0 : search.summary
+    ].filter(Boolean).join('\n');
+    const chunks = [];
+    bounds.forEach((start, i)=>{
+        const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;
+        const text = lines.slice(start - 1, end).join('\n').trim();
+        if (text.length > 0) chunks.push({
+            startLine: start,
+            endLine: end,
+            text: prefix ? `${prefix}\n${text}` : text
+        });
+    });
+    return chunks;
+}
+export const embed = {
+    name: 'embed',
+    schema (db) {
+        db.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))`);
+    },
+    extract (raw, _body, search) {
+        return chunksOf(raw, search);
+    },
+    remove (db, path) {
+        db.prepare('DELETE FROM embeddings WHERE "path" = ?').run(path);
+    },
+    store (db, path, extracted) {
+        const insert = db.prepare('INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');
+        extracted.forEach((c, idx)=>insert.run(path, idx, c.startLine, c.endLine));
+    }
+};
+// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---
+// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,
+// L2-normalize.
+function fetchToFile(url, dest) {
+    return fetch(url).then(async (res)=>{
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);
+        writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));
+        renameSync(`${dest}.part`, dest);
+    });
+}
+async function staticProvider(model) {
+    var _ref;
+    var _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1;
+    let dir = model;
+    if (!existsSync(join(dir, 'model.safetensors'))) {
+        dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\//g, '--'));
+        mkdirSync(dir, {
+            recursive: true
+        });
+        for (const file of [
+            'model.safetensors',
+            'tokenizer.json'
+        ]){
+            if (!existsSync(join(dir, file))) {
+                console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);
+                await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));
+            }
+        }
+    }
+    const raw = readFileSync(join(dir, 'model.safetensors'));
+    const headerLen = Number(raw.readBigUInt64LE(0));
+    const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8'));
+    const entry = Object.entries(header).find(([k])=>k !== '__metadata__');
+    if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);
+    const spec = entry[1];
+    const dims = spec.shape[1];
+    const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];
+    const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));
+    const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));
+    // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.
+    const { Tokenizer } = await import('@huggingface/tokenizers');
+    const tok = new Tokenizer(tokenizerJson, {});
+    const unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
+    function one(text) {
+        const ids = tok.encode(text, {
+            add_special_tokens: false
+        }).ids.filter((id)=>id !== unkId);
+        const v = new Float32Array(dims);
+        if (ids.length === 0) return v;
+        for (const id of ids){
+            const off = id * dims;
+            for(let d = 0; d < dims; d++)v[d] += matrix[off + d];
+        }
+        let norm = 0;
+        for(let d = 0; d < dims; d++){
+            v[d] /= ids.length;
+            norm += v[d] * v[d];
+        }
+        norm = Math.sqrt(norm) + 1e-32;
+        for(let d = 0; d < dims; d++)v[d] /= norm;
+        return v;
+    }
+    return {
+        id: `static:${model}`,
+        dims,
+        embed: async (texts)=>texts.map(one)
+    };
+}
+// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---
+async function apiProvider(model, url, keyEnv) {
+    if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type "api" requires a url');
+    const base = url.replace(/\/+$/, '');
+    const headers = {
+        'content-type': 'application/json'
+    };
+    const key = keyEnv ? process.env[keyEnv] : undefined;
+    if (key) headers.authorization = `Bearer ${key}`;
+    async function post(texts) {
+        const res = await fetch(`${base}/embeddings`, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                model,
+                input: texts
+            })
+        });
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);
+        const body = await res.json();
+        return body.data.map((d)=>Float32Array.from(d.embedding));
+    }
+    const dims = (await post([
+        'dimension probe'
+    ]))[0].length;
+    return {
+        id: `api:${base}:${model}`,
+        dims,
+        embed: post
+    };
+}
+const providers = new Map();
+function getProvider(cfg) {
+    var _e_url;
+    const e = embedConfig(cfg);
+    if (!e) throw new SenseError('EMBED_DISABLED', 'semantic expansion needs features.embed in sense.config.json (e.g. "features": { "embed": true })');
+    const sig = `${e.type}:${e.model}:${(_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : ''}`;
+    let p = providers.get(sig);
+    if (!p) {
+        p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);
+        providers.set(sig, p);
+    }
+    return p;
+}
+// Slice + re-normalize (Matryoshka); optionally round through int8 storage.
+function toStore(full, dims, int8) {
+    const v = new Float32Array(dims);
+    let norm = 0;
+    for(let d = 0; d < dims; d++)norm += full[d] * full[d];
+    norm = Math.sqrt(norm) + 1e-32;
+    for(let d = 0; d < dims; d++)v[d] = full[d] / norm;
+    if (!int8) return {
+        v,
+        scale: 1
+    };
+    let max = 0;
+    for(let d = 0; d < dims; d++)max = Math.max(max, Math.abs(v[d]));
+    return {
+        v,
+        scale: max / 127 || 1
+    };
+}
+// Embed rows whose vector is NULL, re-deriving chunk text from the files through the
+// same parse + chunker that stored the rows.
+export async function embedPending(db, cfg, baseDir) {
+    const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table
+    const dirty = db.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk').all();
+    if (dirty.length === 0) return;
+    const storeDims = Math.min(STORE_DIMS, provider.dims);
+    const byPath = new Map();
+    for (const row of dirty){
+        var _byPath_get;
+        const list = (_byPath_get = byPath.get(row.path)) !== null && _byPath_get !== void 0 ? _byPath_get : [];
+        list.push(row.chunk);
+        byPath.set(row.path, list);
+    }
+    const jobs = [];
+    for (const [path, chunkIdxs] of byPath){
+        let chunks;
+        try {
+            chunks = parseFile({
+                relPath: path,
+                absPath: join(baseDir, path),
+                mtimeMs: 0,
+                size: 0
+            }, [
+                embed
+            ]).doc.extracted.embed;
+        } catch  {
+            continue; // vanished since reconcile; the next reconcile removes its rows
+        }
+        for (const idx of chunkIdxs)if (chunks[idx]) jobs.push({
+            path,
+            chunk: idx,
+            text: chunks[idx].text
+        });
+    }
+    const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE "path" = ? AND chunk = ?');
+    for(let i = 0; i < jobs.length; i += BATCH){
+        const batch = jobs.slice(i, i + BATCH);
+        const vectors = await provider.embed(batch.map((j)=>j.text));
+        db.exec('BEGIN');
+        try {
+            batch.forEach((job, j)=>{
+                const { v, scale } = toStore(vectors[j], storeDims, true);
+                const q = new Int8Array(storeDims);
+                for(let d = 0; d < storeDims; d++)q[d] = Math.round(v[d] / scale);
+                update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);
+            });
+            db.exec('COMMIT');
+        } catch (err) {
+            db.exec('ROLLBACK');
+            throw err;
+        }
+    }
+}
+// Top candidates by cosine for a semantic find: best chunk per file, its line range
+// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped
+// before embedding.
+export async function semanticCandidates(db, cfg, terms, fetch1) {
+    var _terms_match;
+    const baseDir = cfg.baseDir;
+    if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');
+    await embedPending(db, cfg, baseDir);
+    const provider = await getProvider(cfg);
+    const storeDims = Math.min(STORE_DIMS, provider.dims);
+    const text = ((_terms_match = terms.match(/[\p{L}\p{N}]+/gu)) !== null && _terms_match !== void 0 ? _terms_match : []).filter((t)=>![
+            'AND',
+            'OR',
+            'NOT',
+            'NEAR'
+        ].includes(t)).join(' ');
+    const { v: qv } = toStore((await provider.embed([
+        text
+    ]))[0], storeDims, false);
+    const rows = db.prepare('SELECT "path", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
+    const best = new Map();
+    for (const row of rows){
+        const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));
+        let dot = 0;
+        for(let d = 0; d < q.length; d++)dot += q[d] * qv[d];
+        const score = dot * row.scale;
+        const existing = best.get(row.path);
+        if (!existing || score > existing.score) best.set(row.path, {
+            score,
+            lines: `L${row.start_line}-${row.end_line}`
+        });
+    }
+    return [
+        ...best.entries()
+    ].sort((a, b)=>b[1].score - a[1].score).slice(0, fetch1).map(([path, b])=>({
+            path,
+            lines: b.lines
+        }));
+}

package/dist/esm/features/embed.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic expansion needs features.embed in sense.config.json (e.g. \"features\": { \"embed\": true })');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0 }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n  }\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines }));\n}\n"],"names":["existsSync","mkdirSync","readFileSync","renameSync","writeFileSync","homedir","join","embedConfig","SenseError","parseFile","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","chunks","forEach","start","end","text","slice","trim","startLine","endLine","embed","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","fetchToFile","url","dest","fetch","then","res","ok","status","Buffer","from","arrayBuffer","staticProvider","model","tokenizerJson","dir","replace","recursive","file","console","error","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","Tokenizer","tok","unkId","vocab","unk_token","one","ids","encode","add_special_tokens","id","v","off","d","norm","Math","sqrt","texts","map","apiProvider","keyEnv","base","headers","key","process","env","undefined","authorization","post","method","body","stringify","input","json","data","embedding","providers","Map","getProvider","cfg","e","sig","type","p","get","set","toStore","full","int8","scale","max","abs","embedPending","baseDir","provider","dirty","all","storeDims","min","byPath","row","list","chunk","jobs","chunkIdxs","relPath","absPath","mtimeMs","size","doc","update","batch","vectors","j","job","q","Int8Array","round","err","semanticCandidates","terms","match","t","includes","qv","rows","best","vector","byteLength","dot","score","existing","start_line","end_line","sort","a","b"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,EAAEC,YAAY,EAAEC,UAAU,EAAEC,aAAa,QAAQ,UAAU;AACzF,SAASC,OAAO,QAAQ,UAAU;AAClC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,WAAW,QAAQ,eAAe;AAC3C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,SAAS,QAAQ,aAAa;AAGvC,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,MAAMC,aAAa;AACnB,MAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,MAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,MAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,MAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;WAAMA;KAAO,GAAGA;IAC5E,MAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAAStB,IAAI,CAAC;IACrE,MAAMuB,SAAkB,EAAE;IAC1BN,OAAOO,OAAO,CAAC,CAACC,OAAOZ;QACrB,MAAMa,MAAMb,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,MAAMa,OAAOlB,MACVmB,KAAK,CAACH,QAAQ,GAAGC,KACjB1B,IAAI,CAAC,MACL6B,IAAI;QACP,IAAIF,KAAKb,MAAM,GAAG,GAAGS,OAAOP,IAAI,CAAC;YAAEc,WAAWL;YAAOM,SAASL;YAAKC,MAAMT,SAAS,GAAGA,OAAO,EAAE,EAAES,MAAM,GAAGA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEA,OAAO,MAAMS,QAAiB;IAC5BC,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAC1K;IACAC,SAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACAG,OAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,MAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBpB,OAAO,CAAC,CAACsB,GAAGC,MAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEhB,SAAS,EAAEgB,EAAEf,OAAO;IACzF;AACF,EAAE;AAEF,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASiB,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,OAAOC;QAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIpD,WAAW,eAAe,CAAC,uBAAuB,EAAE+C,IAAI,SAAS,EAAEI,IAAIE,MAAM,EAAE;QACtGzD,cAAc,GAAGoD,KAAK,KAAK,CAAC,EAAEM,OAAOC,IAAI,CAAC,MAAMJ,IAAIK,WAAW;QAC/D7D,WAAW,GAAGqD,KAAK,KAAK,CAAC,EAAEA;IAC7B;AACF;AAEA,eAAeS,eAAeC,KAAa;;QA2BEC,sBAA7BA,4BAAAA;IA1Bd,IAAIC,MAAMF;IACV,IAAI,CAAClE,WAAWM,KAAK8D,KAAK,uBAAuB;QAC/CA,MAAM9D,KAAKD,WAAW,UAAU,eAAe,UAAU6D,MAAMG,OAAO,CAAC,OAAO;QAC9EpE,UAAUmE,KAAK;YAAEE,WAAW;QAAK;QACjC,KAAK,MAAMC,QAAQ;YAAC;YAAqB;SAAiB,CAAE;YAC1D,IAAI,CAACvE,WAAWM,KAAK8D,KAAKG,QAAQ;gBAChCC,QAAQC,KAAK,CAAC,CAAC,SAAS,EAAEP,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEH,IAAI,0BAA0B,CAAC;gBAC/E,MAAMd,YAAY,CAAC,uBAAuB,EAAEY,MAAM,cAAc,EAAEK,MAAM,EAAEjE,KAAK8D,KAAKG;YACtF;QACF;IACF;IAEA,MAAM1D,MAAMX,aAAaI,KAAK8D,KAAK;IACnC,MAAMM,YAAYC,OAAO9D,IAAI+D,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAAClE,IAAImE,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAI/E,WAAW,eAAe,GAAG0D,MAAM,oCAAoC,CAAC;IAC1H,MAAMsB,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAY9E,IAAI+E,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAalF,IAAImF,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAalF,IAAImF,MAAM,CAAC9D,KAAK,CAACyD,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAMtB,gBAAgBW,KAAKC,KAAK,CAAC7E,aAAaI,KAAK8D,KAAK,mBAAmB;IAC3E,oFAAoF;IACpF,MAAM,EAAE6B,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC;IACnC,MAAMC,MAAM,IAAID,UAAU9B,eAAe,CAAC;IAC1C,MAAMgC,iBAAQhC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBiC,KAAK,cAA1BjC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBkC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIrE,IAAY;QACvB,MAAMsE,MAAM,AAACL,IAAIM,MAAM,CAACvE,MAAM;YAAEwE,oBAAoB;QAAM,GAAGF,GAAG,CAAc5E,MAAM,CAAC,CAAC+E,KAAOA,OAAOP;QACpG,MAAMQ,IAAI,IAAIZ,aAAaN;QAC3B,IAAIc,IAAInF,MAAM,KAAK,GAAG,OAAOuF;QAC7B,KAAK,MAAMD,MAAMH,IAAK;YACpB,MAAMK,MAAMF,KAAKjB;YACjB,IAAK,IAAIoB,IAAI,GAAGA,IAAIpB,MAAMoB,IAAKF,CAAC,CAACE,EAAE,IAAIf,MAAM,CAACc,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIpB,MAAMoB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIN,IAAInF,MAAM;YAClB0F,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIpB,MAAMoB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,OAAO;QAAED,IAAI,CAAC,OAAO,EAAExC,OAAO;QAAEuB;QAAMnD,OAAO,OAAO2E,QAAUA,MAAMC,GAAG,CAACZ;IAAK;AAC/E;AAEA,gFAAgF;AAEhF,eAAea,YAAYjD,KAAa,EAAEX,GAAuB,EAAE6D,MAA0B;IAC3F,IAAI,CAAC7D,KAAK,MAAM,IAAI/C,WAAW,eAAe;IAC9C,MAAM6G,OAAO9D,IAAIc,OAAO,CAAC,QAAQ;IACjC,MAAMiD,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASI,QAAQC,GAAG,CAACL,OAAO,GAAGM;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKX,KAAe;QACjC,MAAMtD,MAAM,MAAMF,MAAM,GAAG4D,KAAK,WAAW,CAAC,EAAE;YAAEQ,QAAQ;YAAQP;YAASQ,MAAMhD,KAAKiD,SAAS,CAAC;gBAAE7D;gBAAO8D,OAAOf;YAAM;QAAG;QACvH,IAAI,CAACtD,IAAIC,EAAE,EAAE,MAAM,IAAIpD,WAAW,eAAe,GAAG6G,KAAK,oBAAoB,EAAE1D,IAAIE,MAAM,EAAE;QAC3F,MAAMiE,OAAQ,MAAMnE,IAAIsE,IAAI;QAC5B,OAAOH,KAAKI,IAAI,CAAChB,GAAG,CAAC,CAACL,IAAMd,aAAahC,IAAI,CAAC8C,EAAEsB,SAAS;IAC3D;IAEA,MAAM1C,OAAO,AAAC,CAAA,MAAMmC,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACxG,MAAM;IACxD,OAAO;QAAEsF,IAAI,CAAC,IAAI,EAAEW,KAAK,CAAC,EAAEnD,OAAO;QAAEuB;QAAMnD,OAAOsF;IAAK;AACzD;AAEA,MAAMQ,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,MAAMA,IAAIjI,YAAYgI;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIhI,WAAW,kBAAkB;IAC/C,MAAMiI,MAAM,GAAGD,EAAEE,IAAI,CAAC,CAAC,EAAEF,EAAEtE,KAAK,CAAC,CAAC,GAAEsE,SAAAA,EAAEjF,GAAG,cAALiF,oBAAAA,SAAS,IAAI;IACjD,IAAIG,IAAIP,UAAUQ,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIH,EAAEE,IAAI,KAAK,QAAQvB,YAAYqB,EAAEtE,KAAK,EAAEsE,EAAEjF,GAAG,EAAEiF,EAAEjB,GAAG,IAAItD,eAAeuE,EAAEtE,KAAK;QAClFkE,UAAUS,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAEtD,IAAY,EAAEuD,IAAa;IAC9D,MAAMrC,IAAI,IAAIZ,aAAaN;IAC3B,IAAIqB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIpB,MAAMoB,IAAKC,QAAQiC,IAAI,CAAClC,EAAE,GAAGkC,IAAI,CAAClC,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,IAAI,GAAGA,IAAIpB,MAAMoB,IAAKF,CAAC,CAACE,EAAE,GAAGkC,IAAI,CAAClC,EAAE,GAAGC;IAChD,IAAI,CAACkC,MAAM,OAAO;QAAErC;QAAGsC,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIrC,IAAI,GAAGA,IAAIpB,MAAMoB,IAAKqC,MAAMnC,KAAKmC,GAAG,CAACA,KAAKnC,KAAKoC,GAAG,CAACxC,CAAC,CAACE,EAAE;IAChE,OAAO;QAAEF;QAAGsC,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAa3G,EAAgB,EAAE8F,GAAW,EAAEc,OAAe;IAC/E,MAAMC,WAAW,MAAMhB,YAAYC,MAAM,kDAAkD;IAC3F,MAAMgB,QAAQ9G,GAAGM,OAAO,CAAC,oFAAoFyG,GAAG;IAChH,IAAID,MAAMnI,MAAM,KAAK,GAAG;IACxB,MAAMqI,YAAY1C,KAAK2C,GAAG,CAAChJ,YAAY4I,SAAS7D,IAAI;IAEpD,MAAMkE,SAAS,IAAItB;IACnB,KAAK,MAAMuB,OAAOL,MAAO;YACVI;QAAb,MAAME,QAAOF,cAAAA,OAAOf,GAAG,CAACgB,IAAI9G,IAAI,eAAnB6G,yBAAAA,cAAwB,EAAE;QACvCE,KAAKvI,IAAI,CAACsI,IAAIE,KAAK;QACnBH,OAAOd,GAAG,CAACe,IAAI9G,IAAI,EAAE+G;IACvB;IAEA,MAAME,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACjH,MAAMkH,UAAU,IAAIL,OAAQ;QACtC,IAAI9H;QACJ,IAAI;YACFA,SAASpB,UAAU;gBAAEwJ,SAASnH;gBAAMoH,SAAS5J,KAAK+I,SAASvG;gBAAOqH,SAAS;gBAAGC,MAAM;YAAE,GAAG;gBAAC9H;aAAM,EAAE+H,GAAG,CAACnH,SAAS,CAACZ,KAAK;QACvH,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMe,OAAO2G,UAAW,IAAInI,MAAM,CAACwB,IAAI,EAAE0G,KAAKzI,IAAI,CAAC;YAAEwB;YAAMgH,OAAOzG;YAAKpB,MAAMJ,MAAM,CAACwB,IAAI,CAACpB,IAAI;QAAC;IACrG;IAEA,MAAMqI,SAAS7H,GAAGM,OAAO,CAAC;IAC1B,IAAK,IAAI5B,IAAI,GAAGA,IAAI4I,KAAK3I,MAAM,EAAED,KAAKR,MAAO;QAC3C,MAAM4J,QAAQR,KAAK7H,KAAK,CAACf,GAAGA,IAAIR;QAChC,MAAM6J,UAAU,MAAMlB,SAAShH,KAAK,CAACiI,MAAMrD,GAAG,CAAC,CAACuD,IAAMA,EAAExI,IAAI;QAC5DQ,GAAGC,IAAI,CAAC;QACR,IAAI;YACF6H,MAAMzI,OAAO,CAAC,CAAC4I,KAAKD;gBAClB,MAAM,EAAE9D,CAAC,EAAEsC,KAAK,EAAE,GAAGH,QAAQ0B,OAAO,CAACC,EAAE,EAAEhB,WAAW;gBACpD,MAAMkB,IAAI,IAAIC,UAAUnB;gBACxB,IAAK,IAAI5C,IAAI,GAAGA,IAAI4C,WAAW5C,IAAK8D,CAAC,CAAC9D,EAAE,GAAGE,KAAK8D,KAAK,CAAClE,CAAC,CAACE,EAAE,GAAGoC;gBAC7DqB,OAAOtH,GAAG,CAACiG,OAAOnF,OAAOC,IAAI,CAAC4G,EAAE3E,MAAM,GAAG0E,IAAI5H,IAAI,EAAE4H,IAAIZ,KAAK;YAC9D;YACArH,GAAGC,IAAI,CAAC;QACV,EAAE,OAAOoI,KAAK;YACZrI,GAAGC,IAAI,CAAC;YACR,MAAMoI;QACR;IACF;AACF;AAEA,oFAAoF;AACpF,wFAAwF;AACxF,oBAAoB;AACpB,OAAO,eAAeC,mBAAmBtI,EAAgB,EAAE8F,GAAW,EAAEyC,KAAa,EAAEvH,MAAa;QAOpFuH;IANd,MAAM3B,UAAU,AAACd,IAAgCc,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAI7I,WAAW,eAAe;IAClD,MAAM4I,aAAa3G,IAAI8F,KAAKc;IAE5B,MAAMC,WAAW,MAAMhB,YAAYC;IACnC,MAAMkB,YAAY1C,KAAK2C,GAAG,CAAChJ,YAAY4I,SAAS7D,IAAI;IACpD,MAAMxD,OAAO,EAAC+I,eAAAA,MAAMC,KAAK,CAAC,gCAAZD,0BAAAA,eAAkC,EAAE,EAAErJ,MAAM,CAAC,CAACuJ,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAI5K,IAAI,CAAC;IAClH,MAAM,EAAEqG,GAAGyE,EAAE,EAAE,GAAGtC,QAAQ,AAAC,CAAA,MAAMQ,SAAShH,KAAK,CAAC;QAACL;KAAK,CAAA,CAAE,CAAC,EAAE,EAAEwH,WAAW;IAExE,MAAM4B,OAAO5I,GAAGM,OAAO,CAAC,+FAA+FyG,GAAG;IAQ1H,MAAM8B,OAAO,IAAIjD;IACjB,KAAK,MAAMuB,OAAOyB,KAAM;QACtB,MAAMV,IAAI,IAAIC,UAAUhB,IAAI2B,MAAM,CAACvF,MAAM,EAAE4D,IAAI2B,MAAM,CAAC3F,UAAU,EAAEmB,KAAK2C,GAAG,CAACD,WAAWG,IAAI2B,MAAM,CAACC,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAI5E,IAAI,GAAGA,IAAI8D,EAAEvJ,MAAM,EAAEyF,IAAK4E,OAAOd,CAAC,CAAC9D,EAAE,GAAGuE,EAAE,CAACvE,EAAE;QACtD,MAAM6E,QAAQD,MAAM7B,IAAIX,KAAK;QAC7B,MAAM0C,WAAWL,KAAK1C,GAAG,CAACgB,IAAI9G,IAAI;QAClC,IAAI,CAAC6I,YAAYD,QAAQC,SAASD,KAAK,EAAEJ,KAAKzC,GAAG,CAACe,IAAI9G,IAAI,EAAE;YAAE4I;YAAO3K,OAAO,CAAC,CAAC,EAAE6I,IAAIgC,UAAU,CAAC,CAAC,EAAEhC,IAAIiC,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIP,KAAKlG,OAAO;KAAG,CACvB0G,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACN,KAAK,GAAGK,CAAC,CAAC,EAAE,CAACL,KAAK,EACtCxJ,KAAK,CAAC,GAAGuB,QACTyD,GAAG,CAAC,CAAC,CAACpE,MAAMkJ,EAAE,GAAM,CAAA;YAAElJ;YAAM/B,OAAOiL,EAAEjL,KAAK;QAAC,CAAA;AAChD"}

package/dist/esm/features/index.js

@@ -1,4 +1,5 @@
 import { featureEnabled } from '../config.js';
+import { embed } from './embed.js';
 import { links } from './links.js';
 import { rank } from './rank.js';
 import { sections } from './sections.js';
@@ -6,7 +7,8 @@ import { sections } from './sections.js';
 export const FEATURES = [
     links,
     sections,
-    rank
+    rank,
+    embed
 ];
 export function activeFeatures(cfg) {
     return FEATURES.filter((feature)=>featureEnabled(cfg, feature.name));

package/dist/esm/features/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/index.ts"],"sourcesContent":["import type { Config } from '../config.ts';\nimport { featureEnabled } from '../config.ts';\nimport { links } from './links.ts';\nimport { rank } from './rank.ts';\nimport { sections } from './sections.ts';\nimport type { Feature } from './types.ts';\n\n// Registry order matters: rank reads the links table in afterReconcile.\nexport const FEATURES: Feature[] = [links, sections, rank];\n\nexport function activeFeatures(cfg: Config): Feature[] {\n  return FEATURES.filter((feature) => featureEnabled(cfg, feature.name));\n}\n\nexport { linkEdges } from './links.ts';\nexport type { Section } from './sections.ts';\nexport type { Feature } from './types.ts';\n"],"names":["featureEnabled","links","rank","sections","FEATURES","activeFeatures","cfg","filter","feature","name","linkEdges"],"mappings":"AACA,SAASA,cAAc,QAAQ,eAAe;AAC9C,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,QAAQ,QAAQ,gBAAgB;AAGzC,wEAAwE;AACxE,OAAO,MAAMC,WAAsB;IAACH;IAAOE;IAAUD;CAAK,CAAC;AAE3D,OAAO,SAASG,eAAeC,GAAW;IACxC,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAYR,eAAeM,KAAKE,QAAQC,IAAI;AACtE;AAEA,SAASC,SAAS,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/index.ts"],"sourcesContent":["import type { Config } from '../config.ts';\nimport { featureEnabled } from '../config.ts';\nimport { embed } from './embed.ts';\nimport { links } from './links.ts';\nimport { rank } from './rank.ts';\nimport { sections } from './sections.ts';\nimport type { Feature } from './types.ts';\n\n// Registry order matters: rank reads the links table in afterReconcile.\nexport const FEATURES: Feature[] = [links, sections, rank, embed];\n\nexport function activeFeatures(cfg: Config): Feature[] {\n  return FEATURES.filter((feature) => featureEnabled(cfg, feature.name));\n}\n\nexport { linkEdges } from './links.ts';\nexport type { Section } from './sections.ts';\nexport type { Feature } from './types.ts';\n"],"names":["featureEnabled","embed","links","rank","sections","FEATURES","activeFeatures","cfg","filter","feature","name","linkEdges"],"mappings":"AACA,SAASA,cAAc,QAAQ,eAAe;AAC9C,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,QAAQ,QAAQ,gBAAgB;AAGzC,wEAAwE;AACxE,OAAO,MAAMC,WAAsB;IAACH;IAAOE;IAAUD;IAAMF;CAAM,CAAC;AAElE,OAAO,SAASK,eAAeC,GAAW;IACxC,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAYT,eAAeO,KAAKE,QAAQC,IAAI;AACtE;AAEA,SAASC,SAAS,QAAQ,aAAa"}

package/dist/esm/features/types.d.ts

@@ -4,7 +4,10 @@ import type { FileStat } from '../scan.js';
 export interface Feature {
     name: FeatureName;
     schema(db: DatabaseSync): void;
-    extract?(raw: string, body: string): unknown;
+    extract?(raw: string, body: string, search?: {
+        title: string;
+        summary: string;
+    }): unknown;
     remove?(db: DatabaseSync, path: string): void;
     store?(db: DatabaseSync, path: string, extracted: unknown): void;
     afterReconcile?(db: DatabaseSync, files: FileStat[]): void;

package/dist/esm/features/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/types.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { FeatureName } from '../config.ts';\nimport type { FileStat } from '../scan.ts';\n\n// A feature owns its schema, per-file extraction/storage, and any whole-tree pass.\n// Adding a feature = one module here + one entry in the registry (index.ts).\nexport interface Feature {\n  name: FeatureName;\n  schema(db: DatabaseSync): void;\n  // Pure, per file: raw is the full file, body the prose after frontmatter.\n  extract?(raw: string, body: string): unknown;\n  remove?(db: DatabaseSync, path: string): void;\n  store?(db: DatabaseSync, path: string, extracted: unknown): void;\n  // After all rows are current, inside the reconcile transaction (resolution, rank).\n  afterReconcile?(db: DatabaseSync, files: FileStat[]): void;\n}\n"],"names":[],"mappings":"AAIA,mFAAmF;AACnF,6EAA6E;AAC7E,WASC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/types.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { FeatureName } from '../config.ts';\nimport type { FileStat } from '../scan.ts';\n\n// A feature owns its schema, per-file extraction/storage, and any whole-tree pass.\n// Adding a feature = one module here + one entry in the registry (index.ts).\nexport interface Feature {\n  name: FeatureName;\n  schema(db: DatabaseSync): void;\n  // Pure, per file: raw is the full file, body the prose after frontmatter,\n  // search the normalized title/summary strings.\n  extract?(raw: string, body: string, search?: { title: string; summary: string }): unknown;\n  remove?(db: DatabaseSync, path: string): void;\n  store?(db: DatabaseSync, path: string, extracted: unknown): void;\n  // After all rows are current, inside the reconcile transaction (resolution, rank).\n  afterReconcile?(db: DatabaseSync, files: FileStat[]): void;\n}\n"],"names":[],"mappings":"AAIA,mFAAmF;AACnF,6EAA6E;AAC7E,WAUC"}

package/dist/esm/index.d.ts

@@ -1,18 +1,11 @@
-export type { Config, FeatureName, ResolvedConfig } from './config.js';
-export { CONFIG_FILENAME, enabledFeatures, FEATURE_NAMES, featureEnabled, findConfigPath, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
+export type { Config, EmbedConfig, FeatureName, ResolvedConfig } from './config.js';
+export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export type { OpenResult } from './db.js';
-export { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.js';
+export { open, rebuild } from './db.js';
 export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
-export { activeFeatures, FEATURES, linkEdges } from './features/index.js';
-export type { Section } from './features/sections.js';
-export type { Feature } from './features/types.js';
-export type { Edge } from './graph.js';
-export { pagerank, personalizedRank } from './graph.js';
 export type { Row } from './output.js';
 export { printRows } from './output.js';
-export type { FileStat, ParsedDoc } from './scan.js';
-export { listFiles, parseFile } from './scan.js';
 export type { FindOptions, Peek, TreeMap } from './verbs.js';
 export { find, mapTree, peek } from './verbs.js';
 export type { WatchEvent, WatchOptions } from './watch.js';

package/dist/esm/index.js

@@ -1,10 +1,8 @@
-// Public library API.
-export { CONFIG_FILENAME, enabledFeatures, FEATURE_NAMES, featureEnabled, findConfigPath, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
-export { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.js';
+// Public library API. Deliberately small: every export is a stability promise;
+// internals (feature registry, graph, scan, meta) stay module-private.
+export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
+export { open, rebuild } from './db.js';
 export { SenseError } from './errors.js';
-export { activeFeatures, FEATURES, linkEdges } from './features/index.js';
-export { pagerank, personalizedRank } from './graph.js';
 export { printRows } from './output.js';
-export { listFiles, parseFile } from './scan.js';
 export { find, mapTree, peek } from './verbs.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API.\n\nexport type { Config, FeatureName, ResolvedConfig } from './config.ts';\nexport { CONFIG_FILENAME, enabledFeatures, FEATURE_NAMES, featureEnabled, findConfigPath, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\n\nexport type { OpenResult } from './db.ts';\nexport { DB_FILENAME, docCount, getMeta, open, rebuild, reconcile, setMeta } from './db.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport { activeFeatures, FEATURES, linkEdges } from './features/index.ts';\nexport type { Section } from './features/sections.ts';\nexport type { Feature } from './features/types.ts';\n\nexport type { Edge } from './graph.ts';\nexport { pagerank, personalizedRank } from './graph.ts';\n\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { FileStat, ParsedDoc } from './scan.ts';\nexport { listFiles, parseFile } from './scan.ts';\n\nexport type { FindOptions, Peek, TreeMap } from './verbs.ts';\nexport { find, mapTree, peek } from './verbs.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CONFIG_FILENAME","enabledFeatures","FEATURE_NAMES","featureEnabled","findConfigPath","initConfig","loadConfig","migrateConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","DB_FILENAME","docCount","getMeta","open","rebuild","reconcile","setMeta","SenseError","activeFeatures","FEATURES","linkEdges","pagerank","personalizedRank","printRows","listFiles","parseFile","find","mapTree","peek","runWatch"],"mappings":"AAAA,sBAAsB;AAGtB,SAASA,eAAe,EAAEC,eAAe,EAAEC,aAAa,EAAEC,cAAc,EAAEC,cAAc,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAG1L,SAASC,WAAW,EAAEC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,OAAO,EAAEC,SAAS,EAAEC,OAAO,QAAQ,UAAU;AAE5F,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,EAAEC,QAAQ,EAAEC,SAAS,QAAQ,sBAAsB;AAK1E,SAASC,QAAQ,EAAEC,gBAAgB,QAAQ,aAAa;AAGxD,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAGjD,SAASC,IAAI,EAAEC,OAAO,EAAEC,IAAI,QAAQ,aAAa;AAGjD,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Config, EmbedConfig, FeatureName, ResolvedConfig } from './config.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\n\nexport type { OpenResult } from './db.ts';\nexport { open, rebuild } from './db.ts';\n\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\n\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { FindOptions, Peek, TreeMap } from './verbs.ts';\nexport { find, mapTree, peek } from './verbs.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","open","rebuild","SenseError","printRows","find","mapTree","peek","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAG1H,SAASC,IAAI,EAAEC,OAAO,QAAQ,UAAU;AAGxC,SAASC,UAAU,QAAQ,cAAc;AAGzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,IAAI,EAAEC,OAAO,EAAEC,IAAI,QAAQ,aAAa;AAGjD,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/output.d.ts

@@ -7,6 +7,10 @@ export declare function renderMap(result: {
     };
     fields: Row[];
     fieldsTotal: number;
+    features: {
+        on: string[];
+        off: string[];
+    };
     hubs: Row[];
     recent: Row[];
 }): string;
@@ -21,4 +25,5 @@ export declare function renderPeek(result: {
     outboundTotal: number;
     backlinksTotal: number;
     unresolvedTotal: number;
+    off: string[];
 }): string;

package/dist/esm/output.js

@@ -26,13 +26,19 @@ function render(rows, format) {
     ].join('\n');
 }
 // Text renderers for the layer verbs; cli.ts prints what these return.
+function featuresLine(features) {
+    const off = features.off.length > 0 ? ` · off: ${features.off.map((name)=>`${name} (features.${name})`).join(', ')}` : '';
+    return `features: ${features.on.join(', ')}${off}`;
+}
 export function renderMap(result) {
     const parts = [
-        `docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)\n`,
+        `docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`,
+        `${featuresLine(result.features)}\n`,
         render(result.fields, 'table')
     ];
     if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);
     if (result.hubs.length > 0) parts.push('\nhubs (by link rank):', render(result.hubs, 'table'));
+    else if (result.features.off.includes('rank')) parts.push('\nhubs: off (features.rank)');
     parts.push('\nrecent:', render(result.recent, 'table'));
     return parts.join('\n');
 }
@@ -44,6 +50,10 @@ export function renderPeek(result) {
     if (result.sections.length > 0) {
         lines.push('', 'sections:');
         for (const s of result.sections)lines.push(`  ${'#'.repeat(s.level)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);
+    } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');
+    if (result.off.includes('links')) {
+        lines.push('', 'links: off (features.links)');
+        return lines.join('\n');
     }
     const linkLine = (label, shown, total)=>{
         if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);

package/dist/esm/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = render(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\nfunction render(rows: Row[], format: 'table' | 'json'): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const widths = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)\\n`, render(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', render(result.hubs, 'table'));\n  parts.push('\\nrecent:', render(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","render","console","log","length","warn","Math","round","JSON","stringify","columns","Object","keys","cells","map","row","col","widths","i","max","formatRow","values","padEnd","join","w","repeat","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","linkLine","label","shown","total","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,OAAOH,MAAMC;IAC9BG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,SAASH,OAAOH,IAAW,EAAEC,MAAwB;IACnD,IAAIA,WAAW,QAAQ,OAAOS,KAAKC,SAAS,CAACX,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMM,UAAUC,OAAOC,IAAI,CAACd,IAAI,CAAC,EAAE;IACnC,MAAMe,QAAQf,KAAKgB,GAAG,CAAC,CAACC,MAAQL,QAAQI,GAAG,CAAC,CAACE,MAAQvB,KAAKsB,GAAG,CAACC,IAAI;IAClE,MAAMC,SAASP,QAAQI,GAAG,CAAC,CAACE,KAAKE,IAAMZ,KAAKa,GAAG,CAACH,IAAIZ,MAAM,KAAKS,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAACd,MAAM;IAE/F,MAAMgB,YAAY,CAACC,SAAqBA,OAAOP,GAAG,CAAC,CAACpB,OAAOwB,IAAMxB,MAAM4B,MAAM,CAACL,MAAM,CAACC,EAAE,GAAGK,IAAI,CAAC;IAE/F,OAAO;QAACH,UAAUV;QAAUO,OAAOH,GAAG,CAAC,CAACU,IAAM,IAAIC,MAAM,CAACD,IAAID,IAAI,CAAC;WAAUV,MAAMC,GAAG,CAACM;KAAW,CAACG,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,OAAO,SAASG,UAAUC,MAAkH;IAC1I,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAExB,KAAKC,KAAK,CAACoB,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,MAAM,CAAC;QAAE9B,OAAO0B,OAAOK,MAAM,EAAE;KAAS;IAC3H,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC5B,MAAM,EAAEwB,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC5B,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIuB,OAAOQ,IAAI,CAAC/B,MAAM,GAAG,GAAGwB,MAAMM,IAAI,CAAC,0BAA0BjC,OAAO0B,OAAOQ,IAAI,EAAE;IACrFP,MAAMM,IAAI,CAAC,aAAajC,OAAO0B,OAAOS,MAAM,EAAE;IAC9C,OAAOR,MAAML,IAAI,CAAC;AACpB;AAEA,OAAO,SAASc,WAAWV,MAAkN;IAC3O,MAAMW,QAAQ;QAAC,GAAGX,OAAOY,IAAI,CAAC,IAAI,EAAEZ,OAAOa,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAK/C,MAAM,IAAIiB,OAAO+B,OAAO,CAACf,OAAOgB,WAAW,EAAGL,MAAMJ,IAAI,CAAC,CAAC,EAAE,EAAEO,IAAI,EAAE,EAAE/C,OAAO;IAC9F,IAAIiC,OAAOiB,QAAQ,CAACxC,MAAM,GAAG,GAAG;QAC9BkC,MAAMJ,IAAI,CAAC,IAAI;QACf,KAAK,MAAMW,KAAKlB,OAAOiB,QAAQ,CAAEN,MAAMJ,IAAI,CAAC,CAAC,EAAE,EAAE,IAAIT,MAAM,CAACoB,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;IAChJ;IACA,MAAMU,WAAW,CAACC,OAAeC,OAAiBC;QAChD,IAAIA,QAAQ,GAAGf,MAAMJ,IAAI,CAAC,GAAGiB,MAAM,EAAE,EAAEE,MAAM,GAAG,EAAED,MAAM7B,IAAI,CAAC,QAAQ8B,QAAQD,MAAMhD,MAAM,GAAG,CAAC,GAAG,EAAEiD,QAAQD,MAAMhD,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAIuB,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGlB,MAAMJ,IAAI,CAAC;IAC1FgB,SAAS,aAAavB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;IAC3DJ,SAAS,cAAcvB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;IAChEL,SAAS,aAAavB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC7D,OAAOlB,MAAMf,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = render(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\nfunction render(rows: Row[], format: 'table' | 'json'): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const widths = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nfunction featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, `${featuresLine(result.features)}\\n`, render(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', render(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push('\\nhubs: off (features.rank)');\n  parts.push('\\nrecent:', render(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","render","console","log","length","warn","Math","round","JSON","stringify","columns","Object","keys","cells","map","row","col","widths","i","max","formatRow","values","padEnd","join","w","repeat","featuresLine","features","off","name","on","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","linkLine","label","shown","total","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,OAAOH,MAAMC;IAC9BG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,SAASH,OAAOH,IAAW,EAAEC,MAAwB;IACnD,IAAIA,WAAW,QAAQ,OAAOS,KAAKC,SAAS,CAACX,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMM,UAAUC,OAAOC,IAAI,CAACd,IAAI,CAAC,EAAE;IACnC,MAAMe,QAAQf,KAAKgB,GAAG,CAAC,CAACC,MAAQL,QAAQI,GAAG,CAAC,CAACE,MAAQvB,KAAKsB,GAAG,CAACC,IAAI;IAClE,MAAMC,SAASP,QAAQI,GAAG,CAAC,CAACE,KAAKE,IAAMZ,KAAKa,GAAG,CAACH,IAAIZ,MAAM,KAAKS,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAACd,MAAM;IAE/F,MAAMgB,YAAY,CAACC,SAAqBA,OAAOP,GAAG,CAAC,CAACpB,OAAOwB,IAAMxB,MAAM4B,MAAM,CAACL,MAAM,CAACC,EAAE,GAAGK,IAAI,CAAC;IAE/F,OAAO;QAACH,UAAUV;QAAUO,OAAOH,GAAG,CAAC,CAACU,IAAM,IAAIC,MAAM,CAACD,IAAID,IAAI,CAAC;WAAUV,MAAMC,GAAG,CAACM;KAAW,CAACG,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,SAASG,aAAaC,QAAyC;IAC7D,MAAMC,MAAMD,SAASC,GAAG,CAACxB,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEuB,SAASC,GAAG,CAACd,GAAG,CAAC,CAACe,OAAS,GAAGA,KAAK,WAAW,EAAEA,KAAK,CAAC,CAAC,EAAEN,IAAI,CAAC,OAAO,GAAG;IACzH,OAAO,CAAC,UAAU,EAAEI,SAASG,EAAE,CAACP,IAAI,CAAC,QAAQK,KAAK;AACpD;AAEA,OAAO,SAASG,UAAUC,MAA6J;IACrL,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAE7B,KAAKC,KAAK,CAACyB,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE,GAAGV,aAAaM,OAAOL,QAAQ,EAAE,EAAE,CAAC;QAAE1B,OAAO+B,OAAOK,MAAM,EAAE;KAAS;IAC/J,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACjC,MAAM,EAAE6B,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACjC,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI4B,OAAOQ,IAAI,CAACpC,MAAM,GAAG,GAAG6B,MAAMM,IAAI,CAAC,0BAA0BtC,OAAO+B,OAAOQ,IAAI,EAAE;SAChF,IAAIR,OAAOL,QAAQ,CAACC,GAAG,CAACa,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC;IAC1DN,MAAMM,IAAI,CAAC,aAAatC,OAAO+B,OAAOU,MAAM,EAAE;IAC9C,OAAOT,MAAMV,IAAI,CAAC;AACpB;AAEA,OAAO,SAASoB,WAAWX,MAAiO;IAC1P,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAKrD,MAAM,IAAIiB,OAAOqC,OAAO,CAAChB,OAAOiB,WAAW,EAAGL,MAAML,IAAI,CAAC,CAAC,EAAE,EAAEQ,IAAI,EAAE,EAAErD,OAAO;IAC9F,IAAIsC,OAAOkB,QAAQ,CAAC9C,MAAM,GAAG,GAAG;QAC9BwC,MAAML,IAAI,CAAC,IAAI;QACf,KAAK,MAAMY,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAML,IAAI,CAAC,CAAC,EAAE,EAAE,IAAId,MAAM,CAAC0B,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;IAChJ,OAAO,IAAId,OAAOJ,GAAG,CAACa,QAAQ,CAAC,aAAaG,MAAML,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOJ,GAAG,CAACa,QAAQ,CAAC,UAAU;QAChCG,MAAML,IAAI,CAAC,IAAI;QACf,OAAOK,MAAMrB,IAAI,CAAC;IACpB;IACA,MAAMiC,WAAW,CAACC,OAAeC,OAAiBC;QAChD,IAAIA,QAAQ,GAAGf,MAAML,IAAI,CAAC,GAAGkB,MAAM,EAAE,EAAEE,MAAM,GAAG,EAAED,MAAMnC,IAAI,CAAC,QAAQoC,QAAQD,MAAMtD,MAAM,GAAG,CAAC,GAAG,EAAEuD,QAAQD,MAAMtD,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAI4B,OAAO4B,aAAa,GAAG5B,OAAO6B,eAAe,GAAG7B,OAAO8B,cAAc,GAAG,GAAGlB,MAAML,IAAI,CAAC;IAC1FiB,SAAS,aAAaxB,OAAO+B,QAAQ,EAAE/B,OAAO4B,aAAa;IAC3DJ,SAAS,cAAcxB,OAAOgC,UAAU,EAAEhC,OAAO6B,eAAe;IAChEL,SAAS,aAAaxB,OAAOiC,SAAS,EAAEjC,OAAO8B,cAAc;IAC7D,OAAOlB,MAAMrB,IAAI,CAAC;AACpB"}

package/dist/esm/scan.js

@@ -99,24 +99,25 @@ export function parseFile(file, extractors = []) {
         }
         mapped[key] = mapValue(data[key]);
     }
+    // title/summary are plain YAML strings -- whitespace-collapse only;
+    // the prose gets the full markdown strip.
+    const search = {
+        title: normalizeText(data.title),
+        summary: normalizeText(data.summary),
+        text: stripText(content)
+    };
     return {
         doc: {
             relPath: file.relPath,
             mtimeMs: file.mtimeMs,
             size: file.size,
             data: mapped,
-            search: {
-                // title/summary are plain YAML strings -- whitespace-collapse only;
-                // the prose gets the full markdown strip.
-                title: normalizeText(data.title),
-                summary: normalizeText(data.summary),
-                text: stripText(content)
-            },
+            search,
             extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{
                 var _f_extract;
                 return [
                     f.name,
-                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content)
+                    (_f_extract = f.extract) === null || _f_extract === void 0 ? void 0 : _f_extract.call(f, raw, content, search)
                 ];
             }))
         },