sensemaking 0.4.0 → 0.6.0

package/dist/esm/commands/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/shared.ts"],"sourcesContent":["import type { ResolvedConfig } from '../config.ts';\nimport type { OpenResult } from '../db.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport type { Ctx } from './types.ts';\n\n// Shared open-query-close envelope for commands that touch the tree.\n\nexport function printWarnings(warnings: string[]): void {\n  for (const w of warnings) console.warn(w);\n}\n\nexport function withDb(ctx: Ctx, fn: (db: OpenResult['db'], cfg: ResolvedConfig) => void): void {\n  const cfg = ctx.resolveConfig();\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  try {\n    fn(db, cfg);\n  } finally {\n    db.close();\n  }\n}\n\n// An unbound `?` silently binds NULL, so mismatched param counts fail loudly instead.\nexport function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: 'table' | 'json', label: string): void {\n  const placeholderCount = (sql.match(/\\?/g) ?? []).length;\n  if (params.length !== placeholderCount) {\n    console.error(`${label} expects ${placeholderCount} parameter(s), got ${params.length}`);\n    process.exit(2);\n  }\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  printRows(db.prepare(sql).all(...params) as Row[], format);\n  db.close();\n}\n"],"names":["open","printRows","printWarnings","warnings","w","console","warn","withDb","ctx","fn","cfg","resolveConfig","db","close","runSql","sql","params","format","label","placeholderCount","match","length","error","process","exit","prepare","all"],"mappings":"AAEA,SAASA,IAAI,QAAQ,WAAW;AAEhC,SAASC,SAAS,QAAQ,eAAe;AAGzC,qEAAqE;AAErE,OAAO,SAASC,cAAcC,QAAkB;IAC9C,KAAK,MAAMC,KAAKD,SAAUE,QAAQC,IAAI,CAACF;AACzC;AAEA,OAAO,SAASG,OAAOC,GAAQ,EAAEC,EAAuD;IACtF,MAAMC,MAAMF,IAAIG,aAAa;IAC7B,MAAM,EAAEC,EAAE,EAAET,QAAQ,EAAE,GAAGH,KAAKU;IAC9BR,cAAcC;IACd,IAAI;QACFM,GAAGG,IAAIF;IACT,SAAU;QACRE,GAAGC,KAAK;IACV;AACF;AAEA,sFAAsF;AACtF,OAAO,SAASC,OAAOJ,GAAmB,EAAEK,GAAW,EAAEC,MAAgB,EAAEC,MAAwB,EAAEC,KAAa;QACtFH;IAA1B,MAAMI,mBAAmB,EAACJ,aAAAA,IAAIK,KAAK,CAAC,oBAAVL,wBAAAA,aAAoB,EAAE,EAAEM,MAAM;IACxD,IAAIL,OAAOK,MAAM,KAAKF,kBAAkB;QACtCd,QAAQiB,KAAK,CAAC,GAAGJ,MAAM,SAAS,EAAEC,iBAAiB,mBAAmB,EAAEH,OAAOK,MAAM,EAAE;QACvFE,QAAQC,IAAI,CAAC;IACf;IACA,MAAM,EAAEZ,EAAE,EAAET,QAAQ,EAAE,GAAGH,KAAKU;IAC9BR,cAAcC;IACdF,UAAUW,GAAGa,OAAO,CAACV,KAAKW,GAAG,IAAIV,SAAkBC;IACnDL,GAAGC,KAAK;AACV"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/shared.ts"],"sourcesContent":["import type { ResolvedConfig } from '../config.ts';\nimport type { OpenResult } from '../db.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport type { Ctx } from './types.ts';\n\n// Shared open-query-close envelope for commands that touch the tree.\n\nexport function printWarnings(warnings: string[]): void {\n  for (const w of warnings) console.warn(w);\n}\n\nexport async function withDb(ctx: Ctx, fn: (db: OpenResult['db'], cfg: ResolvedConfig) => void | Promise<void>): Promise<void> {\n  const cfg = ctx.resolveConfig();\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  try {\n    await fn(db, cfg);\n  } finally {\n    db.close();\n  }\n}\n\n// An unbound `?` silently binds NULL, so mismatched param counts fail loudly instead.\nexport function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: 'table' | 'json', label: string): void {\n  const placeholderCount = (sql.match(/\\?/g) ?? []).length;\n  if (params.length !== placeholderCount) {\n    console.error(`${label} expects ${placeholderCount} parameter(s), got ${params.length}`);\n    process.exit(2);\n  }\n  const { db, warnings } = open(cfg);\n  printWarnings(warnings);\n  printRows(db.prepare(sql).all(...params) as Row[], format);\n  db.close();\n}\n"],"names":["open","printRows","printWarnings","warnings","w","console","warn","withDb","ctx","fn","cfg","resolveConfig","db","close","runSql","sql","params","format","label","placeholderCount","match","length","error","process","exit","prepare","all"],"mappings":"AAEA,SAASA,IAAI,QAAQ,WAAW;AAEhC,SAASC,SAAS,QAAQ,eAAe;AAGzC,qEAAqE;AAErE,OAAO,SAASC,cAAcC,QAAkB;IAC9C,KAAK,MAAMC,KAAKD,SAAUE,QAAQC,IAAI,CAACF;AACzC;AAEA,OAAO,eAAeG,OAAOC,GAAQ,EAAEC,EAAuE;IAC5G,MAAMC,MAAMF,IAAIG,aAAa;IAC7B,MAAM,EAAEC,EAAE,EAAET,QAAQ,EAAE,GAAGH,KAAKU;IAC9BR,cAAcC;IACd,IAAI;QACF,MAAMM,GAAGG,IAAIF;IACf,SAAU;QACRE,GAAGC,KAAK;IACV;AACF;AAEA,sFAAsF;AACtF,OAAO,SAASC,OAAOJ,GAAmB,EAAEK,GAAW,EAAEC,MAAgB,EAAEC,MAAwB,EAAEC,KAAa;QACtFH;IAA1B,MAAMI,mBAAmB,EAACJ,aAAAA,IAAIK,KAAK,CAAC,oBAAVL,wBAAAA,aAAoB,EAAE,EAAEM,MAAM;IACxD,IAAIL,OAAOK,MAAM,KAAKF,kBAAkB;QACtCd,QAAQiB,KAAK,CAAC,GAAGJ,MAAM,SAAS,EAAEC,iBAAiB,mBAAmB,EAAEH,OAAOK,MAAM,EAAE;QACvFE,QAAQC,IAAI,CAAC;IACf;IACA,MAAM,EAAEZ,EAAE,EAAET,QAAQ,EAAE,GAAGH,KAAKU;IAC9BR,cAAcC;IACdF,UAAUW,GAAGa,OAAO,CAACV,KAAKW,GAAG,IAAIV,SAAkBC;IACnDL,GAAGC,KAAK;AACV"}

package/dist/esm/commands/status.js

@@ -1,10 +1,17 @@
+import { embedConfig, featureStates } from '../config.js';
 import { docCount, getMeta, open } from '../db.js';
 import { printWarnings } from './shared.js';
 const status = (ctx)=>{
-    const { db, dbPath, warnings } = open(ctx.resolveConfig());
+    const cfg = ctx.resolveConfig();
+    const { db, dbPath, warnings } = open(cfg);
     printWarnings(warnings);
     console.log(`db: ${dbPath}`);
     console.log(`docs: ${docCount(db)}`);
+    const features = featureStates(cfg);
+    const off = features.off.length > 0 ? ` · off: ${features.off.map((name)=>`${name} (features.${name})`).join(', ')}` : '';
+    console.log(`features: ${features.on.join(', ')}${off}`);
+    const e = embedConfig(cfg);
+    if (e) console.log(`embed: ${e.type} ${e.model}`);
     const heartbeat = getMeta(db, 'watch_heartbeat');
     if (heartbeat) console.log(`watcher: last heartbeat ${Math.round((Date.now() - Date.parse(heartbeat)) / 1000)}s ago`);
     else console.log('watcher: no watcher');

package/dist/esm/commands/status.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import { docCount, getMeta, open } from '../db.ts';\nimport { printWarnings } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst status: Command = (ctx) => {\n  const { db, dbPath, warnings } = open(ctx.resolveConfig());\n  printWarnings(warnings);\n  console.log(`db: ${dbPath}`);\n  console.log(`docs: ${docCount(db)}`);\n  const heartbeat = getMeta(db, 'watch_heartbeat');\n  if (heartbeat) console.log(`watcher: last heartbeat ${Math.round((Date.now() - Date.parse(heartbeat)) / 1000)}s ago`);\n  else console.log('watcher: no watcher');\n  db.close();\n};\nexport default status;\n"],"names":["docCount","getMeta","open","printWarnings","status","ctx","db","dbPath","warnings","resolveConfig","console","log","heartbeat","Math","round","Date","now","parse","close"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,OAAO,EAAEC,IAAI,QAAQ,WAAW;AACnD,SAASC,aAAa,QAAQ,cAAc;AAG5C,MAAMC,SAAkB,CAACC;IACvB,MAAM,EAAEC,EAAE,EAAEC,MAAM,EAAEC,QAAQ,EAAE,GAAGN,KAAKG,IAAII,aAAa;IACvDN,cAAcK;IACdE,QAAQC,GAAG,CAAC,CAAC,IAAI,EAAEJ,QAAQ;IAC3BG,QAAQC,GAAG,CAAC,CAAC,MAAM,EAAEX,SAASM,KAAK;IACnC,MAAMM,YAAYX,QAAQK,IAAI;IAC9B,IAAIM,WAAWF,QAAQC,GAAG,CAAC,CAAC,wBAAwB,EAAEE,KAAKC,KAAK,CAAC,AAACC,CAAAA,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL,UAAS,IAAK,MAAM,KAAK,CAAC;SAC/GF,QAAQC,GAAG,CAAC;IACjBL,GAAGY,KAAK;AACV;AACA,eAAed,OAAO"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/status.ts"],"sourcesContent":["import { embedConfig, featureStates } from '../config.ts';\nimport { docCount, getMeta, open } from '../db.ts';\nimport { printWarnings } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst status: Command = (ctx) => {\n  const cfg = ctx.resolveConfig();\n  const { db, dbPath, warnings } = open(cfg);\n  printWarnings(warnings);\n  console.log(`db: ${dbPath}`);\n  console.log(`docs: ${docCount(db)}`);\n  const features = featureStates(cfg);\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  console.log(`features: ${features.on.join(', ')}${off}`);\n  const e = embedConfig(cfg);\n  if (e) console.log(`embed: ${e.type} ${e.model}`);\n  const heartbeat = getMeta(db, 'watch_heartbeat');\n  if (heartbeat) console.log(`watcher: last heartbeat ${Math.round((Date.now() - Date.parse(heartbeat)) / 1000)}s ago`);\n  else console.log('watcher: no watcher');\n  db.close();\n};\nexport default status;\n"],"names":["embedConfig","featureStates","docCount","getMeta","open","printWarnings","status","ctx","cfg","resolveConfig","db","dbPath","warnings","console","log","features","off","length","map","name","join","on","e","type","model","heartbeat","Math","round","Date","now","parse","close"],"mappings":"AAAA,SAASA,WAAW,EAAEC,aAAa,QAAQ,eAAe;AAC1D,SAASC,QAAQ,EAAEC,OAAO,EAAEC,IAAI,QAAQ,WAAW;AACnD,SAASC,aAAa,QAAQ,cAAc;AAG5C,MAAMC,SAAkB,CAACC;IACvB,MAAMC,MAAMD,IAAIE,aAAa;IAC7B,MAAM,EAAEC,EAAE,EAAEC,MAAM,EAAEC,QAAQ,EAAE,GAAGR,KAAKI;IACtCH,cAAcO;IACdC,QAAQC,GAAG,CAAC,CAAC,IAAI,EAAEH,QAAQ;IAC3BE,QAAQC,GAAG,CAAC,CAAC,MAAM,EAAEZ,SAASQ,KAAK;IACnC,MAAMK,WAAWd,cAAcO;IAC/B,MAAMQ,MAAMD,SAASC,GAAG,CAACC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEF,SAASC,GAAG,CAACE,GAAG,CAAC,CAACC,OAAS,GAAGA,KAAK,WAAW,EAAEA,KAAK,CAAC,CAAC,EAAEC,IAAI,CAAC,OAAO,GAAG;IACzHP,QAAQC,GAAG,CAAC,CAAC,UAAU,EAAEC,SAASM,EAAE,CAACD,IAAI,CAAC,QAAQJ,KAAK;IACvD,MAAMM,IAAItB,YAAYQ;IACtB,IAAIc,GAAGT,QAAQC,GAAG,CAAC,CAAC,OAAO,EAAEQ,EAAEC,IAAI,CAAC,CAAC,EAAED,EAAEE,KAAK,EAAE;IAChD,MAAMC,YAAYtB,QAAQO,IAAI;IAC9B,IAAIe,WAAWZ,QAAQC,GAAG,CAAC,CAAC,wBAAwB,EAAEY,KAAKC,KAAK,CAAC,AAACC,CAAAA,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL,UAAS,IAAK,MAAM,KAAK,CAAC;SAC/GZ,QAAQC,GAAG,CAAC;IACjBJ,GAAGqB,KAAK;AACV;AACA,eAAezB,OAAO"}

package/dist/esm/commands/types.d.ts

@@ -8,6 +8,7 @@ export interface Ctx {
         where?: string;
         k?: string;
         force: boolean;
+        semantic: boolean;
     };
     resolveConfig(): ResolvedConfig;
     usageError(message: string): never;

package/dist/esm/commands/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/types.ts"],"sourcesContent":["import type { ResolvedConfig } from '../config.ts';\n\n// Handed to every command; argv parsing stays in cli.ts.\nexport interface Ctx {\n  name: string;\n  rest: string[]; // positionals after the command word\n  format: 'table' | 'json';\n  values: { config?: string; where?: string; k?: string; force: boolean };\n  resolveConfig(): ResolvedConfig;\n  usageError(message: string): never;\n}\n\nexport type Command = (ctx: Ctx) => Promise<void> | void;\n"],"names":[],"mappings":"AAYA,WAAyD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/types.ts"],"sourcesContent":["import type { ResolvedConfig } from '../config.ts';\n\n// Handed to every command; argv parsing stays in cli.ts.\nexport interface Ctx {\n  name: string;\n  rest: string[]; // positionals after the command word\n  format: 'table' | 'json';\n  values: { config?: string; where?: string; k?: string; force: boolean; semantic: boolean };\n  resolveConfig(): ResolvedConfig;\n  usageError(message: string): never;\n}\n\nexport type Command = (ctx: Ctx) => Promise<void> | void;\n"],"names":[],"mappings":"AAYA,WAAyD"}

package/dist/esm/config.d.ts

@@ -1,15 +1,27 @@
 export declare const CONFIG_FILENAME = "sense.config.json";
 export declare const STATE_DIR = ".sense";
 export declare const SUPPORTED_CONFIG_VERSION = 2;
-export declare const FEATURE_NAMES: readonly ["links", "sections", "rank"];
+declare const FEATURE_NAMES: readonly ["links", "sections", "rank", "embed"];
 export type FeatureName = (typeof FEATURE_NAMES)[number];
+export interface EmbedConfig {
+    model?: string;
+    type?: 'static' | 'api';
+    url?: string;
+    key?: string;
+}
+export declare const DEFAULT_EMBED_MODEL = "minishlab/potion-retrieval-32M";
 export interface Config {
     $schema?: string;
     version?: number;
     scan: {
         include: string[];
     };
-    features?: Partial<Record<FeatureName, boolean>>;
+    features?: {
+        links?: boolean;
+        sections?: boolean;
+        rank?: boolean;
+        embed?: boolean | EmbedConfig;
+    };
     queries: Record<string, string>;
 }
 export interface ResolvedConfig extends Config {
@@ -19,6 +31,17 @@ export interface ResolvedConfig extends Config {
 }
 export declare function featureEnabled(cfg: Config, name: FeatureName): boolean;
 export declare function enabledFeatures(cfg: Config): FeatureName[];
+export declare function featureStates(cfg: Config): {
+    on: FeatureName[];
+    off: FeatureName[];
+};
+export declare function embedConfig(cfg: Config): {
+    model: string;
+    type: 'static' | 'api';
+    url?: string;
+    key?: string;
+} | null;
+export declare function featureSignature(cfg: Config): string;
 export declare function migrateConfig(cfg: Config): {
     cfg: Config;
     from: number;
@@ -26,3 +49,4 @@ export declare function migrateConfig(cfg: Config): {
 export declare function initConfig(dir: string): string;
 export declare function findConfigPath(startDir: string): string | null;
 export declare function loadConfig(explicitPath?: string): ResolvedConfig;
+export {};

package/dist/esm/config.js

@@ -6,28 +6,69 @@ export const STATE_DIR = '.sense';
 // Highest sense.config.json `version` this build understands. Older versions auto-migrate on load.
 export const SUPPORTED_CONFIG_VERSION = 2;
 // Each feature owns its tables, parse-time extraction, and reconcile step; verbs degrade when one is off.
-export const FEATURE_NAMES = [
+// links/sections/rank are opt-out (absent = on); embed is opt-in (absent = off) -- most trees don't need vectors.
+const FEATURE_NAMES = [
+    'links',
+    'sections',
+    'rank',
+    'embed'
+];
+const OPT_OUT_NAMES = [
     'links',
     'sections',
     'rank'
 ];
-// Absent block or key means enabled -- features are opt-out. `rank` additionally requires `links`.
+export const DEFAULT_EMBED_MODEL = 'minishlab/potion-retrieval-32M';
+// Opt-out features: absent block or key means enabled. `rank` additionally requires `links`.
+// `embed` is the opposite: enabled only when the config says so.
 export function featureEnabled(cfg, name) {
-    var _cfg_features;
-    const enabled = ((_cfg_features = cfg.features) === null || _cfg_features === void 0 ? void 0 : _cfg_features[name]) !== false;
+    var _cfg_features, _cfg_features1;
+    if (name === 'embed') return Boolean((_cfg_features = cfg.features) === null || _cfg_features === void 0 ? void 0 : _cfg_features.embed);
+    const enabled = ((_cfg_features1 = cfg.features) === null || _cfg_features1 === void 0 ? void 0 : _cfg_features1[name]) !== false;
     if (name === 'rank') return enabled && featureEnabled(cfg, 'links');
     return enabled;
 }
 export function enabledFeatures(cfg) {
     return FEATURE_NAMES.filter((name)=>featureEnabled(cfg, name));
 }
+// Every feature with its current state; verbs surface this so "off" and "empty" stay
+// distinguishable in output.
+export function featureStates(cfg) {
+    return {
+        on: FEATURE_NAMES.filter((name)=>featureEnabled(cfg, name)),
+        off: FEATURE_NAMES.filter((name)=>!featureEnabled(cfg, name))
+    };
+}
+// Resolved embed settings, or null when the feature is off.
+export function embedConfig(cfg) {
+    var _o_model, _o_type;
+    var _cfg_features;
+    const e = (_cfg_features = cfg.features) === null || _cfg_features === void 0 ? void 0 : _cfg_features.embed;
+    if (!e) return null;
+    const o = e === true ? {} : e;
+    return {
+        model: (_o_model = o.model) !== null && _o_model !== void 0 ? _o_model : DEFAULT_EMBED_MODEL,
+        type: (_o_type = o.type) !== null && _o_type !== void 0 ? _o_type : 'static',
+        url: o.url,
+        key: o.key
+    };
+}
+// Cache-key string: embed carries its type + model so a model change rebuilds like a toggle.
+export function featureSignature(cfg) {
+    return enabledFeatures(cfg).map((name)=>{
+        if (name !== 'embed') return name;
+        const e = embedConfig(cfg);
+        return `embed:${e === null || e === void 0 ? void 0 : e.type}:${e === null || e === void 0 ? void 0 : e.model}`;
+    }).join(',');
+}
 // Pure per-version steps; loadConfig chains them from the file's version up to SUPPORTED_CONFIG_VERSION.
 const MIGRATIONS = {
-    // v1 -> v2: features block introduced, everything enabled (matches the old implicit behavior of `links` etc. not existing).
+    // v1 -> v2: features block introduced, opt-out features enabled (matches the old implicit
+    // behavior of `links` etc. not existing). embed stays absent -- opt-in.
     1: (cfg)=>({
             ...cfg,
             version: 2,
-            features: Object.fromEntries(FEATURE_NAMES.map((name)=>[
+            features: Object.fromEntries(OPT_OUT_NAMES.map((name)=>[
                     name,
                     true
                 ]))
@@ -56,7 +97,7 @@ function starterConfig() {
                 '**/*.md'
             ]
         },
-        features: Object.fromEntries(FEATURE_NAMES.map((name)=>[
+        features: Object.fromEntries(OPT_OUT_NAMES.map((name)=>[
                 name,
                 true
             ])),
@@ -82,8 +123,42 @@ export function findConfigPath(startDir) {
         dir = parent;
     }
 }
+// Shape check for hand-edited files: a typo'd config fails with a named error, not a
+// TypeError from whatever code touched the missing field first. `queries` is optional
+// on disk (absent = none); `scan.include` has no usable default.
+function validateConfig(parsed, configPath) {
+    var _cfg_features;
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+        throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);
+    }
+    const cfg = parsed;
+    const scan = cfg.scan;
+    if (!scan || !Array.isArray(scan.include) || scan.include.length === 0 || !scan.include.every((g)=>typeof g === 'string')) {
+        throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);
+    }
+    if (cfg.queries === undefined) cfg.queries = {};
+    if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries) || !Object.values(cfg.queries).every((q)=>typeof q === 'string')) {
+        throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> SQL string`);
+    }
+    if (cfg.features !== undefined && (typeof cfg.features !== 'object' || cfg.features === null || Array.isArray(cfg.features))) {
+        throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);
+    }
+    for (const [name, value] of Object.entries((_cfg_features = cfg.features) !== null && _cfg_features !== void 0 ? _cfg_features : {})){
+        if (typeof value === 'boolean') continue;
+        // embed alone takes an object form: { model?, type?: static|api, url?, key? }.
+        const embed = value;
+        const shapeOk = name === 'embed' && typeof value === 'object' && value !== null && !Array.isArray(value) && [
+            'model',
+            'url',
+            'key'
+        ].every((f)=>embed[f] === undefined || typeof embed[f] === 'string') && (embed.type === undefined || embed.type === 'static' || embed.type === 'api');
+        if (!shapeOk) {
+            throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean${name === 'embed' ? ' or { model?, type?: "static"|"api", url?, key? }' : ''}`);
+        }
+    }
+    return cfg;
+}
 export function loadConfig(explicitPath) {
-    var _cfg_version;
     let configPath;
     if (explicitPath) {
         configPath = resolve(process.cwd(), explicitPath);
@@ -96,11 +171,14 @@ export function loadConfig(explicitPath) {
         configPath = found;
     }
     const raw = readFileSync(configPath, 'utf8');
-    let cfg = JSON.parse(raw);
-    const version = (_cfg_version = cfg.version) !== null && _cfg_version !== void 0 ? _cfg_version : 1;
+    const parsed = JSON.parse(raw);
+    // Version gate before shape validation: a config written by a newer sense should fail
+    // with "requires a newer sense", not with shape errors its own version may not have.
+    const version = typeof parsed === 'object' && parsed !== null && typeof parsed.version === 'number' ? parsed.version : 1;
     if (version > SUPPORTED_CONFIG_VERSION) {
         throw new SenseError('CONFIG_VERSION_UNSUPPORTED', `config version ${version} requires a newer sense`);
     }
+    let cfg = validateConfig(parsed, configPath);
     let migratedFrom;
     if (version < SUPPORTED_CONFIG_VERSION) {
         const result = migrateConfig(cfg);

package/dist/esm/config.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config.ts"],"sourcesContent":["import { existsSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join, resolve } from 'node:path';\nimport { SenseError } from './errors.ts';\n\nexport const CONFIG_FILENAME = 'sense.config.json';\nexport const STATE_DIR = '.sense';\n\n// Highest sense.config.json `version` this build understands. Older versions auto-migrate on load.\nexport const SUPPORTED_CONFIG_VERSION = 2;\n\n// Each feature owns its tables, parse-time extraction, and reconcile step; verbs degrade when one is off.\nexport const FEATURE_NAMES = ['links', 'sections', 'rank'] as const;\nexport type FeatureName = (typeof FEATURE_NAMES)[number];\n\nexport interface Config {\n  // Editor-only pointer to schema.json; never read by sense.\n  $schema?: string;\n  version?: number;\n  scan: { include: string[] };\n  features?: Partial<Record<FeatureName, boolean>>;\n  queries: Record<string, string>;\n}\n\nexport interface ResolvedConfig extends Config {\n  baseDir: string;\n  configPath: string | null;\n  // Set when loadConfig auto-migrated the file on disk; cli reports it.\n  migratedFrom?: number;\n}\n\n// Absent block or key means enabled -- features are opt-out. `rank` additionally requires `links`.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Pure per-version steps; loadConfig chains them from the file's version up to SUPPORTED_CONFIG_VERSION.\nconst MIGRATIONS: Record<number, (cfg: Config) => Config> = {\n  // v1 -> v2: features block introduced, everything enabled (matches the old implicit behavior of `links` etc. not existing).\n  1: (cfg) => ({ ...cfg, version: 2, features: Object.fromEntries(FEATURE_NAMES.map((name) => [name, true])) as Config['features'] }),\n};\n\nexport function migrateConfig(cfg: Config): { cfg: Config; from: number } {\n  const from = cfg.version ?? 1;\n  let current = cfg;\n  for (let v = from; v < SUPPORTED_CONFIG_VERSION; v++) {\n    const step = MIGRATIONS[v];\n    if (!step) throw new SenseError('CONFIG_VERSION_UNSUPPORTED', `no migration from config version ${v}`);\n    current = step(current);\n  }\n  return { cfg: current, from };\n}\n\nfunction starterConfig(): Config {\n  return {\n    $schema: 'https://unpkg.com/sensemaking/schema.json',\n    version: SUPPORTED_CONFIG_VERSION,\n    scan: { include: ['**/*.md'] },\n    features: Object.fromEntries(FEATURE_NAMES.map((name) => [name, true])) as Config['features'],\n    queries: {},\n  };\n}\n\n// Refuses to overwrite an existing config.\nexport function initConfig(dir: string): string {\n  const configPath = join(dir, CONFIG_FILENAME);\n  if (existsSync(configPath)) {\n    throw new SenseError('CONFIG_EXISTS', `${CONFIG_FILENAME} already exists in ${dir}`);\n  }\n  writeFileSync(configPath, `${JSON.stringify(starterConfig(), null, 2)}\\n`);\n  return configPath;\n}\n\nexport function findConfigPath(startDir: string): string | null {\n  let dir = resolve(startDir);\n  for (;;) {\n    const candidate = join(dir, CONFIG_FILENAME);\n    if (existsSync(candidate)) return candidate;\n    const parent = dirname(dir);\n    if (parent === dir) return null;\n    dir = parent;\n  }\n}\n\nexport function loadConfig(explicitPath?: string): ResolvedConfig {\n  let configPath: string;\n  if (explicitPath) {\n    configPath = resolve(process.cwd(), explicitPath);\n    if (!existsSync(configPath)) throw new SenseError('CONFIG_NOT_FOUND', `config not found: ${configPath}`);\n  } else {\n    const found = findConfigPath(process.cwd());\n    if (!found) {\n      throw new SenseError('CONFIG_NOT_FOUND', `could not find ${CONFIG_FILENAME} in ${process.cwd()} or any parent directory`);\n    }\n    configPath = found;\n  }\n\n  const raw = readFileSync(configPath, 'utf8');\n  let cfg = JSON.parse(raw) as Config;\n\n  const version = cfg.version ?? 1;\n  if (version > SUPPORTED_CONFIG_VERSION) {\n    throw new SenseError('CONFIG_VERSION_UNSUPPORTED', `config version ${version} requires a newer sense`);\n  }\n\n  let migratedFrom: number | undefined;\n  if (version < SUPPORTED_CONFIG_VERSION) {\n    const result = migrateConfig(cfg);\n    cfg = result.cfg;\n    migratedFrom = result.from;\n    writeFileSync(configPath, `${JSON.stringify(cfg, null, 2)}\\n`);\n  }\n\n  return { ...cfg, baseDir: dirname(configPath), configPath, migratedFrom };\n}\n"],"names":["existsSync","readFileSync","writeFileSync","dirname","join","resolve","SenseError","CONFIG_FILENAME","STATE_DIR","SUPPORTED_CONFIG_VERSION","FEATURE_NAMES","featureEnabled","cfg","name","enabled","features","enabledFeatures","filter","MIGRATIONS","version","Object","fromEntries","map","migrateConfig","from","current","v","step","starterConfig","$schema","scan","include","queries","initConfig","dir","configPath","JSON","stringify","findConfigPath","startDir","candidate","parent","loadConfig","explicitPath","process","cwd","found","raw","parse","migratedFrom","result","baseDir"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,EAAEC,aAAa,QAAQ,UAAU;AAClE,SAASC,OAAO,EAAEC,IAAI,EAAEC,OAAO,QAAQ,YAAY;AACnD,SAASC,UAAU,QAAQ,cAAc;AAEzC,OAAO,MAAMC,kBAAkB,oBAAoB;AACnD,OAAO,MAAMC,YAAY,SAAS;AAElC,mGAAmG;AACnG,OAAO,MAAMC,2BAA2B,EAAE;AAE1C,0GAA0G;AAC1G,OAAO,MAAMC,gBAAgB;IAAC;IAAS;IAAY;CAAO,CAAU;AAmBpE,mGAAmG;AACnG,OAAO,SAASC,eAAeC,GAAW,EAAEC,IAAiB;QAC3CD;IAAhB,MAAME,UAAUF,EAAAA,gBAAAA,IAAIG,QAAQ,cAAZH,oCAAAA,aAAc,CAACC,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOC,WAAWH,eAAeC,KAAK;IAC3D,OAAOE;AACT;AAEA,OAAO,SAASE,gBAAgBJ,GAAW;IACzC,OAAOF,cAAcO,MAAM,CAAC,CAACJ,OAASF,eAAeC,KAAKC;AAC5D;AAEA,yGAAyG;AACzG,MAAMK,aAAsD;IAC1D,4HAA4H;IAC5H,GAAG,CAACN,MAAS,CAAA;YAAE,GAAGA,GAAG;YAAEO,SAAS;YAAGJ,UAAUK,OAAOC,WAAW,CAACX,cAAcY,GAAG,CAAC,CAACT,OAAS;oBAACA;oBAAM;iBAAK;QAAyB,CAAA;AACnI;AAEA,OAAO,SAASU,cAAcX,GAAW;QAC1BA;IAAb,MAAMY,QAAOZ,eAAAA,IAAIO,OAAO,cAAXP,0BAAAA,eAAe;IAC5B,IAAIa,UAAUb;IACd,IAAK,IAAIc,IAAIF,MAAME,IAAIjB,0BAA0BiB,IAAK;QACpD,MAAMC,OAAOT,UAAU,CAACQ,EAAE;QAC1B,IAAI,CAACC,MAAM,MAAM,IAAIrB,WAAW,8BAA8B,CAAC,iCAAiC,EAAEoB,GAAG;QACrGD,UAAUE,KAAKF;IACjB;IACA,OAAO;QAAEb,KAAKa;QAASD;IAAK;AAC9B;AAEA,SAASI;IACP,OAAO;QACLC,SAAS;QACTV,SAASV;QACTqB,MAAM;YAAEC,SAAS;gBAAC;aAAU;QAAC;QAC7BhB,UAAUK,OAAOC,WAAW,CAACX,cAAcY,GAAG,CAAC,CAACT,OAAS;gBAACA;gBAAM;aAAK;QACrEmB,SAAS,CAAC;IACZ;AACF;AAEA,2CAA2C;AAC3C,OAAO,SAASC,WAAWC,GAAW;IACpC,MAAMC,aAAa/B,KAAK8B,KAAK3B;IAC7B,IAAIP,WAAWmC,aAAa;QAC1B,MAAM,IAAI7B,WAAW,iBAAiB,GAAGC,gBAAgB,mBAAmB,EAAE2B,KAAK;IACrF;IACAhC,cAAciC,YAAY,GAAGC,KAAKC,SAAS,CAACT,iBAAiB,MAAM,GAAG,EAAE,CAAC;IACzE,OAAOO;AACT;AAEA,OAAO,SAASG,eAAeC,QAAgB;IAC7C,IAAIL,MAAM7B,QAAQkC;IAClB,OAAS;QACP,MAAMC,YAAYpC,KAAK8B,KAAK3B;QAC5B,IAAIP,WAAWwC,YAAY,OAAOA;QAClC,MAAMC,SAAStC,QAAQ+B;QACvB,IAAIO,WAAWP,KAAK,OAAO;QAC3BA,MAAMO;IACR;AACF;AAEA,OAAO,SAASC,WAAWC,YAAqB;QAgB9B/B;IAfhB,IAAIuB;IACJ,IAAIQ,cAAc;QAChBR,aAAa9B,QAAQuC,QAAQC,GAAG,IAAIF;QACpC,IAAI,CAAC3C,WAAWmC,aAAa,MAAM,IAAI7B,WAAW,oBAAoB,CAAC,kBAAkB,EAAE6B,YAAY;IACzG,OAAO;QACL,MAAMW,QAAQR,eAAeM,QAAQC,GAAG;QACxC,IAAI,CAACC,OAAO;YACV,MAAM,IAAIxC,WAAW,oBAAoB,CAAC,eAAe,EAAEC,gBAAgB,IAAI,EAAEqC,QAAQC,GAAG,GAAG,wBAAwB,CAAC;QAC1H;QACAV,aAAaW;IACf;IAEA,MAAMC,MAAM9C,aAAakC,YAAY;IACrC,IAAIvB,MAAMwB,KAAKY,KAAK,CAACD;IAErB,MAAM5B,WAAUP,eAAAA,IAAIO,OAAO,cAAXP,0BAAAA,eAAe;IAC/B,IAAIO,UAAUV,0BAA0B;QACtC,MAAM,IAAIH,WAAW,8BAA8B,CAAC,eAAe,EAAEa,QAAQ,uBAAuB,CAAC;IACvG;IAEA,IAAI8B;IACJ,IAAI9B,UAAUV,0BAA0B;QACtC,MAAMyC,SAAS3B,cAAcX;QAC7BA,MAAMsC,OAAOtC,GAAG;QAChBqC,eAAeC,OAAO1B,IAAI;QAC1BtB,cAAciC,YAAY,GAAGC,KAAKC,SAAS,CAACzB,KAAK,MAAM,GAAG,EAAE,CAAC;IAC/D;IAEA,OAAO;QAAE,GAAGA,GAAG;QAAEuC,SAAShD,QAAQgC;QAAaA;QAAYc;IAAa;AAC1E"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config.ts"],"sourcesContent":["import { existsSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join, resolve } from 'node:path';\nimport { SenseError } from './errors.ts';\n\nexport const CONFIG_FILENAME = 'sense.config.json';\nexport const STATE_DIR = '.sense';\n\n// Highest sense.config.json `version` this build understands. Older versions auto-migrate on load.\nexport const SUPPORTED_CONFIG_VERSION = 2;\n\n// Each feature owns its tables, parse-time extraction, and reconcile step; verbs degrade when one is off.\n// links/sections/rank are opt-out (absent = on); embed is opt-in (absent = off) -- most trees don't need vectors.\nconst FEATURE_NAMES = ['links', 'sections', 'rank', 'embed'] as const;\nconst OPT_OUT_NAMES = ['links', 'sections', 'rank'] as const;\nexport type FeatureName = (typeof FEATURE_NAMES)[number];\n\n// `true` = all defaults (static type, default model). A local `model` path skips the network.\nexport interface EmbedConfig {\n  model?: string;\n  type?: 'static' | 'api';\n  url?: string; // api type: OpenAI-compatible base URL, e.g. http://localhost:11434/v1\n  key?: string; // api type: name of the env var holding the bearer token, if any\n}\n\nexport const DEFAULT_EMBED_MODEL = 'minishlab/potion-retrieval-32M';\n\nexport interface Config {\n  // Editor-only pointer to schema.json; never read by sense.\n  $schema?: string;\n  version?: number;\n  scan: { include: string[] };\n  features?: { links?: boolean; sections?: boolean; rank?: boolean; embed?: boolean | EmbedConfig };\n  queries: Record<string, string>;\n}\n\nexport interface ResolvedConfig extends Config {\n  baseDir: string;\n  configPath: string | null;\n  // Set when loadConfig auto-migrated the file on disk; cli reports it.\n  migratedFrom?: number;\n}\n\n// Opt-out features: absent block or key means enabled. `rank` additionally requires `links`.\n// `embed` is the opposite: enabled only when the config says so.\nexport function featureEnabled(cfg: Config, name: FeatureName): boolean {\n  if (name === 'embed') return Boolean(cfg.features?.embed);\n  const enabled = cfg.features?.[name] !== false;\n  if (name === 'rank') return enabled && featureEnabled(cfg, 'links');\n  return enabled;\n}\n\nexport function enabledFeatures(cfg: Config): FeatureName[] {\n  return FEATURE_NAMES.filter((name) => featureEnabled(cfg, name));\n}\n\n// Every feature with its current state; verbs surface this so \"off\" and \"empty\" stay\n// distinguishable in output.\nexport function featureStates(cfg: Config): { on: FeatureName[]; off: FeatureName[] } {\n  return {\n    on: FEATURE_NAMES.filter((name) => featureEnabled(cfg, name)),\n    off: FEATURE_NAMES.filter((name) => !featureEnabled(cfg, name)),\n  };\n}\n\n// Resolved embed settings, or null when the feature is off.\nexport function embedConfig(cfg: Config): { model: string; type: 'static' | 'api'; url?: string; key?: string } | null {\n  const e = cfg.features?.embed;\n  if (!e) return null;\n  const o = e === true ? {} : e;\n  return { model: o.model ?? DEFAULT_EMBED_MODEL, type: o.type ?? 'static', url: o.url, key: o.key };\n}\n\n// Cache-key string: embed carries its type + model so a model change rebuilds like a toggle.\nexport function featureSignature(cfg: Config): string {\n  return enabledFeatures(cfg)\n    .map((name) => {\n      if (name !== 'embed') return name;\n      const e = embedConfig(cfg);\n      return `embed:${e?.type}:${e?.model}`;\n    })\n    .join(',');\n}\n\n// Pure per-version steps; loadConfig chains them from the file's version up to SUPPORTED_CONFIG_VERSION.\nconst MIGRATIONS: Record<number, (cfg: Config) => Config> = {\n  // v1 -> v2: features block introduced, opt-out features enabled (matches the old implicit\n  // behavior of `links` etc. not existing). embed stays absent -- opt-in.\n  1: (cfg) => ({ ...cfg, version: 2, features: Object.fromEntries(OPT_OUT_NAMES.map((name) => [name, true])) as Config['features'] }),\n};\n\nexport function migrateConfig(cfg: Config): { cfg: Config; from: number } {\n  const from = cfg.version ?? 1;\n  let current = cfg;\n  for (let v = from; v < SUPPORTED_CONFIG_VERSION; v++) {\n    const step = MIGRATIONS[v];\n    if (!step) throw new SenseError('CONFIG_VERSION_UNSUPPORTED', `no migration from config version ${v}`);\n    current = step(current);\n  }\n  return { cfg: current, from };\n}\n\nfunction starterConfig(): Config {\n  return {\n    $schema: 'https://unpkg.com/sensemaking/schema.json',\n    version: SUPPORTED_CONFIG_VERSION,\n    scan: { include: ['**/*.md'] },\n    features: Object.fromEntries(OPT_OUT_NAMES.map((name) => [name, true])) as Config['features'],\n    queries: {},\n  };\n}\n\n// Refuses to overwrite an existing config.\nexport function initConfig(dir: string): string {\n  const configPath = join(dir, CONFIG_FILENAME);\n  if (existsSync(configPath)) {\n    throw new SenseError('CONFIG_EXISTS', `${CONFIG_FILENAME} already exists in ${dir}`);\n  }\n  writeFileSync(configPath, `${JSON.stringify(starterConfig(), null, 2)}\\n`);\n  return configPath;\n}\n\nexport function findConfigPath(startDir: string): string | null {\n  let dir = resolve(startDir);\n  for (;;) {\n    const candidate = join(dir, CONFIG_FILENAME);\n    if (existsSync(candidate)) return candidate;\n    const parent = dirname(dir);\n    if (parent === dir) return null;\n    dir = parent;\n  }\n}\n\n// Shape check for hand-edited files: a typo'd config fails with a named error, not a\n// TypeError from whatever code touched the missing field first. `queries` is optional\n// on disk (absent = none); `scan.include` has no usable default.\nfunction validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n  const scan = cfg.scan as { include?: unknown } | undefined;\n  if (!scan || !Array.isArray(scan.include) || scan.include.length === 0 || !scan.include.every((g) => typeof g === 'string')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries) || !Object.values(cfg.queries).every((q) => typeof q === 'string')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> SQL string`);\n  }\n  if (cfg.features !== undefined && (typeof cfg.features !== 'object' || cfg.features === null || Array.isArray(cfg.features))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  for (const [name, value] of Object.entries(cfg.features ?? {})) {\n    if (typeof value === 'boolean') continue;\n    // embed alone takes an object form: { model?, type?: static|api, url?, key? }.\n    const embed = value as Record<string, unknown>;\n    const shapeOk = name === 'embed' && typeof value === 'object' && value !== null && !Array.isArray(value) && ['model', 'url', 'key'].every((f) => embed[f] === undefined || typeof embed[f] === 'string') && (embed.type === undefined || embed.type === 'static' || embed.type === 'api');\n    if (!shapeOk) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean${name === 'embed' ? ' or { model?, type?: \"static\"|\"api\", url?, key? }' : ''}`);\n    }\n  }\n  return cfg as unknown as Config;\n}\n\nexport function loadConfig(explicitPath?: string): ResolvedConfig {\n  let configPath: string;\n  if (explicitPath) {\n    configPath = resolve(process.cwd(), explicitPath);\n    if (!existsSync(configPath)) throw new SenseError('CONFIG_NOT_FOUND', `config not found: ${configPath}`);\n  } else {\n    const found = findConfigPath(process.cwd());\n    if (!found) {\n      throw new SenseError('CONFIG_NOT_FOUND', `could not find ${CONFIG_FILENAME} in ${process.cwd()} or any parent directory`);\n    }\n    configPath = found;\n  }\n\n  const raw = readFileSync(configPath, 'utf8');\n  const parsed: unknown = JSON.parse(raw);\n\n  // Version gate before shape validation: a config written by a newer sense should fail\n  // with \"requires a newer sense\", not with shape errors its own version may not have.\n  const version = typeof parsed === 'object' && parsed !== null && typeof (parsed as { version?: unknown }).version === 'number' ? (parsed as { version: number }).version : 1;\n  if (version > SUPPORTED_CONFIG_VERSION) {\n    throw new SenseError('CONFIG_VERSION_UNSUPPORTED', `config version ${version} requires a newer sense`);\n  }\n\n  let cfg = validateConfig(parsed, configPath);\n\n  let migratedFrom: number | undefined;\n  if (version < SUPPORTED_CONFIG_VERSION) {\n    const result = migrateConfig(cfg);\n    cfg = result.cfg;\n    migratedFrom = result.from;\n    writeFileSync(configPath, `${JSON.stringify(cfg, null, 2)}\\n`);\n  }\n\n  return { ...cfg, baseDir: dirname(configPath), configPath, migratedFrom };\n}\n"],"names":["existsSync","readFileSync","writeFileSync","dirname","join","resolve","SenseError","CONFIG_FILENAME","STATE_DIR","SUPPORTED_CONFIG_VERSION","FEATURE_NAMES","OPT_OUT_NAMES","DEFAULT_EMBED_MODEL","featureEnabled","cfg","name","Boolean","features","embed","enabled","enabledFeatures","filter","featureStates","on","off","embedConfig","o","e","model","type","url","key","featureSignature","map","MIGRATIONS","version","Object","fromEntries","migrateConfig","from","current","v","step","starterConfig","$schema","scan","include","queries","initConfig","dir","configPath","JSON","stringify","findConfigPath","startDir","candidate","parent","validateConfig","parsed","Array","isArray","length","every","g","undefined","values","q","value","entries","shapeOk","f","loadConfig","explicitPath","process","cwd","found","raw","parse","migratedFrom","result","baseDir"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,EAAEC,aAAa,QAAQ,UAAU;AAClE,SAASC,OAAO,EAAEC,IAAI,EAAEC,OAAO,QAAQ,YAAY;AACnD,SAASC,UAAU,QAAQ,cAAc;AAEzC,OAAO,MAAMC,kBAAkB,oBAAoB;AACnD,OAAO,MAAMC,YAAY,SAAS;AAElC,mGAAmG;AACnG,OAAO,MAAMC,2BAA2B,EAAE;AAE1C,0GAA0G;AAC1G,kHAAkH;AAClH,MAAMC,gBAAgB;IAAC;IAAS;IAAY;IAAQ;CAAQ;AAC5D,MAAMC,gBAAgB;IAAC;IAAS;IAAY;CAAO;AAWnD,OAAO,MAAMC,sBAAsB,iCAAiC;AAkBpE,6FAA6F;AAC7F,iEAAiE;AACjE,OAAO,SAASC,eAAeC,GAAW,EAAEC,IAAiB;QACtBD,eACrBA;IADhB,IAAIC,SAAS,SAAS,OAAOC,SAAQF,gBAAAA,IAAIG,QAAQ,cAAZH,oCAAAA,cAAcI,KAAK;IACxD,MAAMC,UAAUL,EAAAA,iBAAAA,IAAIG,QAAQ,cAAZH,qCAAAA,cAAc,CAACC,KAAK,MAAK;IACzC,IAAIA,SAAS,QAAQ,OAAOI,WAAWN,eAAeC,KAAK;IAC3D,OAAOK;AACT;AAEA,OAAO,SAASC,gBAAgBN,GAAW;IACzC,OAAOJ,cAAcW,MAAM,CAAC,CAACN,OAASF,eAAeC,KAAKC;AAC5D;AAEA,qFAAqF;AACrF,6BAA6B;AAC7B,OAAO,SAASO,cAAcR,GAAW;IACvC,OAAO;QACLS,IAAIb,cAAcW,MAAM,CAAC,CAACN,OAASF,eAAeC,KAAKC;QACvDS,KAAKd,cAAcW,MAAM,CAAC,CAACN,OAAS,CAACF,eAAeC,KAAKC;IAC3D;AACF;AAEA,4DAA4D;AAC5D,OAAO,SAASU,YAAYX,GAAW;QAIrBY,UAAsCA;QAH5CZ;IAAV,MAAMa,KAAIb,gBAAAA,IAAIG,QAAQ,cAAZH,oCAAAA,cAAcI,KAAK;IAC7B,IAAI,CAACS,GAAG,OAAO;IACf,MAAMD,IAAIC,MAAM,OAAO,CAAC,IAAIA;IAC5B,OAAO;QAAEC,KAAK,GAAEF,WAAAA,EAAEE,KAAK,cAAPF,sBAAAA,WAAWd;QAAqBiB,IAAI,GAAEH,UAAAA,EAAEG,IAAI,cAANH,qBAAAA,UAAU;QAAUI,KAAKJ,EAAEI,GAAG;QAAEC,KAAKL,EAAEK,GAAG;IAAC;AACnG;AAEA,6FAA6F;AAC7F,OAAO,SAASC,iBAAiBlB,GAAW;IAC1C,OAAOM,gBAAgBN,KACpBmB,GAAG,CAAC,CAAClB;QACJ,IAAIA,SAAS,SAAS,OAAOA;QAC7B,MAAMY,IAAIF,YAAYX;QACtB,OAAO,CAAC,MAAM,EAAEa,cAAAA,wBAAAA,EAAGE,IAAI,CAAC,CAAC,EAAEF,cAAAA,wBAAAA,EAAGC,KAAK,EAAE;IACvC,GACCxB,IAAI,CAAC;AACV;AAEA,yGAAyG;AACzG,MAAM8B,aAAsD;IAC1D,0FAA0F;IAC1F,wEAAwE;IACxE,GAAG,CAACpB,MAAS,CAAA;YAAE,GAAGA,GAAG;YAAEqB,SAAS;YAAGlB,UAAUmB,OAAOC,WAAW,CAAC1B,cAAcsB,GAAG,CAAC,CAAClB,OAAS;oBAACA;oBAAM;iBAAK;QAAyB,CAAA;AACnI;AAEA,OAAO,SAASuB,cAAcxB,GAAW;QAC1BA;IAAb,MAAMyB,QAAOzB,eAAAA,IAAIqB,OAAO,cAAXrB,0BAAAA,eAAe;IAC5B,IAAI0B,UAAU1B;IACd,IAAK,IAAI2B,IAAIF,MAAME,IAAIhC,0BAA0BgC,IAAK;QACpD,MAAMC,OAAOR,UAAU,CAACO,EAAE;QAC1B,IAAI,CAACC,MAAM,MAAM,IAAIpC,WAAW,8BAA8B,CAAC,iCAAiC,EAAEmC,GAAG;QACrGD,UAAUE,KAAKF;IACjB;IACA,OAAO;QAAE1B,KAAK0B;QAASD;IAAK;AAC9B;AAEA,SAASI;IACP,OAAO;QACLC,SAAS;QACTT,SAAS1B;QACToC,MAAM;YAAEC,SAAS;gBAAC;aAAU;QAAC;QAC7B7B,UAAUmB,OAAOC,WAAW,CAAC1B,cAAcsB,GAAG,CAAC,CAAClB,OAAS;gBAACA;gBAAM;aAAK;QACrEgC,SAAS,CAAC;IACZ;AACF;AAEA,2CAA2C;AAC3C,OAAO,SAASC,WAAWC,GAAW;IACpC,MAAMC,aAAa9C,KAAK6C,KAAK1C;IAC7B,IAAIP,WAAWkD,aAAa;QAC1B,MAAM,IAAI5C,WAAW,iBAAiB,GAAGC,gBAAgB,mBAAmB,EAAE0C,KAAK;IACrF;IACA/C,cAAcgD,YAAY,GAAGC,KAAKC,SAAS,CAACT,iBAAiB,MAAM,GAAG,EAAE,CAAC;IACzE,OAAOO;AACT;AAEA,OAAO,SAASG,eAAeC,QAAgB;IAC7C,IAAIL,MAAM5C,QAAQiD;IAClB,OAAS;QACP,MAAMC,YAAYnD,KAAK6C,KAAK1C;QAC5B,IAAIP,WAAWuD,YAAY,OAAOA;QAClC,MAAMC,SAASrD,QAAQ8C;QACvB,IAAIO,WAAWP,KAAK,OAAO;QAC3BA,MAAMO;IACR;AACF;AAEA,qFAAqF;AACrF,sFAAsF;AACtF,iEAAiE;AACjE,SAASC,eAAeC,MAAe,EAAER,UAAkB;QAgBdpC;IAf3C,IAAI,OAAO4C,WAAW,YAAYA,WAAW,QAAQC,MAAMC,OAAO,CAACF,SAAS;QAC1E,MAAM,IAAIpD,WAAW,kBAAkB,GAAG4C,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMpC,MAAM4C;IACZ,MAAMb,OAAO/B,IAAI+B,IAAI;IACrB,IAAI,CAACA,QAAQ,CAACc,MAAMC,OAAO,CAACf,KAAKC,OAAO,KAAKD,KAAKC,OAAO,CAACe,MAAM,KAAK,KAAK,CAAChB,KAAKC,OAAO,CAACgB,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,WAAW;QAC3H,MAAM,IAAIzD,WAAW,kBAAkB,GAAG4C,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIpC,IAAIiC,OAAO,KAAKiB,WAAWlD,IAAIiC,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOjC,IAAIiC,OAAO,KAAK,YAAYjC,IAAIiC,OAAO,KAAK,QAAQY,MAAMC,OAAO,CAAC9C,IAAIiC,OAAO,KAAK,CAACX,OAAO6B,MAAM,CAACnD,IAAIiC,OAAO,EAAEe,KAAK,CAAC,CAACI,IAAM,OAAOA,MAAM,WAAW;QAC5J,MAAM,IAAI5D,WAAW,kBAAkB,GAAG4C,WAAW,iDAAiD,CAAC;IACzG;IACA,IAAIpC,IAAIG,QAAQ,KAAK+C,aAAc,CAAA,OAAOlD,IAAIG,QAAQ,KAAK,YAAYH,IAAIG,QAAQ,KAAK,QAAQ0C,MAAMC,OAAO,CAAC9C,IAAIG,QAAQ,CAAA,GAAI;QAC5H,MAAM,IAAIX,WAAW,kBAAkB,GAAG4C,WAAW,+CAA+C,CAAC;IACvG;IACA,KAAK,MAAM,CAACnC,MAAMoD,MAAM,IAAI/B,OAAOgC,OAAO,EAACtD,gBAAAA,IAAIG,QAAQ,cAAZH,2BAAAA,gBAAgB,CAAC,GAAI;QAC9D,IAAI,OAAOqD,UAAU,WAAW;QAChC,+EAA+E;QAC/E,MAAMjD,QAAQiD;QACd,MAAME,UAAUtD,SAAS,WAAW,OAAOoD,UAAU,YAAYA,UAAU,QAAQ,CAACR,MAAMC,OAAO,CAACO,UAAU;YAAC;YAAS;YAAO;SAAM,CAACL,KAAK,CAAC,CAACQ,IAAMpD,KAAK,CAACoD,EAAE,KAAKN,aAAa,OAAO9C,KAAK,CAACoD,EAAE,KAAK,aAAcpD,CAAAA,MAAMW,IAAI,KAAKmC,aAAa9C,MAAMW,IAAI,KAAK,YAAYX,MAAMW,IAAI,KAAK,KAAI;QACvR,IAAI,CAACwC,SAAS;YACZ,MAAM,IAAI/D,WAAW,kBAAkB,GAAG4C,WAAW,WAAW,EAAEnC,KAAK,kBAAkB,EAAEA,SAAS,UAAU,sDAAsD,IAAI;QAC1K;IACF;IACA,OAAOD;AACT;AAEA,OAAO,SAASyD,WAAWC,YAAqB;IAC9C,IAAItB;IACJ,IAAIsB,cAAc;QAChBtB,aAAa7C,QAAQoE,QAAQC,GAAG,IAAIF;QACpC,IAAI,CAACxE,WAAWkD,aAAa,MAAM,IAAI5C,WAAW,oBAAoB,CAAC,kBAAkB,EAAE4C,YAAY;IACzG,OAAO;QACL,MAAMyB,QAAQtB,eAAeoB,QAAQC,GAAG;QACxC,IAAI,CAACC,OAAO;YACV,MAAM,IAAIrE,WAAW,oBAAoB,CAAC,eAAe,EAAEC,gBAAgB,IAAI,EAAEkE,QAAQC,GAAG,GAAG,wBAAwB,CAAC;QAC1H;QACAxB,aAAayB;IACf;IAEA,MAAMC,MAAM3E,aAAaiD,YAAY;IACrC,MAAMQ,SAAkBP,KAAK0B,KAAK,CAACD;IAEnC,sFAAsF;IACtF,qFAAqF;IACrF,MAAMzC,UAAU,OAAOuB,WAAW,YAAYA,WAAW,QAAQ,OAAO,AAACA,OAAiCvB,OAAO,KAAK,WAAW,AAACuB,OAA+BvB,OAAO,GAAG;IAC3K,IAAIA,UAAU1B,0BAA0B;QACtC,MAAM,IAAIH,WAAW,8BAA8B,CAAC,eAAe,EAAE6B,QAAQ,uBAAuB,CAAC;IACvG;IAEA,IAAIrB,MAAM2C,eAAeC,QAAQR;IAEjC,IAAI4B;IACJ,IAAI3C,UAAU1B,0BAA0B;QACtC,MAAMsE,SAASzC,cAAcxB;QAC7BA,MAAMiE,OAAOjE,GAAG;QAChBgE,eAAeC,OAAOxC,IAAI;QAC1BrC,cAAcgD,YAAY,GAAGC,KAAKC,SAAS,CAACtC,KAAK,MAAM,GAAG,EAAE,CAAC;IAC/D;IAEA,OAAO;QAAE,GAAGA,GAAG;QAAEkE,SAAS7E,QAAQ+C;QAAaA;QAAY4B;IAAa;AAC1E"}

package/dist/esm/db.js

@@ -1,7 +1,7 @@
 import { mkdirSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
-import { enabledFeatures, STATE_DIR } from './config.js';
+import { featureSignature, STATE_DIR } from './config.js';
 import { activeFeatures } from './features/index.js';
 import { listFiles, parseFile } from './scan.js';
 // Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;
@@ -45,7 +45,7 @@ function ensureSchema(db, cfg) {
     db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);
     for (const feature of activeFeatures(cfg))feature.schema(db);
     if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);
-    if (getMeta(db, 'features') === null) setMeta(db, 'features', enabledFeatures(cfg).join(','));
+    if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));
 }
 export function getMeta(db, key) {
     const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key);
@@ -118,6 +118,7 @@ export function reconcile(db, cfg, baseDir) {
             const insert = db.prepare(insertSql);
             const insertBody = db.prepare(`INSERT INTO content (title, summary, text, "path") VALUES (?, ?, ?, ?)`);
             for (const doc of parsedDocs){
+                var _feature_store;
                 const values = allColumns.map((col)=>{
                     var _doc_data_col;
                     if (col === 'path') return doc.relPath;
@@ -126,13 +127,17 @@ export function reconcile(db, cfg, baseDir) {
                     return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
                 });
                 insert.run(...values);
-                delBody.run(doc.relPath);
-                insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);
-                for (const feature of features){
-                    var _feature_remove1, _feature_store;
-                    (_feature_remove1 = feature.remove) === null || _feature_remove1 === void 0 ? void 0 : _feature_remove1.call(feature, db, doc.relPath);
-                    (_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, db, doc.relPath, doc.extracted[feature.name]);
+                // Delete-before-insert only for docs that have rows: an FTS5 DELETE by column
+                // cannot use an index and scans the whole table, so running it per doc on a
+                // cold build (empty table, nothing to delete) made the crawl quadratic --
+                // measured 4x time per note-count doubling at 13k/26k notes.
+                if (existing.has(doc.relPath)) {
+                    var _feature_remove1;
+                    delBody.run(doc.relPath);
+                    for (const feature of features)(_feature_remove1 = feature.remove) === null || _feature_remove1 === void 0 ? void 0 : _feature_remove1.call(feature, db, doc.relPath);
                 }
+                insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);
+                for (const feature of features)(_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, db, doc.relPath, doc.extracted[feature.name]);
             }
         }
         for (const feature of features)(_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, db, files);
@@ -154,14 +159,18 @@ export function open(cfg) {
     const dbPath = join(stateDir, DB_FILENAME);
     const db = new DatabaseSync(dbPath);
     db.exec('PRAGMA journal_mode = WAL');
-    db.exec('PRAGMA busy_timeout = 5000');
+    // Covers a concurrent watcher's bulk reconcile: the write transaction for 500 changed
+    // files measures ~5s at 26k notes, so 5s expired exactly at the boundary and queries
+    // racing the watcher got SQLITE_BUSY. 30s bounds the wait at ~3x the largest measured
+    // reconcile; a query that outwaits it still fails loudly.
+    db.exec('PRAGMA busy_timeout = 30000');
     registerFunctions(db);
     db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
     // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an
     // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).
     const version = getMeta(db, 'schema_version');
     const features = getMeta(db, 'features');
-    const wantFeatures = enabledFeatures(cfg).join(',');
+    const wantFeatures = featureSignature(cfg);
     if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
         db.close();
         rmSync(stateDir, {

package/dist/esm/db.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["import { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { enabledFeatures, STATE_DIR } from './config.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile } from './scan.ts';\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', enabledFeatures(cfg).join(','));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  for (const file of toReparse) {\n    const { doc, warnings: fileWarnings } = parseFile(file, features);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n\n  const allColumns = [...seenColumns];\n  const insertSql = `INSERT OR REPLACE INTO frontmatter (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`;\n\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`.\n    const delBody = db.prepare(`DELETE FROM content WHERE \"path\" = ?`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        del.run(path);\n        delBody.run(path);\n        for (const feature of features) feature.remove?.(db, path);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (title, summary, text, \"path\") VALUES (?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = allColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        insert.run(...values);\n        delBody.run(doc.relPath);\n        insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        for (const feature of features) {\n          feature.remove?.(db, doc.relPath);\n          feature.store?.(db, doc.relPath, doc.extracted[feature.name]);\n        }\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, files);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  db.exec('PRAGMA busy_timeout = 5000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = enabledFeatures(cfg).join(',');\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    db.close();\n    rmSync(stateDir, { recursive: true, force: true });\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Manual reset for a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","enabledFeatures","STATE_DIR","activeFeatures","listFiles","parseFile","DB_FILENAME","SCHEMA_VERSION","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","Set","map","r","ensureSchema","cfg","exec","feature","schema","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","file","doc","fileWarnings","push","Object","keys","data","add","allColumns","insertSql","col","delBody","del","remove","insert","insertBody","values","search","title","summary","text","store","extracted","afterReconcile","err","open","stateDir","recursive","dbPath","version","wantFeatures","close","force","rebuild"],"mappings":"AAAA,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,SAAS,QAAQ,cAAc;AACzD,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAEjD,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,kEAAkE;AAClE,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKX,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASY,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAIC,IAAIH,KAAKI,GAAG,CAAC,CAACC,IAAMA,EAAE3B,IAAI;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS4B,aAAazB,EAAgB,EAAE0B,GAAW;IACjD1B,GAAG2B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G3B,GAAG2B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,KAAK,MAAMC,WAAWrC,eAAemC,KAAME,QAAQC,MAAM,CAAC7B;IAC1D,IAAI8B,QAAQ9B,IAAI,sBAAsB,MAAM+B,QAAQ/B,IAAI,kBAAkBL;IAC1E,IAAImC,QAAQ9B,IAAI,gBAAgB,MAAM+B,QAAQ/B,IAAI,YAAYX,gBAAgBqC,KAAKvC,IAAI,CAAC;AAC1F;AAEA,OAAO,SAAS2C,QAAQ9B,EAAgB,EAAEgC,GAAW;IACnD,MAAMC,MAAMjC,GAAGoB,OAAO,CAAC,wCAAwCc,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI5B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAAS0B,QAAQ/B,EAAgB,EAAEgC,GAAW,EAAE3B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCe,GAAG,CAACH;QACjD;IACF;IACAhC,GAAGoB,OAAO,CAAC,qGAAqGe,GAAG,CAACH,KAAK3B;AAC3H;AAEA,OAAO,SAAS+B,SAASpC,EAAgB;IACvC,MAAMiC,MAAMjC,GAAGoB,OAAO,CAAC,yCAAyCc,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUtC,EAAgB,EAAE0B,GAAW,EAAEa,OAAe;IACtE,MAAMC,QAAQhD,UAAUkC,KAAKa;IAC7B,MAAME,aAAa,IAAInB,IAAIkB,MAAMjB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe5C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMwB,WAAW,IAAIC,IAAIF,aAAarB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEuB,IAAI;YAAEvB;SAAE;IAC5D,MAAMwB,WAAWJ,aAAaK,MAAM,CAAC,CAACzB,IAAM,CAACiB,WAAWS,GAAG,CAAC1B,EAAEuB,IAAI,GAAGxB,GAAG,CAAC,CAACC,IAAMA,EAAEuB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMT,MAAMY,SAASX,GAAG,CAACQ,EAAEC,OAAO;QAClC,OAAO,CAACV,OAAOA,IAAImB,MAAM,KAAKV,EAAEW,OAAO,IAAIpB,IAAIqB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE9C,QAAQ;QAAG+C,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWnE,eAAemC;IAChC,MAAMiC,cAAczC,WAAWlB;IAC/B,MAAM4D,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,KAAK,MAAMK,QAAQX,UAAW;QAC5B,MAAM,EAAEY,GAAG,EAAEN,UAAUO,YAAY,EAAE,GAAGvE,UAAUqE,MAAMJ;QACxDD,SAASQ,IAAI,IAAID;QACjB,KAAK,MAAMhC,OAAOkC,OAAOC,IAAI,CAACJ,IAAIK,IAAI,EAAG;YACvC,IAAI,CAACT,YAAYT,GAAG,CAAClB,MAAM;gBACzB2B,YAAYU,GAAG,CAACrC;gBAChB4B,WAAWK,IAAI,CAACjC;YAClB;QACF;QACA6B,WAAWI,IAAI,CAACF;IAClB;IAEA,MAAMO,aAAa;WAAIX;KAAY;IACnC,MAAMY,YAAY,CAAC,oCAAoC,EAAED,WAAW/C,GAAG,CAAC3B,YAAYT,IAAI,CAAC,MAAM,UAAU,EAAEmF,WAAW/C,GAAG,CAAC,IAAM,KAAKpC,IAAI,CAAC,MAAM,CAAC,CAAC;IAElJa,GAAG2B,IAAI,CAAC;IACR,IAAI;YA+B8BC;QA9BhC,KAAK,MAAM4C,OAAOZ,WAAY5D,GAAG2B,IAAI,CAAC,CAAC,mCAAmC,EAAE/B,WAAW4E,MAAM;QAC7F,8DAA8D;QAC9D,MAAMC,UAAUzE,GAAGoB,OAAO,CAAC,CAAC,oCAAoC,CAAC;QACjE,IAAI4B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMkB,MAAM1E,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM2B,QAAQC,SAAU;oBAGKpB;gBAFhC8C,IAAIvC,GAAG,CAACY;gBACR0B,QAAQtC,GAAG,CAACY;gBACZ,KAAK,MAAMnB,WAAW8B,UAAU9B,kBAAAA,QAAQ+C,MAAM,cAAd/C,sCAAAA,qBAAAA,SAAiB5B,IAAI+C;YACvD;QACF;QACA,IAAIc,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMoB,SAAS5E,GAAGoB,OAAO,CAACmD;YAC1B,MAAMM,aAAa7E,GAAGoB,OAAO,CAAC,CAAC,sEAAsE,CAAC;YACtG,KAAK,MAAM2C,OAAOF,WAAY;gBAC5B,MAAMiB,SAASR,WAAW/C,GAAG,CAAC,CAACiD;wBAItBT;oBAHP,IAAIS,QAAQ,QAAQ,OAAOT,IAAIpB,OAAO;oBACtC,IAAI6B,QAAQ,UAAU,OAAOT,IAAIV,OAAO;oBACxC,IAAImB,QAAQ,SAAS,OAAOT,IAAIR,IAAI;oBACpC,QAAOQ,gBAAAA,IAAIK,IAAI,CAACI,IAAI,cAAbT,2BAAAA,gBAAiB;gBAC1B;gBACAa,OAAOzC,GAAG,IAAI2C;gBACdL,QAAQtC,GAAG,CAAC4B,IAAIpB,OAAO;gBACvBkC,WAAW1C,GAAG,CAAC4B,IAAIgB,MAAM,CAACC,KAAK,EAAEjB,IAAIgB,MAAM,CAACE,OAAO,EAAElB,IAAIgB,MAAM,CAACG,IAAI,EAAEnB,IAAIpB,OAAO;gBACjF,KAAK,MAAMf,WAAW8B,SAAU;wBAC9B9B,kBACAA;qBADAA,mBAAAA,QAAQ+C,MAAM,cAAd/C,uCAAAA,sBAAAA,SAAiB5B,IAAI+D,IAAIpB,OAAO;qBAChCf,iBAAAA,QAAQuD,KAAK,cAAbvD,qCAAAA,oBAAAA,SAAgB5B,IAAI+D,IAAIpB,OAAO,EAAEoB,IAAIqB,SAAS,CAACxD,QAAQ/B,IAAI,CAAC;gBAC9D;YACF;QACF;QACA,KAAK,MAAM+B,WAAW8B,UAAU9B,0BAAAA,QAAQyD,cAAc,cAAtBzD,8CAAAA,6BAAAA,SAAyB5B,IAAIwC;QAC7DxC,GAAG2B,IAAI,CAAC;IACV,EAAE,OAAO2D,KAAK;QACZtF,GAAG2B,IAAI,CAAC;QACR,MAAM2D;IACR;IAEA,OAAO;QAAE5E,QAAQmD,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,OAAO,SAAS8B,KAAK7D,GAAmB;IACtC,MAAM8D,WAAWrG,KAAKuC,IAAIa,OAAO,EAAEjD;IACnCL,UAAUuG,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAASvG,KAAKqG,UAAU9F;IAE9B,MAAMM,KAAK,IAAIZ,aAAasG;IAC5B1F,GAAG2B,IAAI,CAAC;IACR3B,GAAG2B,IAAI,CAAC;IACR5B,kBAAkBC;IAElBA,GAAG2B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,MAAMgE,UAAU7D,QAAQ9B,IAAI;IAC5B,MAAM0D,WAAW5B,QAAQ9B,IAAI;IAC7B,MAAM4F,eAAevG,gBAAgBqC,KAAKvC,IAAI,CAAC;IAC/C,IAAI,AAACwG,YAAY,QAAQA,YAAYhG,kBAAoB+D,aAAa,QAAQA,aAAakC,cAAe;QACxG5F,GAAG6F,KAAK;QACR3G,OAAOsG,UAAU;YAAEC,WAAW;YAAMK,OAAO;QAAK;QAChD,OAAOP,KAAK7D;IACd;IAEAD,aAAazB,IAAI0B;IACjB,MAAM,EAAEhB,MAAM,EAAE+C,QAAQ,EAAE,GAAGnB,UAAUtC,IAAI0B,KAAKA,IAAIa,OAAO;IAE3D,OAAO;QAAEvC;QAAI0B;QAAKgE;QAAQhF;QAAQ+C;IAAS;AAC7C;AAEA,oCAAoC;AACpC,OAAO,SAASsC,QAAQrE,GAAmB;IACzCxC,OAAOC,KAAKuC,IAAIa,OAAO,EAAEjD,YAAY;QAAEmG,WAAW;QAAMK,OAAO;IAAK;IACpE,OAAOP,KAAK7D;AACd"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["import { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile } from './scan.ts';\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  for (const file of toReparse) {\n    const { doc, warnings: fileWarnings } = parseFile(file, features);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n\n  const allColumns = [...seenColumns];\n  const insertSql = `INSERT OR REPLACE INTO frontmatter (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`;\n\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`.\n    const delBody = db.prepare(`DELETE FROM content WHERE \"path\" = ?`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        del.run(path);\n        delBody.run(path);\n        for (const feature of features) feature.remove?.(db, path);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (title, summary, text, \"path\") VALUES (?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = allColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        insert.run(...values);\n        // Delete-before-insert only for docs that have rows: an FTS5 DELETE by column\n        // cannot use an index and scans the whole table, so running it per doc on a\n        // cold build (empty table, nothing to delete) made the crawl quadratic --\n        // measured 4x time per note-count doubling at 13k/26k notes.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath);\n        }\n        insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name]);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, files);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile: the write transaction for 500 changed\n  // files measures ~5s at 26k notes, so 5s expired exactly at the boundary and queries\n  // racing the watcher got SQLITE_BUSY. 30s bounds the wait at ~3x the largest measured\n  // reconcile; a query that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    db.close();\n    rmSync(stateDir, { recursive: true, force: true });\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Manual reset for a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","featureSignature","STATE_DIR","activeFeatures","listFiles","parseFile","DB_FILENAME","SCHEMA_VERSION","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","Set","map","r","ensureSchema","cfg","exec","feature","schema","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","file","doc","fileWarnings","push","Object","keys","data","add","allColumns","insertSql","col","delBody","del","remove","insert","insertBody","values","search","title","summary","text","store","extracted","afterReconcile","err","open","stateDir","recursive","dbPath","version","wantFeatures","close","force","rebuild"],"mappings":"AAAA,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,cAAc;AAC1D,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,SAAS,EAAEC,SAAS,QAAQ,YAAY;AAEjD,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,kEAAkE;AAClE,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKX,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASY,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAIC,IAAIH,KAAKI,GAAG,CAAC,CAACC,IAAMA,EAAE3B,IAAI;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS4B,aAAazB,EAAgB,EAAE0B,GAAW;IACjD1B,GAAG2B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G3B,GAAG2B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,KAAK,MAAMC,WAAWrC,eAAemC,KAAME,QAAQC,MAAM,CAAC7B;IAC1D,IAAI8B,QAAQ9B,IAAI,sBAAsB,MAAM+B,QAAQ/B,IAAI,kBAAkBL;IAC1E,IAAImC,QAAQ9B,IAAI,gBAAgB,MAAM+B,QAAQ/B,IAAI,YAAYX,iBAAiBqC;AACjF;AAEA,OAAO,SAASI,QAAQ9B,EAAgB,EAAEgC,GAAW;IACnD,MAAMC,MAAMjC,GAAGoB,OAAO,CAAC,wCAAwCc,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI5B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAAS0B,QAAQ/B,EAAgB,EAAEgC,GAAW,EAAE3B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCe,GAAG,CAACH;QACjD;IACF;IACAhC,GAAGoB,OAAO,CAAC,qGAAqGe,GAAG,CAACH,KAAK3B;AAC3H;AAEA,OAAO,SAAS+B,SAASpC,EAAgB;IACvC,MAAMiC,MAAMjC,GAAGoB,OAAO,CAAC,yCAAyCc,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUtC,EAAgB,EAAE0B,GAAW,EAAEa,OAAe;IACtE,MAAMC,QAAQhD,UAAUkC,KAAKa;IAC7B,MAAME,aAAa,IAAInB,IAAIkB,MAAMjB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe5C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMwB,WAAW,IAAIC,IAAIF,aAAarB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEuB,IAAI;YAAEvB;SAAE;IAC5D,MAAMwB,WAAWJ,aAAaK,MAAM,CAAC,CAACzB,IAAM,CAACiB,WAAWS,GAAG,CAAC1B,EAAEuB,IAAI,GAAGxB,GAAG,CAAC,CAACC,IAAMA,EAAEuB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMT,MAAMY,SAASX,GAAG,CAACQ,EAAEC,OAAO;QAClC,OAAO,CAACV,OAAOA,IAAImB,MAAM,KAAKV,EAAEW,OAAO,IAAIpB,IAAIqB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE9C,QAAQ;QAAG+C,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWnE,eAAemC;IAChC,MAAMiC,cAAczC,WAAWlB;IAC/B,MAAM4D,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,KAAK,MAAMK,QAAQX,UAAW;QAC5B,MAAM,EAAEY,GAAG,EAAEN,UAAUO,YAAY,EAAE,GAAGvE,UAAUqE,MAAMJ;QACxDD,SAASQ,IAAI,IAAID;QACjB,KAAK,MAAMhC,OAAOkC,OAAOC,IAAI,CAACJ,IAAIK,IAAI,EAAG;YACvC,IAAI,CAACT,YAAYT,GAAG,CAAClB,MAAM;gBACzB2B,YAAYU,GAAG,CAACrC;gBAChB4B,WAAWK,IAAI,CAACjC;YAClB;QACF;QACA6B,WAAWI,IAAI,CAACF;IAClB;IAEA,MAAMO,aAAa;WAAIX;KAAY;IACnC,MAAMY,YAAY,CAAC,oCAAoC,EAAED,WAAW/C,GAAG,CAAC3B,YAAYT,IAAI,CAAC,MAAM,UAAU,EAAEmF,WAAW/C,GAAG,CAAC,IAAM,KAAKpC,IAAI,CAAC,MAAM,CAAC,CAAC;IAElJa,GAAG2B,IAAI,CAAC;IACR,IAAI;YAmC8BC;QAlChC,KAAK,MAAM4C,OAAOZ,WAAY5D,GAAG2B,IAAI,CAAC,CAAC,mCAAmC,EAAE/B,WAAW4E,MAAM;QAC7F,8DAA8D;QAC9D,MAAMC,UAAUzE,GAAGoB,OAAO,CAAC,CAAC,oCAAoC,CAAC;QACjE,IAAI4B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMkB,MAAM1E,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM2B,QAAQC,SAAU;oBAGKpB;gBAFhC8C,IAAIvC,GAAG,CAACY;gBACR0B,QAAQtC,GAAG,CAACY;gBACZ,KAAK,MAAMnB,WAAW8B,UAAU9B,kBAAAA,QAAQ+C,MAAM,cAAd/C,sCAAAA,qBAAAA,SAAiB5B,IAAI+C;YACvD;QACF;QACA,IAAIc,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMoB,SAAS5E,GAAGoB,OAAO,CAACmD;YAC1B,MAAMM,aAAa7E,GAAGoB,OAAO,CAAC,CAAC,sEAAsE,CAAC;YACtG,KAAK,MAAM2C,OAAOF,WAAY;oBAiBIjC;gBAhBhC,MAAMkD,SAASR,WAAW/C,GAAG,CAAC,CAACiD;wBAItBT;oBAHP,IAAIS,QAAQ,QAAQ,OAAOT,IAAIpB,OAAO;oBACtC,IAAI6B,QAAQ,UAAU,OAAOT,IAAIV,OAAO;oBACxC,IAAImB,QAAQ,SAAS,OAAOT,IAAIR,IAAI;oBACpC,QAAOQ,gBAAAA,IAAIK,IAAI,CAACI,IAAI,cAAbT,2BAAAA,gBAAiB;gBAC1B;gBACAa,OAAOzC,GAAG,IAAI2C;gBACd,8EAA8E;gBAC9E,4EAA4E;gBAC5E,0EAA0E;gBAC1E,6DAA6D;gBAC7D,IAAIjC,SAASK,GAAG,CAACa,IAAIpB,OAAO,GAAG;wBAEGf;oBADhC6C,QAAQtC,GAAG,CAAC4B,IAAIpB,OAAO;oBACvB,KAAK,MAAMf,WAAW8B,UAAU9B,mBAAAA,QAAQ+C,MAAM,cAAd/C,uCAAAA,sBAAAA,SAAiB5B,IAAI+D,IAAIpB,OAAO;gBAClE;gBACAkC,WAAW1C,GAAG,CAAC4B,IAAIgB,MAAM,CAACC,KAAK,EAAEjB,IAAIgB,MAAM,CAACE,OAAO,EAAElB,IAAIgB,MAAM,CAACG,IAAI,EAAEnB,IAAIpB,OAAO;gBACjF,KAAK,MAAMf,WAAW8B,UAAU9B,iBAAAA,QAAQuD,KAAK,cAAbvD,qCAAAA,oBAAAA,SAAgB5B,IAAI+D,IAAIpB,OAAO,EAAEoB,IAAIqB,SAAS,CAACxD,QAAQ/B,IAAI,CAAC;YAC9F;QACF;QACA,KAAK,MAAM+B,WAAW8B,UAAU9B,0BAAAA,QAAQyD,cAAc,cAAtBzD,8CAAAA,6BAAAA,SAAyB5B,IAAIwC;QAC7DxC,GAAG2B,IAAI,CAAC;IACV,EAAE,OAAO2D,KAAK;QACZtF,GAAG2B,IAAI,CAAC;QACR,MAAM2D;IACR;IAEA,OAAO;QAAE5E,QAAQmD,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,OAAO,SAAS8B,KAAK7D,GAAmB;IACtC,MAAM8D,WAAWrG,KAAKuC,IAAIa,OAAO,EAAEjD;IACnCL,UAAUuG,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAASvG,KAAKqG,UAAU9F;IAE9B,MAAMM,KAAK,IAAIZ,aAAasG;IAC5B1F,GAAG2B,IAAI,CAAC;IACR,sFAAsF;IACtF,qFAAqF;IACrF,sFAAsF;IACtF,0DAA0D;IAC1D3B,GAAG2B,IAAI,CAAC;IACR5B,kBAAkBC;IAElBA,GAAG2B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,MAAMgE,UAAU7D,QAAQ9B,IAAI;IAC5B,MAAM0D,WAAW5B,QAAQ9B,IAAI;IAC7B,MAAM4F,eAAevG,iBAAiBqC;IACtC,IAAI,AAACiE,YAAY,QAAQA,YAAYhG,kBAAoB+D,aAAa,QAAQA,aAAakC,cAAe;QACxG5F,GAAG6F,KAAK;QACR3G,OAAOsG,UAAU;YAAEC,WAAW;YAAMK,OAAO;QAAK;QAChD,OAAOP,KAAK7D;IACd;IAEAD,aAAazB,IAAI0B;IACjB,MAAM,EAAEhB,MAAM,EAAE+C,QAAQ,EAAE,GAAGnB,UAAUtC,IAAI0B,KAAKA,IAAIa,OAAO;IAE3D,OAAO;QAAEvC;QAAI0B;QAAKgE;QAAQhF;QAAQ+C;IAAS;AAC7C;AAEA,oCAAoC;AACpC,OAAO,SAASsC,QAAQrE,GAAmB;IACzCxC,OAAOC,KAAKuC,IAAIa,OAAO,EAAEjD,YAAY;QAAEmG,WAAW;QAAMK,OAAO;IAAK;IACpE,OAAOP,KAAK7D;AACd"}

package/dist/esm/errors.d.ts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/esm/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AAGrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AAGrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}

package/dist/esm/features/embed.d.ts

@@ -0,0 +1,14 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Config } from '../config.js';
+import type { Feature } from './types.js';
+export interface EmbedProvider {
+    id: string;
+    dims: number;
+    embed(texts: string[]): Promise<Float32Array[]>;
+}
+export declare const embed: Feature;
+export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{
+    path: string;
+    lines: string;
+}>>;