sensemaking 0.11.2 → 0.11.4

package/dist/cjs/db.d.cts

@@ -1,7 +1,7 @@
 import { DatabaseSync } from 'node:sqlite';
 import type { Config, ResolvedConfig } from './config.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "9";
+export declare const SCHEMA_VERSION = "10";
 export interface OpenResult {
     db: DatabaseSync;
     cfg: ResolvedConfig;

package/dist/cjs/db.d.ts

@@ -1,7 +1,7 @@
 import { DatabaseSync } from 'node:sqlite';
 import type { Config, ResolvedConfig } from './config.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "9";
+export declare const SCHEMA_VERSION = "10";
 export interface OpenResult {
     db: DatabaseSync;
     cfg: ResolvedConfig;

package/dist/cjs/db.js

@@ -111,7 +111,7 @@ var CORE_FRONTMATTER_COLUMNS = new Set([
     '_parse_error'
 ]);
 var DB_FILENAME = 'cache.db';
-var SCHEMA_VERSION = '9';
+var SCHEMA_VERSION = '10';
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 var MAX_FRONTMATTER_COLUMNS = 2000;
 function quoteIdent(name) {

package/dist/cjs/db.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size', '_parse_error']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '9';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, \"path\") VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Names what moved between two feature signatures (see config.featureSignature's format:\n// global features, embed provider, then one segment per preset) for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n    }\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","getMeta","open","reconcile","setMeta","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","join","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","map","r","ensureSchema","cfg","exec","activeFeatures","feature","schema","featureSignature","key","row","get","run","n","baseDir","files","listFiles","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","progress","parsedCount","file","fileFeatures","enabledForFile","parseFile","doc","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","SenseError","writableColumns","c","RESERVED_COLUMNS","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","insertBody","values","parseError","search","title","summary","text","presets","presetName","store","extracted","afterReconcile","err","durationMs","prevRaw","prevMax","Number","signatureDiff","before","after","keyOf","part","slice","sig","a","b","changed","val","label","stateDir","STATE_DIR","mkdirSync","recursive","dbPath","DatabaseSync","version","wantFeatures","console","error","close","recordedMaxMs","Math","min","max","rmSync","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAoBpFA;eAAAA;;QAGAC;eAAAA;;QAoRGC;eAAAA;;QA5MAC;eAAAA;;QAbAC;eAAAA;;QAwKAC;eAAAA;;QAtJAC;eAAAA;;QAbAC;eAAAA;;;sBAtFkB;wBACb;0BACQ;wBAEe;wBACjB;uBACI;0BAEN;sBAE8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,6FAA6F;AAC7F,4EAA4E;AAC5E,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAe;AAK7E,IAAMT,cAAc;AAGpB,IAAMC,iBAAiB;AAE9B,8FAA8F;AAC9F,IAAMS,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEA,+FAA+F;AAC/F,SAASC,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,SAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,IAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,IAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,SAACC;mCAASR,OAAOQ,UAAUT;6BAAU,IAAI;oBAC9D;gBACF,EAAE,eAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,IAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAI5B,IAAI0B,KAAKG,GAAG,CAAC,SAACC;eAAMA,EAAE3B,IAAI;;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS4B,aAAaxB,EAAgB,EAAEyB,GAAW;IACjDzB,GAAG0B,IAAI,CAAC;IACR1B,GAAG0B,IAAI,CAAC;IACR,6FAA6F;IAC7F,qFAAqF;IACrF1B,GAAG0B,IAAI,CAAC;IACR1B,GAAG0B,IAAI,CAAC;QACH,kCAAA,2BAAA;;QAAL,QAAK,YAAiBC,IAAAA,uBAAc,EAACF,yBAAhC,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMG,UAAN;YAAsCA,QAAQC,MAAM,CAAC7B;;;QAArD;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAIZ,QAAQY,IAAI,sBAAsB,MAAMT,QAAQS,IAAI,kBAAkBf;IAC1E,IAAIG,QAAQY,IAAI,gBAAgB,MAAMT,QAAQS,IAAI,YAAY8B,IAAAA,0BAAgB,EAACL;AACjF;AAEO,SAASrC,QAAQY,EAAgB,EAAE+B,GAAW;IACnD,IAAMC,MAAMhC,GAAGoB,OAAO,CAAC,wCAAwCa,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI3B,KAAK,GAAG;AAC3B;AAEO,SAASd,QAAQS,EAAgB,EAAE+B,GAAW,EAAE1B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCc,GAAG,CAACH;QACjD;IACF;IACA/B,GAAGoB,OAAO,CAAC,qGAAqGc,GAAG,CAACH,KAAK1B;AAC3H;AAEO,SAASlB,SAASa,EAAgB;IACvC,IAAMgC,MAAMhC,GAAGoB,OAAO,CAAC,yCAAyCa,GAAG;IACnE,OAAOD,IAAIG,CAAC;AACd;AAEO,SAAS7C,UAAUU,EAAgB,EAAEyB,GAAW,EAAEW,OAAe;IACtE,IAAMC,QAAQC,IAAAA,iBAAS,EAACb,KAAKW;IAC7B,IAAMG,aAAa,IAAI9C,IAAI4C,MAAMf,GAAG,CAAC,SAACkB;eAAMA,EAAEC,OAAO;;IAErD,IAAMC,eAAe1C,GAAGoB,OAAO,CAAC,qDAAqDC,GAAG;IAKxF,IAAMsB,WAAW,IAAIC,IAAIF,aAAapB,GAAG,CAAC,SAACC;eAAM;YAACA,EAAEsB,IAAI;YAAEtB;SAAE;;IAC5D,IAAMuB,WAAWJ,aAAaK,MAAM,CAAC,SAACxB;eAAM,CAACgB,WAAWS,GAAG,CAACzB,EAAEsB,IAAI;OAAGvB,GAAG,CAAC,SAACC;eAAMA,EAAEsB,IAAI;;IAEtF,IAAMI,YAAYZ,MAAMU,MAAM,CAAC,SAACP;QAC9B,IAAMR,MAAMW,SAASV,GAAG,CAACO,EAAEC,OAAO;QAClC,OAAO,CAACT,OAAOA,IAAIkB,MAAM,KAAKV,EAAEW,OAAO,IAAInB,IAAIoB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE5C,QAAQ;QAAG6C,UAAU,EAAE;IAAC;IAEtF,IAAMC,WAAW7B,IAAAA,uBAAc,EAACF;IAChC,IAAMgC,cAAcvC,WAAWlB;IAC/B,IAAM0D,aAAuB,EAAE;IAC/B,IAAMC,aAA0B,EAAE;IAClC,IAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,IAAMK,SAASC,IAAAA,oBAAQ,EAAC,mBAAmBZ,UAAUK,MAAM;IAC3D,IAAIQ,cAAc;QACb,kCAAA,2BAAA;;;YAAA,IAAMC,OAAN;gBAMHR;YALA,sFAAsF;YACtF,mEAAmE;YACnE,IAAMS,eAAeR,SAAST,MAAM,CAAC,SAACnB;uBAAY,CAACA,QAAQqC,cAAc,IAAIrC,QAAQqC,cAAc,CAACxC,KAAKsC;;YACzG,IAAwCG,aAAAA,IAAAA,iBAAS,EAACH,MAAMC,eAAhDG,MAAgCD,WAAhCC,KAAKZ,AAAUa,eAAiBF,WAA3BX;YACbK,OAAOS,IAAI,CAAC,EAAEP;YACdP,CAAAA,YAAAA,UAASe,IAAI,OAAbf,WAAc,qBAAGa;gBACZ,kCAAA,2BAAA;;gBAAL,QAAK,YAAaG,OAAOC,IAAI,CAACL,IAAIM,IAAI,sBAAjC,UAAA,6BAAA,SAAA,yBAAA,iCAAoC;oBAApC,IAAM1C,MAAN;oBACH,IAAI,CAAC0B,YAAYT,GAAG,CAACjB,MAAM;wBACzB0B,YAAYiB,GAAG,CAAC3C;wBAChB2B,WAAWY,IAAI,CAACvC;oBAClB;gBACF;;gBALK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAML4B,WAAWW,IAAI,CAACH;QAClB;QAdA,QAAK,YAAclB,8BAAd,SAAA,6BAAA,QAAA,yBAAA;;QAAA;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAeLW,OAAOe,MAAM;IAEb,IAAMC,aAAc,qBAAGnB;IACvB,uEAAuE;IACvE,sGAAsG;IACtG,IAAImB,WAAWtB,MAAM,GAAG5D,yBAAyB;QAC/C,MAAM,IAAImF,oBAAU,CAClB,gBACA,AAAC,0BAAuHnF,OAA9FkF,WAAWtB,MAAM,EAAC,8EAAoG,OAAxB5D,yBAAwB;IAEpJ;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,IAAMoF,kBAAkBF,WAAW7B,MAAM,CAAC,SAACgC;eAAMvF,yBAAyBwD,GAAG,CAAC+B,MAAM,CAACC,wBAAgB,CAAChC,GAAG,CAAC+B;;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,IAAME,YAAY,AAAC,4BAAkFH,OAAvDA,gBAAgBxD,GAAG,CAAC3B,YAAYG,IAAI,CAAC,OAAM,cAA4FgF,OAAhFA,gBAAgBxD,GAAG,CAAC;eAAM;OAAKxB,IAAI,CAAC,OAAM,wCAGjI,OAHuKgF,gBAClL/B,MAAM,CAAC,SAACgC;eAAMA,MAAM;OACpBzD,GAAG,CAAC,SAACyD;eAAM,AAAC,GAA8BpF,OAA5BA,WAAWoF,IAAG,gBAA4B,OAAdpF,WAAWoF;OACrDjF,IAAI,CAAC;IAER,IAAMoF,QAAQjC,UAAUF,MAAM,CAAC,SAACP;eAAM,CAACG,SAASK,GAAG,CAACR,EAAEC,OAAO;OAAGnB,GAAG,CAAC,SAACkB;eAAMA,EAAEC,OAAO;;IACpF,IAAM0C,QAAwB;QAAE9C,OAAAA;QAAO+C,UAAUzB,WAAWrC,GAAG,CAAC,SAAC+D;mBAAMA,EAAE5C,OAAO;;QAAGyC,OAAAA;QAAOpC,UAAAA;IAAS;IAEnG,IAAMwC,UAAUC,KAAKC,GAAG;IACxBxF,GAAG0B,IAAI,CAAC;IACR,IAAI;YA8C8BE;YA7C3B,mCAAA,4BAAA;;YAAL,QAAK,aAAa8B,+BAAb,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAM+B,MAAN;gBAAyBzF,GAAG0B,IAAI,CAAC,AAAC,sCAAqD,OAAhB/B,WAAW8F;;;YAAlF;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACL,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,IAAMC,UAAU1F,GAAGoB,OAAO,CAAC;QAC3B,IAAMuE,iBAAiB3F,GAAGoB,OAAO,CAAC;QAClC,IAAMwE,mBAAmB5F,GAAGoB,OAAO,CAAC;QACpC,IAAI0B,SAASQ,MAAM,GAAG,GAAG;YACvB,IAAMuC,MAAM7F,GAAGoB,OAAO,CAAC;gBAClB,mCAAA,4BAAA;;gBAAL,QAAK,aAAc0B,6BAAd,UAAA,8BAAA,SAAA,0BAAA,kCAAwB;oBAAxB,IAAMD,OAAN;wBAM6BjB;oBALhC,kFAAkF;oBAClF,qEAAqE;oBACrE8D,QAAQxD,GAAG,CAACW;oBACZgD,IAAI3D,GAAG,CAACW;oBACR8C,eAAezD,GAAG,CAACW;wBACd,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBW,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,UAAN;6BAA2BA,kBAAAA,QAAQkE,MAAM,cAAdlE,sCAAAA,qBAAAA,SAAiB5B,IAAI6C,MAAMsC;;;wBAAtD;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;;gBAPK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAQP;QACA,IAAIxB,WAAWL,MAAM,GAAG,GAAG;YACzB,IAAMyC,SAAS/F,GAAGoB,OAAO,CAAC6D;YAC1B,IAAMe,aAAahG,GAAGoB,OAAO,CAAC;gBACzB,mCAAA,4BAAA;;;oBAAA,IAAM+C,MAAN;wBASH,uFAAuF;oBACvF4B;wBAWgCnE;oBApBhC,IAAMqE,SAASnB,gBAAgBxD,GAAG,CAAC,SAACmE;4BAM3BtB;wBALP,IAAIsB,QAAQ,QAAQ,OAAOtB,IAAI1B,OAAO;wBACtC,IAAIgD,QAAQ,UAAU,OAAOtB,IAAIhB,OAAO;wBACxC,IAAIsC,QAAQ,SAAS,OAAOtB,IAAId,IAAI;wBACpC,mFAAmF;wBACnF,IAAIoC,QAAQ,gBAAgB,OAAOtB,IAAI+B,UAAU;wBACjD,QAAO/B,gBAAAA,IAAIM,IAAI,CAACgB,IAAI,cAAbtB,2BAAAA,gBAAiB;oBAC1B;oBAEA4B,CAAAA,UAAAA,QAAO7D,GAAG,OAAV6D,SAAW,qBAAGE;oBACd,oFAAoF;oBACpF,IAAItD,SAASK,GAAG,CAACmB,IAAI1B,OAAO,GAAG;4BAEGb;wBADhC8D,QAAQxD,GAAG,CAACiC,IAAI1B,OAAO;4BAClB,kCAAA,2BAAA;;4BAAL,QAAK,YAAiBe,6BAAjB,SAAA,6BAAA,QAAA,yBAAA;gCAAA,IAAM5B,UAAN;iCAA2BA,kBAAAA,QAAQkE,MAAM,cAAdlE,sCAAAA,qBAAAA,SAAiB5B,IAAImE,IAAI1B,OAAO,EAAE0C;;;4BAA7D;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;oBACP;oBACAa,WAAW9D,GAAG,CAACiC,IAAI1B,OAAO,EAAE0B,IAAIgC,MAAM,CAACC,KAAK,EAAEjC,IAAIgC,MAAM,CAACE,OAAO,EAAElC,IAAIgC,MAAM,CAACG,IAAI,EAAEnC,IAAI1B,OAAO;oBAC9F,iFAAiF;oBACjF,2EAA2E;oBAC3E,IAAIE,SAASK,GAAG,CAACmB,IAAI1B,OAAO,GAAGkD,eAAezD,GAAG,CAACiC,IAAI1B,OAAO;wBACxD,mCAAA,4BAAA;;wBAAL,QAAK,aAAoB0B,IAAIoC,OAAO,qBAA/B,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAMC,aAAN;4BAAiCZ,iBAAiB1D,GAAG,CAACiC,IAAI1B,OAAO,EAAE+D;;;wBAAnE;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;wBACA,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBhD,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,WAAN;6BAA2BA,iBAAAA,SAAQ6E,KAAK,cAAb7E,qCAAAA,oBAAAA,UAAgB5B,IAAImE,IAAI1B,OAAO,EAAE0B,IAAIuC,SAAS,CAAC9E,SAAQhC,IAAI,CAAC,EAAEuF;;;wBAAzF;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;gBAtBA,QAAK,aAAaxB,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;gBAAA;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAuBP;YACK,mCAAA,4BAAA;;YAAL,QAAK,aAAiBH,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAM5B,WAAN;iBAA2BA,0BAAAA,SAAQ+E,cAAc,cAAtB/E,8CAAAA,6BAAAA,UAAyB5B,IAAImF;;;YAAxD;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACLnF,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAOkF,KAAK;QACZ5G,GAAG0B,IAAI,CAAC;QACR,MAAMkF;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,IAAMC,aAAatB,KAAKC,GAAG,KAAKF;IAChC,IAAMwB,UAAU1H,QAAQY,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,IAAM+G,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAASxH,QAAQS,IAAI,oBAAoBQ,OAAOqG;IAEjE,OAAO;QAAEnG,QAAQiD,WAAWL,MAAM;QAAEC,UAAAA;IAAS;AAC/C;AAEA,yFAAyF;AACzF,wFAAwF;AACxF,SAAS0D,cAAcC,MAAc,EAAEC,KAAa;IAClD,yFAAyF;IACzF,IAAMC,QAAQ,eAACC;eAAkBA,KAAK5G,UAAU,CAAC,aAAa4G,KAAKxH,KAAK,CAAC,KAAKyH,KAAK,CAAC,GAAG,GAAGxH,IAAI,CAAC,OAAOuH,KAAKxH,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAMe,QAAQ,eAAC2G;eAAgB,IAAI3E,IAAI2E,IAAI1H,KAAK,CAAC,KAAKyB,GAAG,CAAC,SAAC+F;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMG,IAAI5G,MAAMsG;IAChB,IAAMO,IAAI7G,MAAMuG;IAChB,IAAMO,UAAU,IAAIjI;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBgI,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAO1F,sBAAK4F;YAAW,IAAIH,EAAEvF,GAAG,CAACF,SAAS4F,KAAKD,QAAQhD,GAAG,CAAC3C;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAayF,EAAEhD,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMzC,OAAN;YAAuB,IAAI,CAAC0F,EAAEzE,GAAG,CAACjB,OAAM2F,QAAQhD,GAAG,CAAC3C;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAM6F,QAAQ,eAAC7F;eAAiBA,QAAQ,UAAU,mBAAmBA,IAAItB,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbsB,IAAIuF,KAAK,CAAC,IAAG,OAAK;;IAC9H,OAAOI,QAAQrE,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAGqE,SAASpG,GAAG,CAACsG,OAAO9H,IAAI,CAAC;AACxE;AAEO,SAAST,KAAKoC,GAAmB;QAqCTrC;IApC7B,IAAMyI,WAAW/H,IAAAA,cAAI,EAAC2B,IAAIW,OAAO,EAAE0F,mBAAS;IAC5CC,IAAAA,iBAAS,EAACF,UAAU;QAAEG,WAAW;IAAK;IACtC,IAAMC,SAASnI,IAAAA,cAAI,EAAC+H,UAAU7I;IAE9B,IAAMgB,KAAK,IAAIkI,wBAAY,CAACD;IAC5BjI,GAAG0B,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvC1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,IAAMyG,UAAU/I,QAAQY,IAAI;IAC5B,IAAMwD,WAAWpE,QAAQY,IAAI;IAC7B,IAAMoI,eAAetG,IAAAA,0BAAgB,EAACL;IACtC,IAAI,AAAC0G,YAAY,QAAQA,YAAYlJ,kBAAoBuE,aAAa,QAAQA,aAAa4E,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYlJ,gBAAgB;YAClDoJ,QAAQC,KAAK,CAAC;QAChB,OAAO;YACL,IAAMZ,UAAUT,cAAczD,qBAAAA,sBAAAA,WAAY,IAAI4E;YAC9CC,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARZ,SAAQ;QACjD;QACA1H,GAAGuI,KAAK;QACRrJ,WAAWuC;QACX,OAAOpC,KAAKoC;IACd;IAEAD,aAAaxB,IAAIyB;IAEjB,wFAAwF;IACxF,2FAA2F;IAC3F,IAAM+G,gBAAgBxB,QAAO5H,WAAAA,QAAQY,IAAI,iCAAZZ,sBAAAA,WAAmC;IAChEY,GAAG0B,IAAI,CAAC,AAAC,yBAA8E,OAAtD+G,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB;IAE9E,IAA6BlJ,aAAAA,UAAUU,IAAIyB,KAAKA,IAAIW,OAAO,GAAnD1B,SAAqBpB,WAArBoB,QAAQ6C,WAAajE,WAAbiE;IAEhB,OAAO;QAAEvD,IAAAA;QAAIyB,KAAAA;QAAKwG,QAAAA;QAAQvH,QAAAA;QAAQ6C,UAAAA;IAAS;AAC7C;AAMO,SAASrE,WAAWuC,GAAmB;IAC5CmH,IAAAA,cAAM,EAAC9I,IAAAA,cAAI,EAAC2B,IAAIW,OAAO,EAAE0F,mBAAS,GAAG;QAAEE,WAAW;QAAMa,OAAO;IAAK;AACtE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size', '_parse_error']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '10';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, \"path\") VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Names what moved between two feature signatures (see config.featureSignature's format:\n// global features, embed provider, then one segment per preset) for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n    }\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","getMeta","open","reconcile","setMeta","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","join","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","map","r","ensureSchema","cfg","exec","activeFeatures","feature","schema","featureSignature","key","row","get","run","n","baseDir","files","listFiles","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","progress","parsedCount","file","fileFeatures","enabledForFile","parseFile","doc","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","SenseError","writableColumns","c","RESERVED_COLUMNS","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","insertBody","values","parseError","search","title","summary","text","presets","presetName","store","extracted","afterReconcile","err","durationMs","prevRaw","prevMax","Number","signatureDiff","before","after","keyOf","part","slice","sig","a","b","changed","val","label","stateDir","STATE_DIR","mkdirSync","recursive","dbPath","DatabaseSync","version","wantFeatures","console","error","close","recordedMaxMs","Math","min","max","rmSync","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAoBpFA;eAAAA;;QAGAC;eAAAA;;QAoRGC;eAAAA;;QA5MAC;eAAAA;;QAbAC;eAAAA;;QAwKAC;eAAAA;;QAtJAC;eAAAA;;QAbAC;eAAAA;;;sBAtFkB;wBACb;0BACQ;wBAEe;wBACjB;uBACI;0BAEN;sBAE8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,6FAA6F;AAC7F,4EAA4E;AAC5E,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAe;AAK7E,IAAMT,cAAc;AAGpB,IAAMC,iBAAiB;AAE9B,8FAA8F;AAC9F,IAAMS,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEA,+FAA+F;AAC/F,SAASC,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,SAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,IAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,IAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,SAACC;mCAASR,OAAOQ,UAAUT;6BAAU,IAAI;oBAC9D;gBACF,EAAE,eAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,IAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAI5B,IAAI0B,KAAKG,GAAG,CAAC,SAACC;eAAMA,EAAE3B,IAAI;;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS4B,aAAaxB,EAAgB,EAAEyB,GAAW;IACjDzB,GAAG0B,IAAI,CAAC;IACR1B,GAAG0B,IAAI,CAAC;IACR,6FAA6F;IAC7F,qFAAqF;IACrF1B,GAAG0B,IAAI,CAAC;IACR1B,GAAG0B,IAAI,CAAC;QACH,kCAAA,2BAAA;;QAAL,QAAK,YAAiBC,IAAAA,uBAAc,EAACF,yBAAhC,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMG,UAAN;YAAsCA,QAAQC,MAAM,CAAC7B;;;QAArD;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAIZ,QAAQY,IAAI,sBAAsB,MAAMT,QAAQS,IAAI,kBAAkBf;IAC1E,IAAIG,QAAQY,IAAI,gBAAgB,MAAMT,QAAQS,IAAI,YAAY8B,IAAAA,0BAAgB,EAACL;AACjF;AAEO,SAASrC,QAAQY,EAAgB,EAAE+B,GAAW;IACnD,IAAMC,MAAMhC,GAAGoB,OAAO,CAAC,wCAAwCa,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI3B,KAAK,GAAG;AAC3B;AAEO,SAASd,QAAQS,EAAgB,EAAE+B,GAAW,EAAE1B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCc,GAAG,CAACH;QACjD;IACF;IACA/B,GAAGoB,OAAO,CAAC,qGAAqGc,GAAG,CAACH,KAAK1B;AAC3H;AAEO,SAASlB,SAASa,EAAgB;IACvC,IAAMgC,MAAMhC,GAAGoB,OAAO,CAAC,yCAAyCa,GAAG;IACnE,OAAOD,IAAIG,CAAC;AACd;AAEO,SAAS7C,UAAUU,EAAgB,EAAEyB,GAAW,EAAEW,OAAe;IACtE,IAAMC,QAAQC,IAAAA,iBAAS,EAACb,KAAKW;IAC7B,IAAMG,aAAa,IAAI9C,IAAI4C,MAAMf,GAAG,CAAC,SAACkB;eAAMA,EAAEC,OAAO;;IAErD,IAAMC,eAAe1C,GAAGoB,OAAO,CAAC,qDAAqDC,GAAG;IAKxF,IAAMsB,WAAW,IAAIC,IAAIF,aAAapB,GAAG,CAAC,SAACC;eAAM;YAACA,EAAEsB,IAAI;YAAEtB;SAAE;;IAC5D,IAAMuB,WAAWJ,aAAaK,MAAM,CAAC,SAACxB;eAAM,CAACgB,WAAWS,GAAG,CAACzB,EAAEsB,IAAI;OAAGvB,GAAG,CAAC,SAACC;eAAMA,EAAEsB,IAAI;;IAEtF,IAAMI,YAAYZ,MAAMU,MAAM,CAAC,SAACP;QAC9B,IAAMR,MAAMW,SAASV,GAAG,CAACO,EAAEC,OAAO;QAClC,OAAO,CAACT,OAAOA,IAAIkB,MAAM,KAAKV,EAAEW,OAAO,IAAInB,IAAIoB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE5C,QAAQ;QAAG6C,UAAU,EAAE;IAAC;IAEtF,IAAMC,WAAW7B,IAAAA,uBAAc,EAACF;IAChC,IAAMgC,cAAcvC,WAAWlB;IAC/B,IAAM0D,aAAuB,EAAE;IAC/B,IAAMC,aAA0B,EAAE;IAClC,IAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,IAAMK,SAASC,IAAAA,oBAAQ,EAAC,mBAAmBZ,UAAUK,MAAM;IAC3D,IAAIQ,cAAc;QACb,kCAAA,2BAAA;;;YAAA,IAAMC,OAAN;gBAMHR;YALA,sFAAsF;YACtF,mEAAmE;YACnE,IAAMS,eAAeR,SAAST,MAAM,CAAC,SAACnB;uBAAY,CAACA,QAAQqC,cAAc,IAAIrC,QAAQqC,cAAc,CAACxC,KAAKsC;;YACzG,IAAwCG,aAAAA,IAAAA,iBAAS,EAACH,MAAMC,eAAhDG,MAAgCD,WAAhCC,KAAKZ,AAAUa,eAAiBF,WAA3BX;YACbK,OAAOS,IAAI,CAAC,EAAEP;YACdP,CAAAA,YAAAA,UAASe,IAAI,OAAbf,WAAc,qBAAGa;gBACZ,kCAAA,2BAAA;;gBAAL,QAAK,YAAaG,OAAOC,IAAI,CAACL,IAAIM,IAAI,sBAAjC,UAAA,6BAAA,SAAA,yBAAA,iCAAoC;oBAApC,IAAM1C,MAAN;oBACH,IAAI,CAAC0B,YAAYT,GAAG,CAACjB,MAAM;wBACzB0B,YAAYiB,GAAG,CAAC3C;wBAChB2B,WAAWY,IAAI,CAACvC;oBAClB;gBACF;;gBALK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAML4B,WAAWW,IAAI,CAACH;QAClB;QAdA,QAAK,YAAclB,8BAAd,SAAA,6BAAA,QAAA,yBAAA;;QAAA;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAeLW,OAAOe,MAAM;IAEb,IAAMC,aAAc,qBAAGnB;IACvB,uEAAuE;IACvE,sGAAsG;IACtG,IAAImB,WAAWtB,MAAM,GAAG5D,yBAAyB;QAC/C,MAAM,IAAImF,oBAAU,CAClB,gBACA,AAAC,0BAAuHnF,OAA9FkF,WAAWtB,MAAM,EAAC,8EAAoG,OAAxB5D,yBAAwB;IAEpJ;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,IAAMoF,kBAAkBF,WAAW7B,MAAM,CAAC,SAACgC;eAAMvF,yBAAyBwD,GAAG,CAAC+B,MAAM,CAACC,wBAAgB,CAAChC,GAAG,CAAC+B;;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,IAAME,YAAY,AAAC,4BAAkFH,OAAvDA,gBAAgBxD,GAAG,CAAC3B,YAAYG,IAAI,CAAC,OAAM,cAA4FgF,OAAhFA,gBAAgBxD,GAAG,CAAC;eAAM;OAAKxB,IAAI,CAAC,OAAM,wCAGjI,OAHuKgF,gBAClL/B,MAAM,CAAC,SAACgC;eAAMA,MAAM;OACpBzD,GAAG,CAAC,SAACyD;eAAM,AAAC,GAA8BpF,OAA5BA,WAAWoF,IAAG,gBAA4B,OAAdpF,WAAWoF;OACrDjF,IAAI,CAAC;IAER,IAAMoF,QAAQjC,UAAUF,MAAM,CAAC,SAACP;eAAM,CAACG,SAASK,GAAG,CAACR,EAAEC,OAAO;OAAGnB,GAAG,CAAC,SAACkB;eAAMA,EAAEC,OAAO;;IACpF,IAAM0C,QAAwB;QAAE9C,OAAAA;QAAO+C,UAAUzB,WAAWrC,GAAG,CAAC,SAAC+D;mBAAMA,EAAE5C,OAAO;;QAAGyC,OAAAA;QAAOpC,UAAAA;IAAS;IAEnG,IAAMwC,UAAUC,KAAKC,GAAG;IACxBxF,GAAG0B,IAAI,CAAC;IACR,IAAI;YA8C8BE;YA7C3B,mCAAA,4BAAA;;YAAL,QAAK,aAAa8B,+BAAb,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAM+B,MAAN;gBAAyBzF,GAAG0B,IAAI,CAAC,AAAC,sCAAqD,OAAhB/B,WAAW8F;;;YAAlF;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACL,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,IAAMC,UAAU1F,GAAGoB,OAAO,CAAC;QAC3B,IAAMuE,iBAAiB3F,GAAGoB,OAAO,CAAC;QAClC,IAAMwE,mBAAmB5F,GAAGoB,OAAO,CAAC;QACpC,IAAI0B,SAASQ,MAAM,GAAG,GAAG;YACvB,IAAMuC,MAAM7F,GAAGoB,OAAO,CAAC;gBAClB,mCAAA,4BAAA;;gBAAL,QAAK,aAAc0B,6BAAd,UAAA,8BAAA,SAAA,0BAAA,kCAAwB;oBAAxB,IAAMD,OAAN;wBAM6BjB;oBALhC,kFAAkF;oBAClF,qEAAqE;oBACrE8D,QAAQxD,GAAG,CAACW;oBACZgD,IAAI3D,GAAG,CAACW;oBACR8C,eAAezD,GAAG,CAACW;wBACd,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBW,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,UAAN;6BAA2BA,kBAAAA,QAAQkE,MAAM,cAAdlE,sCAAAA,qBAAAA,SAAiB5B,IAAI6C,MAAMsC;;;wBAAtD;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;;gBAPK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAQP;QACA,IAAIxB,WAAWL,MAAM,GAAG,GAAG;YACzB,IAAMyC,SAAS/F,GAAGoB,OAAO,CAAC6D;YAC1B,IAAMe,aAAahG,GAAGoB,OAAO,CAAC;gBACzB,mCAAA,4BAAA;;;oBAAA,IAAM+C,MAAN;wBASH,uFAAuF;oBACvF4B;wBAWgCnE;oBApBhC,IAAMqE,SAASnB,gBAAgBxD,GAAG,CAAC,SAACmE;4BAM3BtB;wBALP,IAAIsB,QAAQ,QAAQ,OAAOtB,IAAI1B,OAAO;wBACtC,IAAIgD,QAAQ,UAAU,OAAOtB,IAAIhB,OAAO;wBACxC,IAAIsC,QAAQ,SAAS,OAAOtB,IAAId,IAAI;wBACpC,mFAAmF;wBACnF,IAAIoC,QAAQ,gBAAgB,OAAOtB,IAAI+B,UAAU;wBACjD,QAAO/B,gBAAAA,IAAIM,IAAI,CAACgB,IAAI,cAAbtB,2BAAAA,gBAAiB;oBAC1B;oBAEA4B,CAAAA,UAAAA,QAAO7D,GAAG,OAAV6D,SAAW,qBAAGE;oBACd,oFAAoF;oBACpF,IAAItD,SAASK,GAAG,CAACmB,IAAI1B,OAAO,GAAG;4BAEGb;wBADhC8D,QAAQxD,GAAG,CAACiC,IAAI1B,OAAO;4BAClB,kCAAA,2BAAA;;4BAAL,QAAK,YAAiBe,6BAAjB,SAAA,6BAAA,QAAA,yBAAA;gCAAA,IAAM5B,UAAN;iCAA2BA,kBAAAA,QAAQkE,MAAM,cAAdlE,sCAAAA,qBAAAA,SAAiB5B,IAAImE,IAAI1B,OAAO,EAAE0C;;;4BAA7D;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;oBACP;oBACAa,WAAW9D,GAAG,CAACiC,IAAI1B,OAAO,EAAE0B,IAAIgC,MAAM,CAACC,KAAK,EAAEjC,IAAIgC,MAAM,CAACE,OAAO,EAAElC,IAAIgC,MAAM,CAACG,IAAI,EAAEnC,IAAI1B,OAAO;oBAC9F,iFAAiF;oBACjF,2EAA2E;oBAC3E,IAAIE,SAASK,GAAG,CAACmB,IAAI1B,OAAO,GAAGkD,eAAezD,GAAG,CAACiC,IAAI1B,OAAO;wBACxD,mCAAA,4BAAA;;wBAAL,QAAK,aAAoB0B,IAAIoC,OAAO,qBAA/B,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAMC,aAAN;4BAAiCZ,iBAAiB1D,GAAG,CAACiC,IAAI1B,OAAO,EAAE+D;;;wBAAnE;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;wBACA,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBhD,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,WAAN;6BAA2BA,iBAAAA,SAAQ6E,KAAK,cAAb7E,qCAAAA,oBAAAA,UAAgB5B,IAAImE,IAAI1B,OAAO,EAAE0B,IAAIuC,SAAS,CAAC9E,SAAQhC,IAAI,CAAC,EAAEuF;;;wBAAzF;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;gBAtBA,QAAK,aAAaxB,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;gBAAA;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAuBP;YACK,mCAAA,4BAAA;;YAAL,QAAK,aAAiBH,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAM5B,WAAN;iBAA2BA,0BAAAA,SAAQ+E,cAAc,cAAtB/E,8CAAAA,6BAAAA,UAAyB5B,IAAImF;;;YAAxD;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACLnF,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAOkF,KAAK;QACZ5G,GAAG0B,IAAI,CAAC;QACR,MAAMkF;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,IAAMC,aAAatB,KAAKC,GAAG,KAAKF;IAChC,IAAMwB,UAAU1H,QAAQY,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,IAAM+G,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAASxH,QAAQS,IAAI,oBAAoBQ,OAAOqG;IAEjE,OAAO;QAAEnG,QAAQiD,WAAWL,MAAM;QAAEC,UAAAA;IAAS;AAC/C;AAEA,yFAAyF;AACzF,wFAAwF;AACxF,SAAS0D,cAAcC,MAAc,EAAEC,KAAa;IAClD,yFAAyF;IACzF,IAAMC,QAAQ,eAACC;eAAkBA,KAAK5G,UAAU,CAAC,aAAa4G,KAAKxH,KAAK,CAAC,KAAKyH,KAAK,CAAC,GAAG,GAAGxH,IAAI,CAAC,OAAOuH,KAAKxH,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAMe,QAAQ,eAAC2G;eAAgB,IAAI3E,IAAI2E,IAAI1H,KAAK,CAAC,KAAKyB,GAAG,CAAC,SAAC+F;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMG,IAAI5G,MAAMsG;IAChB,IAAMO,IAAI7G,MAAMuG;IAChB,IAAMO,UAAU,IAAIjI;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBgI,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAO1F,sBAAK4F;YAAW,IAAIH,EAAEvF,GAAG,CAACF,SAAS4F,KAAKD,QAAQhD,GAAG,CAAC3C;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAayF,EAAEhD,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMzC,OAAN;YAAuB,IAAI,CAAC0F,EAAEzE,GAAG,CAACjB,OAAM2F,QAAQhD,GAAG,CAAC3C;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAM6F,QAAQ,eAAC7F;eAAiBA,QAAQ,UAAU,mBAAmBA,IAAItB,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbsB,IAAIuF,KAAK,CAAC,IAAG,OAAK;;IAC9H,OAAOI,QAAQrE,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAGqE,SAASpG,GAAG,CAACsG,OAAO9H,IAAI,CAAC;AACxE;AAEO,SAAST,KAAKoC,GAAmB;QAqCTrC;IApC7B,IAAMyI,WAAW/H,IAAAA,cAAI,EAAC2B,IAAIW,OAAO,EAAE0F,mBAAS;IAC5CC,IAAAA,iBAAS,EAACF,UAAU;QAAEG,WAAW;IAAK;IACtC,IAAMC,SAASnI,IAAAA,cAAI,EAAC+H,UAAU7I;IAE9B,IAAMgB,KAAK,IAAIkI,wBAAY,CAACD;IAC5BjI,GAAG0B,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvC1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,IAAMyG,UAAU/I,QAAQY,IAAI;IAC5B,IAAMwD,WAAWpE,QAAQY,IAAI;IAC7B,IAAMoI,eAAetG,IAAAA,0BAAgB,EAACL;IACtC,IAAI,AAAC0G,YAAY,QAAQA,YAAYlJ,kBAAoBuE,aAAa,QAAQA,aAAa4E,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYlJ,gBAAgB;YAClDoJ,QAAQC,KAAK,CAAC;QAChB,OAAO;YACL,IAAMZ,UAAUT,cAAczD,qBAAAA,sBAAAA,WAAY,IAAI4E;YAC9CC,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARZ,SAAQ;QACjD;QACA1H,GAAGuI,KAAK;QACRrJ,WAAWuC;QACX,OAAOpC,KAAKoC;IACd;IAEAD,aAAaxB,IAAIyB;IAEjB,wFAAwF;IACxF,2FAA2F;IAC3F,IAAM+G,gBAAgBxB,QAAO5H,WAAAA,QAAQY,IAAI,iCAAZZ,sBAAAA,WAAmC;IAChEY,GAAG0B,IAAI,CAAC,AAAC,yBAA8E,OAAtD+G,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB;IAE9E,IAA6BlJ,aAAAA,UAAUU,IAAIyB,KAAKA,IAAIW,OAAO,GAAnD1B,SAAqBpB,WAArBoB,QAAQ6C,WAAajE,WAAbiE;IAEhB,OAAO;QAAEvD,IAAAA;QAAIyB,KAAAA;QAAKwG,QAAAA;QAAQvH,QAAAA;QAAQ6C,UAAAA;IAAS;AAC7C;AAMO,SAASrE,WAAWuC,GAAmB;IAC5CmH,IAAAA,cAAM,EAAC9I,IAAAA,cAAI,EAAC2B,IAAIW,OAAO,EAAE0F,mBAAS,GAAG;QAAEE,WAAW;QAAMa,OAAO;IAAK;AACtE"}

package/dist/cjs/scan.d.cts

@@ -25,6 +25,8 @@ export interface ParsedDoc {
     };
     extracted: Record<string, unknown>;
 }
+export declare function looksLikeDatetime(value: string): boolean;
+export declare function normalizeDate(value: string): string;
 export declare function parseFile(file: FileStat, extractors?: Feature[]): {
     doc: ParsedDoc;
     warnings: string[];

package/dist/cjs/scan.d.ts

@@ -25,6 +25,8 @@ export interface ParsedDoc {
     };
     extracted: Record<string, unknown>;
 }
+export declare function looksLikeDatetime(value: string): boolean;
+export declare function normalizeDate(value: string): string;
 export declare function parseFile(file: FileStat, extractors?: Feature[]): {
     doc: ParsedDoc;
     warnings: string[];

package/dist/cjs/scan.js

@@ -15,6 +15,12 @@ _export(exports, {
     get listFiles () {
         return listFiles;
     },
+    get looksLikeDatetime () {
+        return looksLikeDatetime;
+    },
+    get normalizeDate () {
+        return normalizeDate;
+    },
     get parseFile () {
         return parseFile;
     },
@@ -32,6 +38,9 @@ function _array_like_to_array(arr, len) {
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
     return arr2;
 }
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
 function _array_without_holes(arr) {
     if (Array.isArray(arr)) return _array_like_to_array(arr);
 }
@@ -43,9 +52,39 @@ function _interop_require_default(obj) {
 function _iterable_to_array(iter) {
     if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter);
 }
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
 function _non_iterable_spread() {
     throw new TypeError("Invalid attempt to spread non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
 }
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
 function _to_consumable_array(arr) {
     return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
 }
@@ -196,13 +235,31 @@ function listFiles(cfg, baseDir) {
     }
     return files;
 }
+// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601
+// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.
+var ISO_DATETIME = /^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?)(Z|[+-]\d{2}(?::?\d{2})?)?$/;
+// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it
+// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.
+var MEANT_AS_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d/;
+function looksLikeDatetime(value) {
+    return MEANT_AS_DATETIME.test(value);
+}
+function normalizeDate(value) {
+    var m = ISO_DATETIME.exec(value);
+    if (m === null) return value;
+    var _m = _sliced_to_array(m, 4), date = _m[1], time = _m[2], zone = _m[3];
+    var digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');
+    var offset = digits === '' ? zone !== null && zone !== void 0 ? zone : '' : digits.length === 3 ? "".concat(digits, ":00") : "".concat(digits.slice(0, 3), ":").concat(digits.slice(3));
+    var normalized = "".concat(date, "T").concat(time).concat(offset);
+    return Number.isNaN(Date.parse(normalized)) ? value : normalized;
+}
 // Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches
 // and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.
 function mapValue(value) {
     if (value === null || value === undefined) return null;
     if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
     if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;
-    if (typeof value === 'string') return value;
+    if (typeof value === 'string') return normalizeDate(value);
     return JSON.stringify(value);
 }
 // The delimiter split is all this package used gray-matter for.
@@ -319,7 +376,11 @@ function parseFile(file) {
                 warnings.push("warning: ".concat(file.relPath, ' has a frontmatter key named "').concat(key, '", which is reserved; ignoring it'));
                 continue;
             }
-            mapped[key] = mapValue(data[key]);
+            var value = mapValue(data[key]);
+            if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {
+                warnings.push("warning: ".concat(file.relPath, ": ").concat(key, " is not a valid date (").concat(value, "), so it is invisible to every date comparison"));
+            }
+            mapped[key] = value;
         }
     } catch (err) {
         _didIteratorError = true;

package/dist/cjs/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","parseFile","toPosixPath","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","relPath","separator","sep","split","join","NO_THROW","throwIfNoEntry","cfg","baseDir","coverage","Map","posixNeeded","presetNames","name","preset","presets","globSync","include","cwd","exclude","matched","set","get","add","embedding","embedEnabled","semanticPresets","filter","presetSemanticEnabled","files","keys","sort","absPath","st","statSync","isFile","embed","some","has","push","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","warnStringifiedKeys","doc","warnings","found","visit","Pair","_key","pair","isCollection","key","BREAK","firstLine","message","parseFrontmatter","parseDocument","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","file","extractors","readFileSync","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QA4CGC;eAAAA;;QAkIAC;eAAAA;;QA3IAC;eAAAA;;;sBA/CiC;wBACvB;qEACC;oBACwB;wBAEc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO1D,IAAMH,mBAAmB,IAAII,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW;AAE5H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,mGAAmG;AACnG,IAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AAaO,SAASb,YAAYc,OAAe;QAAEC,YAAAA,iEAAoBC,aAAG;IAClE,OAAOD,cAAc,OAAOD,QAAQG,KAAK,CAACF,WAAWG,IAAI,CAAC,OAAOJ;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,IAAMK,WAAW;IAAEC,gBAAgB;AAAM;AAElC,SAAStB,UAAUuB,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAW,IAAIC;IACrB,IAAMC,cAAcT,aAAG,KAAK;QACvB,kCAAA,2BAAA;;QAAL,QAAK,YAAcU,IAAAA,qBAAW,EAACL,yBAA1B,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMM,OAAN;YACH,IAAMC,SAASP,IAAIQ,OAAO,CAACF,KAAK;gBAC3B,mCAAA,4BAAA;;gBAAL,QAAK,aAAiBG,IAAAA,gBAAQ,EAACF,OAAOG,OAAO,EAAE;oBAAEC,KAAKV;oBAASW,SAASL,OAAOK,OAAO;gBAAC,uBAAlF,UAAA,8BAAA,SAAA,0BAAA,kCAAsF;oBAAtF,IAAMC,UAAN;wBAESX;oBADZ,IAAMT,UAAUW,cAAczB,YAAYkC,WAAWA;oBACrD,IAAMC,OAAMZ,gBAAAA,SAASa,GAAG,CAACtB,sBAAbS,2BAAAA,gBAAyB,IAAItB;oBACzCkC,IAAIE,GAAG,CAACV;oBACRJ,SAASY,GAAG,CAACrB,SAASqB;gBACxB;;gBALK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAMP;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,2EAA2E;IAC3E,IAAMG,YAAYC,IAAAA,sBAAY,EAAClB;IAC/B,IAAMmB,kBAAkBF,YAAY,IAAIrC,IAAIyB,IAAAA,qBAAW,EAACL,KAAKoB,MAAM,CAAC,SAACd;eAASe,IAAAA,+BAAqB,EAACrB,KAAKM;UAAU;IAEnH,IAAMgB,QAAoB,EAAE;QACvB,mCAAA,4BAAA;;QAAL,QAAK,aAAiB,AAAC,qBAAGpB,SAASqB,IAAI,IAAIC,IAAI,uBAA1C,UAAA,8BAAA,SAAA,0BAAA,kCAA8C;YAA9C,IAAM/B,WAAN;YACH,IAAMgC,UAAU5B,IAAAA,cAAI,EAACI,SAASR,WAAU,sDAAsD;YAC9F,yFAAyF;YACzF,uFAAuF;YACvF,IAAMiC,KAAKC,IAAAA,gBAAQ,EAACF,SAAS3B;YAC7B,IAAI,EAAC4B,eAAAA,yBAAAA,GAAIE,MAAM,KAAI;YACnB,IAAMpB,UAAU,AAAC,qBAAIN,SAASa,GAAG,CAACtB,WAA0B+B,IAAI;YAChE,IAAMK,QAAQV,oBAAoB,QAAQX,QAAQsB,IAAI,CAAC,SAACxB;uBAASa,gBAAgBY,GAAG,CAACzB;;YACrFgB,MAAMU,IAAI,CAAC;gBAAEvC,SAAAA;gBAASgC,SAAAA;gBAASQ,SAASP,GAAGO,OAAO;gBAAEC,MAAMR,GAAGQ,IAAI;gBAAE1B,SAAAA;gBAASqB,OAAAA;YAAM;QACpF;;QATK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAOP;AACT;AAiBA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASa,SAASpD,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOqD,OAAOrD,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOsD,OAAOC,aAAa,CAACvD,SAASqD,OAAOrD,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOwD,KAAKC,SAAS,CAACzD;AACxB;AAEA,gEAAgE;AAChE,SAAS0D,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,IAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKnE,WAAW,OAAO;QAAE6D,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASG,oBAAoB3D,OAAe,EAAE4D,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCC,IAAAA,WAAK,EAACH,KAAK;QACTI,MAAAA,SAAAA,KAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACC,IAAAA,kBAAY,EAACD,KAAKE,GAAG,GAAG,OAAO7E;YACpCuE,QAAQ;YACR,OAAOC,WAAK,CAACM,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIP,OAAOD,SAAStB,IAAI,CAAC,AAAC,YAAmB,OAARvC,SAAQ;AAC/C;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASsE,UAAUC,OAAe;IAChC,OAAOA,QAAQpE,KAAK,CAAC,KAAK,CAAC,EAAE,CAACV,OAAO,CAAC,SAAS;AACjD;AAEA,SAAS+E,iBAAiBxE,OAAe,EAAEoD,EAAU,EAAES,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,IAAMD,MAAMa,IAAAA,mBAAa,EAACrB,IAAI;QAAEsB,UAAU;IAAS;IACnD,IAAMC,UAAUf,IAAIgB,MAAM,CAACjD,MAAM,CAAC,SAACkD;eAAQ,CAACzF,oBAAoBkD,GAAG,CAACuC,IAAIC,IAAI;;IAC5E,IAAIH,QAAQnB,MAAM,GAAG,GAAG;QACtB,IAAMuB,SAASJ,QAAQnB,MAAM,GAAG,IAAI,AAAC,SAA2B,OAAnBmB,QAAQnB,MAAM,GAAG,GAAE,YAAU;QAC1E,IAAMwB,aAAa,AAAC,GAAkCD,OAAhCT,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,GAAW,OAAPQ;QACtDlB,SAAStB,IAAI,CAAC,AAAC,YAA2EyC,OAAhEhF,SAAQ,0DAAmE,OAAXgF;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOrB,IAAIsB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,IAAMG,cAAaV,UAAU,AAACO,IAAcN,OAAO;QACnDV,SAAStB,IAAI,CAAC,AAAC,YAA2EyC,OAAhEhF,SAAQ,0DAAmE,OAAXgF;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAAS1F,WAAW,OAAO;QAAE0F,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,CAAA,OAAOC,qCAAP,SAAOA,KAAG,MAAM,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,IAAMD,cAAa;QACnBnB,SAAStB,IAAI,CAAC,AAAC,YAAsByC,OAAXhF,SAAQ,KAAc,OAAXgF,aAAW;QAChD,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IACArB,oBAAoB3D,SAAS4D,KAAKC;IAClC,OAAO;QAAEoB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEO,SAAS/F,UAAUoG,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMrC,MAAMsC,IAAAA,oBAAY,EAACF,KAAKrD,OAAO,EAAE;IACvC,IAAM6B,WAAqB,EAAE;IAE7B,IAA8Bb,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMmC,UAAYxC,kBAAlBK;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAE6B,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIR,iBAAiBa,KAAKrF,OAAO,EAAEoD,IAAIS,WAAlIoB,OAAqB7B,KAArB6B,MAAMD,aAAe5B,KAAf4B;IACd,IAAMS,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAO5D,IAAI,CAACmD,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMb,MAAN;YACH,IAAIrF,iBAAiBuD,GAAG,CAAC8B,MAAM;gBAC7BP,SAAStB,IAAI,CAAC,AAAC,YAAwD6B,OAA7CiB,KAAKrF,OAAO,EAAC,kCAAoC,OAAJoE,KAAI;gBAC3E;YACF;YACAqB,MAAM,CAACrB,IAAI,GAAG1B,SAASuC,IAAI,CAACb,IAAI;QAClC;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMuB,SAAS;QAAEC,OAAOvG,cAAc4F,KAAKW,KAAK;QAAGC,SAASxG,cAAc4F,KAAKY,OAAO;QAAGC,MAAMnG,UAAU6F;IAAS;IAElH,OAAO;QACL5B,KAAK;YACH5D,SAASqF,KAAKrF,OAAO;YACrBwC,SAAS6C,KAAK7C,OAAO;YACrBC,MAAM4C,KAAK5C,IAAI;YACf1B,SAASsE,KAAKtE,OAAO;YACrBkE,MAAMQ;YACNT,YAAAA;YACAW,QAAAA;YACAI,WAAWL,OAAOM,WAAW,CAACV,WAAW3D,MAAM,CAAC,SAACsE;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAEpF,IAAI;qBAAEoF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYhD,KAAKuC,SAASG;iBAAQ;;QAC1H;QACA9B,UAAAA;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["RESERVED_COLUMNS","listFiles","looksLikeDatetime","normalizeDate","parseFile","toPosixPath","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","relPath","separator","sep","split","join","NO_THROW","throwIfNoEntry","cfg","baseDir","coverage","Map","posixNeeded","presetNames","name","preset","presets","globSync","include","cwd","exclude","matched","set","get","add","embedding","embedEnabled","semanticPresets","filter","presetSemanticEnabled","files","keys","sort","absPath","st","statSync","isFile","embed","some","has","push","mtimeMs","size","ISO_DATETIME","MEANT_AS_DATETIME","test","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","doc","warnings","found","visit","Pair","_key","pair","isCollection","key","BREAK","firstLine","message","parseFrontmatter","parseDocument","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","file","extractors","readFileSync","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QA4CGC;eAAAA;;QAsDAC;eAAAA;;QAMAC;eAAAA;;QA8FAC;eAAAA;;QAnKAC;eAAAA;;;sBA/CiC;wBACvB;qEACC;oBACwB;wBAEc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO1D,IAAML,mBAAmB,IAAIM,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW;AAE5H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,mGAAmG;AACnG,IAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AAaO,SAASb,YAAYc,OAAe;QAAEC,YAAAA,iEAAoBC,aAAG;IAClE,OAAOD,cAAc,OAAOD,QAAQG,KAAK,CAACF,WAAWG,IAAI,CAAC,OAAOJ;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,IAAMK,WAAW;IAAEC,gBAAgB;AAAM;AAElC,SAASxB,UAAUyB,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAW,IAAIC;IACrB,IAAMC,cAAcT,aAAG,KAAK;QACvB,kCAAA,2BAAA;;QAAL,QAAK,YAAcU,IAAAA,qBAAW,EAACL,yBAA1B,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMM,OAAN;YACH,IAAMC,SAASP,IAAIQ,OAAO,CAACF,KAAK;gBAC3B,mCAAA,4BAAA;;gBAAL,QAAK,aAAiBG,IAAAA,gBAAQ,EAACF,OAAOG,OAAO,EAAE;oBAAEC,KAAKV;oBAASW,SAASL,OAAOK,OAAO;gBAAC,uBAAlF,UAAA,8BAAA,SAAA,0BAAA,kCAAsF;oBAAtF,IAAMC,UAAN;wBAESX;oBADZ,IAAMT,UAAUW,cAAczB,YAAYkC,WAAWA;oBACrD,IAAMC,OAAMZ,gBAAAA,SAASa,GAAG,CAACtB,sBAAbS,2BAAAA,gBAAyB,IAAItB;oBACzCkC,IAAIE,GAAG,CAACV;oBACRJ,SAASY,GAAG,CAACrB,SAASqB;gBACxB;;gBALK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAMP;;QARK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,2EAA2E;IAC3E,IAAMG,YAAYC,IAAAA,sBAAY,EAAClB;IAC/B,IAAMmB,kBAAkBF,YAAY,IAAIrC,IAAIyB,IAAAA,qBAAW,EAACL,KAAKoB,MAAM,CAAC,SAACd;eAASe,IAAAA,+BAAqB,EAACrB,KAAKM;UAAU;IAEnH,IAAMgB,QAAoB,EAAE;QACvB,mCAAA,4BAAA;;QAAL,QAAK,aAAiB,AAAC,qBAAGpB,SAASqB,IAAI,IAAIC,IAAI,uBAA1C,UAAA,8BAAA,SAAA,0BAAA,kCAA8C;YAA9C,IAAM/B,WAAN;YACH,IAAMgC,UAAU5B,IAAAA,cAAI,EAACI,SAASR,WAAU,sDAAsD;YAC9F,yFAAyF;YACzF,uFAAuF;YACvF,IAAMiC,KAAKC,IAAAA,gBAAQ,EAACF,SAAS3B;YAC7B,IAAI,EAAC4B,eAAAA,yBAAAA,GAAIE,MAAM,KAAI;YACnB,IAAMpB,UAAU,AAAC,qBAAIN,SAASa,GAAG,CAACtB,WAA0B+B,IAAI;YAChE,IAAMK,QAAQV,oBAAoB,QAAQX,QAAQsB,IAAI,CAAC,SAACxB;uBAASa,gBAAgBY,GAAG,CAACzB;;YACrFgB,MAAMU,IAAI,CAAC;gBAAEvC,SAAAA;gBAASgC,SAAAA;gBAASQ,SAASP,GAAGO,OAAO;gBAAEC,MAAMR,GAAGQ,IAAI;gBAAE1B,SAAAA;gBAASqB,OAAAA;YAAM;QACpF;;QATK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,OAAOP;AACT;AAiBA,iGAAiG;AACjG,mGAAmG;AACnG,IAAMa,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,oBAAoB;AAEnB,SAAS5D,kBAAkBO,KAAa;IAC7C,OAAOqD,kBAAkBC,IAAI,CAACtD;AAChC;AAIO,SAASN,cAAcM,KAAa;IACzC,IAAMuD,IAAIH,aAAaI,IAAI,CAACxD;IAC5B,IAAIuD,MAAM,MAAM,OAAOvD;IACvB,IAA6BuD,sBAAAA,OAApBE,OAAoBF,OAAdG,OAAcH,OAARI,OAAQJ;IAC7B,IAAMK,SAASD,SAAS1D,aAAa0D,SAAS,MAAM,KAAKA,KAAKxD,OAAO,CAAC,KAAK;IAC3E,IAAM0D,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,AAAC,GAAS,OAAPF,QAAO,SAAO,AAAC,GAAwBA,OAAtBA,OAAOG,KAAK,CAAC,GAAG,IAAG,KAAmB,OAAhBH,OAAOG,KAAK,CAAC;IAC3H,IAAMC,aAAa,AAAC,GAAUN,OAARD,MAAK,KAAUI,OAAPH,MAAc,OAAPG;IACrC,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAehE,QAAQgE;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASK,SAASrE,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOsE,OAAOtE,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOiE,OAAOM,aAAa,CAACvE,SAASsE,OAAOtE,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAON,cAAcM;IACpD,OAAOwE,KAAKC,SAAS,CAACzE;AACxB;AAEA,gEAAgE;AAChE,SAAS0E,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,IAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAKjF,WAAW,OAAO;QAAE6E,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBzE,OAAe,EAAE0E,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzCC,IAAAA,WAAK,EAACH,KAAK;QACTI,MAAAA,SAAAA,KAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAACC,IAAAA,kBAAY,EAACD,KAAKE,GAAG,GAAG,OAAO3F;YACpCqF,QAAQ;YACR,OAAOC,WAAK,CAACM,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIP,OAAOD,SAASpC,IAAI,CAAC,AAAC,YAAmB,OAARvC,SAAQ;AAC/C;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAASoF,UAAUC,OAAe;IAChC,OAAOA,QAAQlF,KAAK,CAAC,KAAK,CAAC,EAAE,CAACV,OAAO,CAAC,SAAS;AACjD;AAEA,SAAS6F,iBAAiBtF,OAAe,EAAEoE,EAAU,EAAEO,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,IAAMD,MAAMa,IAAAA,mBAAa,EAACnB,IAAI;QAAEoB,UAAU;IAAS;IACnD,IAAMC,UAAUf,IAAIgB,MAAM,CAAC/D,MAAM,CAAC,SAACgE;eAAQ,CAACvG,oBAAoBkD,GAAG,CAACqD,IAAIC,IAAI;;IAC5E,IAAIH,QAAQrC,MAAM,GAAG,GAAG;QACtB,IAAMyC,SAASJ,QAAQrC,MAAM,GAAG,IAAI,AAAC,SAA2B,OAAnBqC,QAAQrC,MAAM,GAAG,GAAE,YAAU;QAC1E,IAAM0C,aAAa,AAAC,GAAkCD,OAAhCT,UAAUK,OAAO,CAAC,EAAE,CAACJ,OAAO,GAAW,OAAPQ;QACtDlB,SAASpC,IAAI,CAAC,AAAC,YAA2EuD,OAAhE9F,SAAQ,0DAAmE,OAAX8F;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOrB,IAAIsB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,IAAMG,cAAaV,UAAU,AAACO,IAAcN,OAAO;QACnDV,SAASpC,IAAI,CAAC,AAAC,YAA2EuD,OAAhE9F,SAAQ,0DAAmE,OAAX8F;QAC1F,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASxG,WAAW,OAAO;QAAEwG,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,CAAA,OAAOC,qCAAP,SAAOA,KAAG,MAAM,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,IAAMD,cAAa;QACnBnB,SAASpC,IAAI,CAAC,AAAC,YAAsBuD,OAAX9F,SAAQ,KAAc,OAAX8F,aAAW;QAChD,OAAO;YAAEC,MAAM,CAAC;YAAGD,YAAAA;QAAW;IAChC;IACArB,oBAAoBzE,SAAS0E,KAAKC;IAClC,OAAO;QAAEoB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEO,SAAS7G,UAAUkH,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMnC,MAAMoC,IAAAA,oBAAY,EAACF,KAAKnE,OAAO,EAAE;IACvC,IAAM2C,WAAqB,EAAE;IAE7B,IAA8BX,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMiC,UAAYtC,kBAAlBK;IACZ,IAA6BD,OAAAA,OAAO,OAAO;QAAE2B,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIR,iBAAiBa,KAAKnG,OAAO,EAAEoE,IAAIO,WAAlIoB,OAAqB3B,KAArB2B,MAAMD,aAAe1B,KAAf0B;IACd,IAAMS,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAO1E,IAAI,CAACiE,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMb,MAAN;YACH,IAAIrG,iBAAiByD,GAAG,CAAC4C,MAAM;gBAC7BP,SAASpC,IAAI,CAAC,AAAC,YAAwD2C,OAA7CiB,KAAKnG,OAAO,EAAC,kCAAoC,OAAJkF,KAAI;gBAC3E;YACF;YACA,IAAM5F,QAAQqE,SAASoC,IAAI,CAACb,IAAI;YAChC,IAAI,OAAO5F,UAAU,YAAYP,kBAAkBO,UAAUiE,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACpE,SAAS;gBAC5FqF,SAASpC,IAAI,CAAC,AAAC,YAA4B2C,OAAjBiB,KAAKnG,OAAO,EAAC,MAAgCV,OAA5B4F,KAAI,0BAA8B,OAAN5F,OAAM;YAC/E;YACAiH,MAAM,CAACrB,IAAI,GAAG5F;QAChB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAMmH,SAAS;QAAEC,OAAOrH,cAAc0G,KAAKW,KAAK;QAAGC,SAAStH,cAAc0G,KAAKY,OAAO;QAAGC,MAAMjH,UAAU2G;IAAS;IAElH,OAAO;QACL5B,KAAK;YACH1E,SAASmG,KAAKnG,OAAO;YACrBwC,SAAS2D,KAAK3D,OAAO;YACrBC,MAAM0D,KAAK1D,IAAI;YACf1B,SAASoF,KAAKpF,OAAO;YACrBgF,MAAMQ;YACNT,YAAAA;YACAW,QAAAA;YACAI,WAAWL,OAAOM,WAAW,CAACV,WAAWzE,MAAM,CAAC,SAACoF;uBAAMA,EAAEC,OAAO;eAAEC,GAAG,CAAC,SAACF;oBAAeA;uBAAT;oBAACA,EAAElG,IAAI;qBAAEkG,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY9C,KAAKqC,SAASG;iBAAQ;;QAC1H;QACA9B,UAAAA;IACF;AACF"}

package/dist/esm/db.d.ts

@@ -1,7 +1,7 @@
 import { DatabaseSync } from 'node:sqlite';
 import type { Config, ResolvedConfig } from './config.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "9";
+export declare const SCHEMA_VERSION = "10";
 export interface OpenResult {
     db: DatabaseSync;
     cfg: ResolvedConfig;

package/dist/esm/db.js

@@ -21,7 +21,7 @@ const CORE_FRONTMATTER_COLUMNS = new Set([
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '9';
+export const SCHEMA_VERSION = '10';
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 const MAX_FRONTMATTER_COLUMNS = 2000;
 function quoteIdent(name) {

package/dist/esm/db.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size', '_parse_error']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '9';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, \"path\") VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Names what moved between two feature signatures (see config.featureSignature's format:\n// global features, embed provider, then one segment per preset) for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n    }\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","featureSignature","STATE_DIR","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","CORE_FRONTMATTER_COLUMNS","Set","DB_FILENAME","SCHEMA_VERSION","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","map","r","ensureSchema","cfg","exec","feature","schema","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","enabledForFile","doc","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","insertBody","values","parseError","search","title","summary","text","presetName","presets","store","extracted","afterReconcile","err","durationMs","prevRaw","prevMax","Number","signatureDiff","before","after","keyOf","part","slice","sig","a","b","changed","val","label","open","stateDir","recursive","dbPath","version","wantFeatures","console","error","close","clearCache","recordedMaxMs","Math","min","max","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,cAAc;AAC1D,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,QAAQ,QAAQ,gBAAgB;AAEzC,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,YAAY;AAEnE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAe;AAEpF,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,IAAI;AAElC,8FAA8F;AAC9F,MAAMC,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKjB,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASkB,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAI7B,IAAI2B,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAE1B,IAAI;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS2B,aAAaxB,EAAgB,EAAEyB,GAAW;IACjDzB,GAAG0B,IAAI,CAAC,CAAC,qHAAqH,CAAC;IAC/H1B,GAAG0B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,6FAA6F;IAC7F,qFAAqF;IACrF1B,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC;IACR,KAAK,MAAMC,WAAWzC,eAAeuC,KAAME,QAAQC,MAAM,CAAC5B;IAC1D,IAAI6B,QAAQ7B,IAAI,sBAAsB,MAAM8B,QAAQ9B,IAAI,kBAAkBN;IAC1E,IAAImC,QAAQ7B,IAAI,gBAAgB,MAAM8B,QAAQ9B,IAAI,YAAYjB,iBAAiB0C;AACjF;AAEA,OAAO,SAASI,QAAQ7B,EAAgB,EAAE+B,GAAW;IACnD,MAAMC,MAAMhC,GAAGoB,OAAO,CAAC,wCAAwCa,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI3B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASyB,QAAQ9B,EAAgB,EAAE+B,GAAW,EAAE1B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCc,GAAG,CAACH;QACjD;IACF;IACA/B,GAAGoB,OAAO,CAAC,qGAAqGc,GAAG,CAACH,KAAK1B;AAC3H;AAEA,OAAO,SAAS8B,SAASnC,EAAgB;IACvC,MAAMgC,MAAMhC,GAAGoB,OAAO,CAAC,yCAAyCa,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUrC,EAAgB,EAAEyB,GAAW,EAAEa,OAAe;IACtE,MAAMC,QAAQnD,UAAUqC,KAAKa;IAC7B,MAAME,aAAa,IAAIhD,IAAI+C,MAAMjB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe3C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMuB,WAAW,IAAIC,IAAIF,aAAarB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEuB,IAAI;YAAEvB;SAAE;IAC5D,MAAMwB,WAAWJ,aAAaK,MAAM,CAAC,CAACzB,IAAM,CAACiB,WAAWS,GAAG,CAAC1B,EAAEuB,IAAI,GAAGxB,GAAG,CAAC,CAACC,IAAMA,EAAEuB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMT,MAAMY,SAASX,GAAG,CAACQ,EAAEC,OAAO;QAClC,OAAO,CAACV,OAAOA,IAAImB,MAAM,KAAKV,EAAEW,OAAO,IAAIpB,IAAIqB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE7C,QAAQ;QAAG8C,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWvE,eAAeuC;IAChC,MAAMiC,cAAcxC,WAAWlB;IAC/B,MAAM2D,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAAS1E,SAAS,mBAAmB+D,UAAUK,MAAM;IAC3D,IAAIO,cAAc;IAClB,KAAK,MAAMC,QAAQb,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMc,eAAeP,SAAST,MAAM,CAAC,CAACrB,UAAY,CAACA,QAAQsC,cAAc,IAAItC,QAAQsC,cAAc,CAACxC,KAAKsC;QACzG,MAAM,EAAEG,GAAG,EAAEV,UAAUW,YAAY,EAAE,GAAG9E,UAAU0E,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMpC,OAAOuC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACd,YAAYT,GAAG,CAAClB,MAAM;gBACzB2B,YAAYe,GAAG,CAAC1C;gBAChB4B,WAAWU,IAAI,CAACtC;YAClB;QACF;QACA6B,WAAWS,IAAI,CAACH;IAClB;IACAL,OAAOa,MAAM;IAEb,MAAMC,aAAa;WAAIjB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIiB,WAAWpB,MAAM,GAAG5D,yBAAyB;QAC/C,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAE0F,WAAWpB,MAAM,CAAC,0EAA0E,EAAE5D,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiF,kBAAkBD,WAAW3B,MAAM,CAAC,CAAC6B,IAAMtF,yBAAyB0D,GAAG,CAAC4B,MAAM,CAACvF,iBAAiB2D,GAAG,CAAC4B;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBtD,GAAG,CAAC1B,YAAYf,IAAI,CAAC,MAAM,UAAU,EAAE+F,gBAAgBtD,GAAG,CAAC,IAAM,KAAKzC,IAAI,CAAC,MAAM,oCAAoC,EAAE+F,gBAClL5B,MAAM,CAAC,CAAC6B,IAAMA,MAAM,QACpBvD,GAAG,CAAC,CAACuD,IAAM,GAAGjF,WAAWiF,GAAG,YAAY,EAAEjF,WAAWiF,IAAI,EACzDhG,IAAI,CAAC,OAAO;IAEf,MAAMkG,QAAQ7B,UAAUF,MAAM,CAAC,CAACP,IAAM,CAACG,SAASK,GAAG,CAACR,EAAEC,OAAO,GAAGpB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IACpF,MAAMsC,QAAwB;QAAEzC;QAAO0C,UAAUrB,WAAWtC,GAAG,CAAC,CAAC4D,IAAMA,EAAExC,OAAO;QAAGqC;QAAOhC;IAAS;IAEnG,MAAMoC,UAAUC,KAAKC,GAAG;IACxBrF,GAAG0B,IAAI,CAAC;IACR,IAAI;YA8C8BC;QA7ChC,KAAK,MAAM2D,OAAO3B,WAAY3D,GAAG0B,IAAI,CAAC,CAAC,mCAAmC,EAAE9B,WAAW0F,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAUvF,GAAGoB,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMoE,iBAAiBxF,GAAGoB,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMqE,mBAAmBzF,GAAGoB,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAI2B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMmC,MAAM1F,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM0B,QAAQC,SAAU;oBAMKpB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrE4D,QAAQrD,GAAG,CAACY;gBACZ4C,IAAIxD,GAAG,CAACY;gBACR0C,eAAetD,GAAG,CAACY;gBACnB,KAAK,MAAMnB,WAAW8B,UAAU9B,kBAAAA,QAAQgE,MAAM,cAAdhE,sCAAAA,qBAAAA,SAAiB3B,IAAI8C,MAAMkC;YAC7D;QACF;QACA,IAAIpB,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMqC,SAAS5F,GAAGoB,OAAO,CAAC0D;YAC1B,MAAMe,aAAa7F,GAAGoB,OAAO,CAAC,CAAC,+HAA+H,CAAC;YAC/J,KAAK,MAAM8C,OAAON,WAAY;oBAqBIjC;gBApBhC,MAAMmE,SAASlB,gBAAgBtD,GAAG,CAAC,CAACgE;wBAM3BpB;oBALP,IAAIoB,QAAQ,QAAQ,OAAOpB,IAAIxB,OAAO;oBACtC,IAAI4C,QAAQ,UAAU,OAAOpB,IAAId,OAAO;oBACxC,IAAIkC,QAAQ,SAAS,OAAOpB,IAAIZ,IAAI;oBACpC,mFAAmF;oBACnF,IAAIgC,QAAQ,gBAAgB,OAAOpB,IAAI6B,UAAU;oBACjD,QAAO7B,gBAAAA,IAAIM,IAAI,CAACc,IAAI,cAAbpB,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0B,OAAO1D,GAAG,IAAI4D;gBACd,oFAAoF;gBACpF,IAAIlD,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG;wBAEGf;oBADhC4D,QAAQrD,GAAG,CAACgC,IAAIxB,OAAO;oBACvB,KAAK,MAAMf,WAAW8B,UAAU9B,mBAAAA,QAAQgE,MAAM,cAAdhE,uCAAAA,sBAAAA,SAAiB3B,IAAIkE,IAAIxB,OAAO,EAAEsC;gBACpE;gBACAa,WAAW3D,GAAG,CAACgC,IAAIxB,OAAO,EAAEwB,IAAI8B,MAAM,CAACC,KAAK,EAAE/B,IAAI8B,MAAM,CAACE,OAAO,EAAEhC,IAAI8B,MAAM,CAACG,IAAI,EAAEjC,IAAIxB,OAAO;gBAC9F,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIE,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG8C,eAAetD,GAAG,CAACgC,IAAIxB,OAAO;gBAC7D,KAAK,MAAM0D,cAAclC,IAAImC,OAAO,CAAEZ,iBAAiBvD,GAAG,CAACgC,IAAIxB,OAAO,EAAE0D;gBACxE,KAAK,MAAMzE,WAAW8B,UAAU9B,iBAAAA,QAAQ2E,KAAK,cAAb3E,qCAAAA,oBAAAA,SAAgB3B,IAAIkE,IAAIxB,OAAO,EAAEwB,IAAIqC,SAAS,CAAC5E,QAAQ9B,IAAI,CAAC,EAAEmF;YAChG;QACF;QACA,KAAK,MAAMrD,WAAW8B,UAAU9B,0BAAAA,QAAQ6E,cAAc,cAAtB7E,8CAAAA,6BAAAA,SAAyB3B,IAAIgF;QAC7DhF,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAO+E,KAAK;QACZzG,GAAG0B,IAAI,CAAC;QACR,MAAM+E;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU9E,QAAQ7B,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4G,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS9E,QAAQ9B,IAAI,oBAAoBQ,OAAOkG;IAEjE,OAAO;QAAEhG,QAAQkD,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,yFAAyF;AACzF,wFAAwF;AACxF,SAASsD,cAAcC,MAAc,EAAEC,KAAa;IAClD,yFAAyF;IACzF,MAAMC,QAAQ,CAACC,OAAkBA,KAAKzG,UAAU,CAAC,aAAayG,KAAKpH,KAAK,CAAC,KAAKqH,KAAK,CAAC,GAAG,GAAGtI,IAAI,CAAC,OAAOqI,KAAKpH,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMc,QAAQ,CAACwG,MAAgB,IAAIvE,IAAIuE,IAAItH,KAAK,CAAC,KAAKwB,GAAG,CAAC,CAAC4F,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMG,IAAIzG,MAAMmG;IAChB,MAAMO,IAAI1G,MAAMoG;IAChB,MAAMO,UAAU,IAAI/H;IACpB,KAAK,MAAM,CAACuC,KAAKyF,IAAI,IAAIF,EAAG,IAAID,EAAEpF,GAAG,CAACF,SAASyF,KAAKD,QAAQ9C,GAAG,CAAC1C;IAChE,KAAK,MAAMA,OAAOsF,EAAE9C,IAAI,GAAI,IAAI,CAAC+C,EAAErE,GAAG,CAAClB,MAAMwF,QAAQ9C,GAAG,CAAC1C;IACzD,MAAM0F,QAAQ,CAAC1F,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAItB,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEsB,IAAIoF,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IAC9H,OAAOI,QAAQjE,IAAI,KAAK,IAAI,aAAa;WAAIiE;KAAQ,CAACjG,GAAG,CAACmG,OAAO5I,IAAI,CAAC;AACxE;AAEA,OAAO,SAAS6I,KAAKjG,GAAmB;QAqCTI;IApC7B,MAAM8F,WAAW9I,KAAK4C,IAAIa,OAAO,EAAEtD;IACnCL,UAAUgJ,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAAShJ,KAAK8I,UAAUlI;IAE9B,MAAMO,KAAK,IAAIlB,aAAa+I;IAC5B7H,GAAG0B,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvC1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,MAAMoG,UAAUjG,QAAQ7B,IAAI;IAC5B,MAAMyD,WAAW5B,QAAQ7B,IAAI;IAC7B,MAAM+H,eAAehJ,iBAAiB0C;IACtC,IAAI,AAACqG,YAAY,QAAQA,YAAYpI,kBAAoB+D,aAAa,QAAQA,aAAasE,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYpI,gBAAgB;YAClDsI,QAAQC,KAAK,CAAC;QAChB,OAAO;YACL,MAAMV,UAAUT,cAAcrD,qBAAAA,sBAAAA,WAAY,IAAIsE;YAC9CC,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEV,QAAQ,oBAAoB,CAAC;QACtE;QACAvH,GAAGkI,KAAK;QACRC,WAAW1G;QACX,OAAOiG,KAAKjG;IACd;IAEAD,aAAaxB,IAAIyB;IAEjB,wFAAwF;IACxF,2FAA2F;IAC3F,MAAM2G,gBAAgBvB,QAAOhF,WAAAA,QAAQ7B,IAAI,iCAAZ6B,sBAAAA,WAAmC;IAChE7B,GAAG0B,IAAI,CAAC,CAAC,sBAAsB,EAAE2G,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB,SAAU;IAExF,MAAM,EAAE1H,MAAM,EAAE8C,QAAQ,EAAE,GAAGnB,UAAUrC,IAAIyB,KAAKA,IAAIa,OAAO;IAE3D,OAAO;QAAEtC;QAAIyB;QAAKoG;QAAQnH;QAAQ8C;IAAS;AAC7C;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAAS2E,WAAW1G,GAAmB;IAC5C7C,OAAOC,KAAK4C,IAAIa,OAAO,EAAEtD,YAAY;QAAE4I,WAAW;QAAMY,OAAO;IAAK;AACtE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { SenseError } from './errors.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ReconcileDelta } from './features/types.ts';\nimport { progress } from './progress.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from './scan.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_size', '_parse_error']);\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '10';\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (rowid, title, summary, text, \"path\") VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Names what moved between two feature signatures (see config.featureSignature's format:\n// global features, embed provider, then one segment per preset) for the rebuild notice.\nfunction signatureDiff(before: string, after: string): string {\n  // Segment keys: `features`, `embed`, `preset:<name>` (config.featureSignature's format).\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n    }\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","featureSignature","STATE_DIR","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","CORE_FRONTMATTER_COLUMNS","Set","DB_FILENAME","SCHEMA_VERSION","MAX_FRONTMATTER_COLUMNS","quoteIdent","name","split","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","map","r","ensureSchema","cfg","exec","feature","schema","getMeta","setMeta","key","row","get","run","docCount","n","reconcile","baseDir","files","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","enabledForFile","doc","fileWarnings","tick","push","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","added","delta","reparsed","d","txStart","Date","now","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","insertBody","values","parseError","search","title","summary","text","presetName","presets","store","extracted","afterReconcile","err","durationMs","prevRaw","prevMax","Number","signatureDiff","before","after","keyOf","part","slice","sig","a","b","changed","val","label","open","stateDir","recursive","dbPath","version","wantFeatures","console","error","close","clearCache","recordedMaxMs","Math","min","max","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,cAAc;AAC1D,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,cAAc,QAAQ,sBAAsB;AAErD,SAASC,QAAQ,QAAQ,gBAAgB;AAEzC,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,YAAY;AAEnE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;CAAe;AAEpF,gFAAgF;AAChF,kEAAkE;AAElE,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAEnC,8FAA8F;AAC9F,MAAMC,0BAA0B;AAUhC,SAASC,WAAWC,IAAY;IAC9B,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKjB,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,+FAA+F;AAC/F,SAASkB,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,MAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAI7B,IAAI2B,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAE1B,IAAI;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS2B,aAAaxB,EAAgB,EAAEyB,GAAW;IACjDzB,GAAG0B,IAAI,CAAC,CAAC,qHAAqH,CAAC;IAC/H1B,GAAG0B,IAAI,CAAC,CAAC,0HAA0H,CAAC;IACpI,6FAA6F;IAC7F,qFAAqF;IACrF1B,GAAG0B,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1G1B,GAAG0B,IAAI,CAAC;IACR,KAAK,MAAMC,WAAWzC,eAAeuC,KAAME,QAAQC,MAAM,CAAC5B;IAC1D,IAAI6B,QAAQ7B,IAAI,sBAAsB,MAAM8B,QAAQ9B,IAAI,kBAAkBN;IAC1E,IAAImC,QAAQ7B,IAAI,gBAAgB,MAAM8B,QAAQ9B,IAAI,YAAYjB,iBAAiB0C;AACjF;AAEA,OAAO,SAASI,QAAQ7B,EAAgB,EAAE+B,GAAW;IACnD,MAAMC,MAAMhC,GAAGoB,OAAO,CAAC,wCAAwCa,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI3B,KAAK,GAAG;AAC3B;AAEA,OAAO,SAASyB,QAAQ9B,EAAgB,EAAE+B,GAAW,EAAE1B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCc,GAAG,CAACH;QACjD;IACF;IACA/B,GAAGoB,OAAO,CAAC,qGAAqGc,GAAG,CAACH,KAAK1B;AAC3H;AAEA,OAAO,SAAS8B,SAASnC,EAAgB;IACvC,MAAMgC,MAAMhC,GAAGoB,OAAO,CAAC,yCAAyCa,GAAG;IACnE,OAAOD,IAAII,CAAC;AACd;AAEA,OAAO,SAASC,UAAUrC,EAAgB,EAAEyB,GAAW,EAAEa,OAAe;IACtE,MAAMC,QAAQnD,UAAUqC,KAAKa;IAC7B,MAAME,aAAa,IAAIhD,IAAI+C,MAAMjB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe3C,GAAGoB,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMuB,WAAW,IAAIC,IAAIF,aAAarB,GAAG,CAAC,CAACC,IAAM;YAACA,EAAEuB,IAAI;YAAEvB;SAAE;IAC5D,MAAMwB,WAAWJ,aAAaK,MAAM,CAAC,CAACzB,IAAM,CAACiB,WAAWS,GAAG,CAAC1B,EAAEuB,IAAI,GAAGxB,GAAG,CAAC,CAACC,IAAMA,EAAEuB,IAAI;IAEtF,MAAMI,YAAYX,MAAMS,MAAM,CAAC,CAACP;QAC9B,MAAMT,MAAMY,SAASX,GAAG,CAACQ,EAAEC,OAAO;QAClC,OAAO,CAACV,OAAOA,IAAImB,MAAM,KAAKV,EAAEW,OAAO,IAAIpB,IAAIqB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE7C,QAAQ;QAAG8C,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWvE,eAAeuC;IAChC,MAAMiC,cAAcxC,WAAWlB;IAC/B,MAAM2D,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAAS1E,SAAS,mBAAmB+D,UAAUK,MAAM;IAC3D,IAAIO,cAAc;IAClB,KAAK,MAAMC,QAAQb,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMc,eAAeP,SAAST,MAAM,CAAC,CAACrB,UAAY,CAACA,QAAQsC,cAAc,IAAItC,QAAQsC,cAAc,CAACxC,KAAKsC;QACzG,MAAM,EAAEG,GAAG,EAAEV,UAAUW,YAAY,EAAE,GAAG9E,UAAU0E,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMpC,OAAOuC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACd,YAAYT,GAAG,CAAClB,MAAM;gBACzB2B,YAAYe,GAAG,CAAC1C;gBAChB4B,WAAWU,IAAI,CAACtC;YAClB;QACF;QACA6B,WAAWS,IAAI,CAACH;IAClB;IACAL,OAAOa,MAAM;IAEb,MAAMC,aAAa;WAAIjB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIiB,WAAWpB,MAAM,GAAG5D,yBAAyB;QAC/C,MAAM,IAAIV,WACR,gBACA,CAAC,uBAAuB,EAAE0F,WAAWpB,MAAM,CAAC,0EAA0E,EAAE5D,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiF,kBAAkBD,WAAW3B,MAAM,CAAC,CAAC6B,IAAMtF,yBAAyB0D,GAAG,CAAC4B,MAAM,CAACvF,iBAAiB2D,GAAG,CAAC4B;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBtD,GAAG,CAAC1B,YAAYf,IAAI,CAAC,MAAM,UAAU,EAAE+F,gBAAgBtD,GAAG,CAAC,IAAM,KAAKzC,IAAI,CAAC,MAAM,oCAAoC,EAAE+F,gBAClL5B,MAAM,CAAC,CAAC6B,IAAMA,MAAM,QACpBvD,GAAG,CAAC,CAACuD,IAAM,GAAGjF,WAAWiF,GAAG,YAAY,EAAEjF,WAAWiF,IAAI,EACzDhG,IAAI,CAAC,OAAO;IAEf,MAAMkG,QAAQ7B,UAAUF,MAAM,CAAC,CAACP,IAAM,CAACG,SAASK,GAAG,CAACR,EAAEC,OAAO,GAAGpB,GAAG,CAAC,CAACmB,IAAMA,EAAEC,OAAO;IACpF,MAAMsC,QAAwB;QAAEzC;QAAO0C,UAAUrB,WAAWtC,GAAG,CAAC,CAAC4D,IAAMA,EAAExC,OAAO;QAAGqC;QAAOhC;IAAS;IAEnG,MAAMoC,UAAUC,KAAKC,GAAG;IACxBrF,GAAG0B,IAAI,CAAC;IACR,IAAI;YA8C8BC;QA7ChC,KAAK,MAAM2D,OAAO3B,WAAY3D,GAAG0B,IAAI,CAAC,CAAC,mCAAmC,EAAE9B,WAAW0F,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAUvF,GAAGoB,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMoE,iBAAiBxF,GAAGoB,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMqE,mBAAmBzF,GAAGoB,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAI2B,SAASQ,MAAM,GAAG,GAAG;YACvB,MAAMmC,MAAM1F,GAAGoB,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAM0B,QAAQC,SAAU;oBAMKpB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrE4D,QAAQrD,GAAG,CAACY;gBACZ4C,IAAIxD,GAAG,CAACY;gBACR0C,eAAetD,GAAG,CAACY;gBACnB,KAAK,MAAMnB,WAAW8B,UAAU9B,kBAAAA,QAAQgE,MAAM,cAAdhE,sCAAAA,qBAAAA,SAAiB3B,IAAI8C,MAAMkC;YAC7D;QACF;QACA,IAAIpB,WAAWL,MAAM,GAAG,GAAG;YACzB,MAAMqC,SAAS5F,GAAGoB,OAAO,CAAC0D;YAC1B,MAAMe,aAAa7F,GAAGoB,OAAO,CAAC,CAAC,+HAA+H,CAAC;YAC/J,KAAK,MAAM8C,OAAON,WAAY;oBAqBIjC;gBApBhC,MAAMmE,SAASlB,gBAAgBtD,GAAG,CAAC,CAACgE;wBAM3BpB;oBALP,IAAIoB,QAAQ,QAAQ,OAAOpB,IAAIxB,OAAO;oBACtC,IAAI4C,QAAQ,UAAU,OAAOpB,IAAId,OAAO;oBACxC,IAAIkC,QAAQ,SAAS,OAAOpB,IAAIZ,IAAI;oBACpC,mFAAmF;oBACnF,IAAIgC,QAAQ,gBAAgB,OAAOpB,IAAI6B,UAAU;oBACjD,QAAO7B,gBAAAA,IAAIM,IAAI,CAACc,IAAI,cAAbpB,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0B,OAAO1D,GAAG,IAAI4D;gBACd,oFAAoF;gBACpF,IAAIlD,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG;wBAEGf;oBADhC4D,QAAQrD,GAAG,CAACgC,IAAIxB,OAAO;oBACvB,KAAK,MAAMf,WAAW8B,UAAU9B,mBAAAA,QAAQgE,MAAM,cAAdhE,uCAAAA,sBAAAA,SAAiB3B,IAAIkE,IAAIxB,OAAO,EAAEsC;gBACpE;gBACAa,WAAW3D,GAAG,CAACgC,IAAIxB,OAAO,EAAEwB,IAAI8B,MAAM,CAACC,KAAK,EAAE/B,IAAI8B,MAAM,CAACE,OAAO,EAAEhC,IAAI8B,MAAM,CAACG,IAAI,EAAEjC,IAAIxB,OAAO;gBAC9F,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIE,SAASK,GAAG,CAACiB,IAAIxB,OAAO,GAAG8C,eAAetD,GAAG,CAACgC,IAAIxB,OAAO;gBAC7D,KAAK,MAAM0D,cAAclC,IAAImC,OAAO,CAAEZ,iBAAiBvD,GAAG,CAACgC,IAAIxB,OAAO,EAAE0D;gBACxE,KAAK,MAAMzE,WAAW8B,UAAU9B,iBAAAA,QAAQ2E,KAAK,cAAb3E,qCAAAA,oBAAAA,SAAgB3B,IAAIkE,IAAIxB,OAAO,EAAEwB,IAAIqC,SAAS,CAAC5E,QAAQ9B,IAAI,CAAC,EAAEmF;YAChG;QACF;QACA,KAAK,MAAMrD,WAAW8B,UAAU9B,0BAAAA,QAAQ6E,cAAc,cAAtB7E,8CAAAA,6BAAAA,SAAyB3B,IAAIgF;QAC7DhF,GAAG0B,IAAI,CAAC;IACV,EAAE,OAAO+E,KAAK;QACZzG,GAAG0B,IAAI,CAAC;QACR,MAAM+E;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU9E,QAAQ7B,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4G,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS9E,QAAQ9B,IAAI,oBAAoBQ,OAAOkG;IAEjE,OAAO;QAAEhG,QAAQkD,WAAWL,MAAM;QAAEC;IAAS;AAC/C;AAEA,yFAAyF;AACzF,wFAAwF;AACxF,SAASsD,cAAcC,MAAc,EAAEC,KAAa;IAClD,yFAAyF;IACzF,MAAMC,QAAQ,CAACC,OAAkBA,KAAKzG,UAAU,CAAC,aAAayG,KAAKpH,KAAK,CAAC,KAAKqH,KAAK,CAAC,GAAG,GAAGtI,IAAI,CAAC,OAAOqI,KAAKpH,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAMc,QAAQ,CAACwG,MAAgB,IAAIvE,IAAIuE,IAAItH,KAAK,CAAC,KAAKwB,GAAG,CAAC,CAAC4F,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMG,IAAIzG,MAAMmG;IAChB,MAAMO,IAAI1G,MAAMoG;IAChB,MAAMO,UAAU,IAAI/H;IACpB,KAAK,MAAM,CAACuC,KAAKyF,IAAI,IAAIF,EAAG,IAAID,EAAEpF,GAAG,CAACF,SAASyF,KAAKD,QAAQ9C,GAAG,CAAC1C;IAChE,KAAK,MAAMA,OAAOsF,EAAE9C,IAAI,GAAI,IAAI,CAAC+C,EAAErE,GAAG,CAAClB,MAAMwF,QAAQ9C,GAAG,CAAC1C;IACzD,MAAM0F,QAAQ,CAAC1F,MAAiBA,QAAQ,UAAU,mBAAmBA,IAAItB,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEsB,IAAIoF,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IAC9H,OAAOI,QAAQjE,IAAI,KAAK,IAAI,aAAa;WAAIiE;KAAQ,CAACjG,GAAG,CAACmG,OAAO5I,IAAI,CAAC;AACxE;AAEA,OAAO,SAAS6I,KAAKjG,GAAmB;QAqCTI;IApC7B,MAAM8F,WAAW9I,KAAK4C,IAAIa,OAAO,EAAEtD;IACnCL,UAAUgJ,UAAU;QAAEC,WAAW;IAAK;IACtC,MAAMC,SAAShJ,KAAK8I,UAAUlI;IAE9B,MAAMO,KAAK,IAAIlB,aAAa+I;IAC5B7H,GAAG0B,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvC1B,GAAG0B,IAAI,CAAC;IACR3B,kBAAkBC;IAElBA,GAAG0B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,MAAMoG,UAAUjG,QAAQ7B,IAAI;IAC5B,MAAMyD,WAAW5B,QAAQ7B,IAAI;IAC7B,MAAM+H,eAAehJ,iBAAiB0C;IACtC,IAAI,AAACqG,YAAY,QAAQA,YAAYpI,kBAAoB+D,aAAa,QAAQA,aAAasE,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAID,YAAY,QAAQA,YAAYpI,gBAAgB;YAClDsI,QAAQC,KAAK,CAAC;QAChB,OAAO;YACL,MAAMV,UAAUT,cAAcrD,qBAAAA,sBAAAA,WAAY,IAAIsE;YAC9CC,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEV,QAAQ,oBAAoB,CAAC;QACtE;QACAvH,GAAGkI,KAAK;QACRC,WAAW1G;QACX,OAAOiG,KAAKjG;IACd;IAEAD,aAAaxB,IAAIyB;IAEjB,wFAAwF;IACxF,2FAA2F;IAC3F,MAAM2G,gBAAgBvB,QAAOhF,WAAAA,QAAQ7B,IAAI,iCAAZ6B,sBAAAA,WAAmC;IAChE7B,GAAG0B,IAAI,CAAC,CAAC,sBAAsB,EAAE2G,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIH,gBAAgB,SAAU;IAExF,MAAM,EAAE1H,MAAM,EAAE8C,QAAQ,EAAE,GAAGnB,UAAUrC,IAAIyB,KAAKA,IAAIa,OAAO;IAE3D,OAAO;QAAEtC;QAAIyB;QAAKoG;QAAQnH;QAAQ8C;IAAS;AAC7C;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAAS2E,WAAW1G,GAAmB;IAC5C7C,OAAOC,KAAK4C,IAAIa,OAAO,EAAEtD,YAAY;QAAE4I,WAAW;QAAMY,OAAO;IAAK;AACtE"}

package/dist/esm/scan.d.ts

@@ -25,6 +25,8 @@ export interface ParsedDoc {
     };
     extracted: Record<string, unknown>;
 }
+export declare function looksLikeDatetime(value: string): boolean;
+export declare function normalizeDate(value: string): string;
 export declare function parseFile(file: FileStat, extractors?: Feature[]): {
     doc: ParsedDoc;
     warnings: string[];

package/dist/esm/scan.js

@@ -91,13 +91,33 @@ export function listFiles(cfg, baseDir) {
     }
     return files;
 }
+// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601
+// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.
+const ISO_DATETIME = /^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?)(Z|[+-]\d{2}(?::?\d{2})?)?$/;
+// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it
+// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.
+const MEANT_AS_DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d/;
+export function looksLikeDatetime(value) {
+    return MEANT_AS_DATETIME.test(value);
+}
+// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still
+// the local date. A shape that is not a real instant is left as written, and stays auditable.
+export function normalizeDate(value) {
+    const m = ISO_DATETIME.exec(value);
+    if (m === null) return value;
+    const [, date, time, zone] = m;
+    const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');
+    const offset = digits === '' ? zone !== null && zone !== void 0 ? zone : '' : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;
+    const normalized = `${date}T${time}${offset}`;
+    return Number.isNaN(Date.parse(normalized)) ? value : normalized;
+}
 // Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches
 // and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.
 function mapValue(value) {
     if (value === null || value === undefined) return null;
     if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
     if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;
-    if (typeof value === 'string') return value;
+    if (typeof value === 'string') return normalizeDate(value);
     return JSON.stringify(value);
 }
 // The delimiter split is all this package used gray-matter for.
@@ -208,7 +228,11 @@ export function parseFile(file, extractors = []) {
             warnings.push(`warning: ${file.relPath} has a frontmatter key named "${key}", which is reserved; ignoring it`);
             continue;
         }
-        mapped[key] = mapValue(data[key]);
+        const value = mapValue(data[key]);
+        if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {
+            warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);
+        }
+        mapped[key] = value;
     }
     // title/summary are plain YAML strings -- whitespace-collapse only;
     // the prose gets the full markdown strip.

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["globSync","readFileSync","statSync","join","sep","removeMarkdown","isCollection","parseDocument","visit","embedEnabled","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","add","embedding","semanticPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","warnStringifiedKeys","doc","warnings","found","Pair","_key","pair","key","BREAK","firstLine","message","parseFrontmatter","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","parseFile","file","extractors","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AAC3D,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AACtC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,SAASC,YAAY,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAG/E,6EAA6E;AAE7E,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAE9H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,mGAAmG;AACnG,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBlB,eAAeiB;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAWA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBvB,GAAG;IAClE,OAAOuB,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWxB,IAAI,CAAC,OAAOuB;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAchC,QAAQ;IAC5B,KAAK,MAAMiC,QAAQ3B,YAAYsB,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWxC,SAASsC,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIrB;YACzC+B,IAAIE,GAAG,CAACT;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMG,YAAYtC,aAAauB;IAC/B,MAAMgB,kBAAkBD,YAAY,IAAIlC,IAAIH,YAAYsB,KAAKiB,MAAM,CAAC,CAACZ,OAAS1B,sBAAsBqB,KAAKK,UAAU;IAEnH,MAAMa,QAAoB,EAAE;IAC5B,KAAK,MAAMxB,WAAW;WAAIQ,SAASiB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlD,KAAK8B,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM4B,KAAKpD,SAASmD,SAASxB;QAC7B,IAAI,EAACyB,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMhB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC0B,IAAI;QAChE,MAAMI,QAAQR,oBAAoB,QAAQT,QAAQkB,IAAI,CAAC,CAACpB,OAASW,gBAAgBU,GAAG,CAACrB;QACrFa,MAAMS,IAAI,CAAC;YAAEjC;YAAS2B;YAASO,SAASN,GAAGM,OAAO;YAAEC,MAAMP,GAAGO,IAAI;YAAEtB;YAASiB;QAAM;IACpF;IACA,OAAON;AACT;AAiBA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASY,SAAS9C,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAO+C,OAAO/C,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOgD,OAAOC,aAAa,CAACjD,SAAS+C,OAAO/C,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOkD,KAAKC,SAAS,CAACnD;AACxB;AAEA,gEAAgE;AAChE,SAASoD,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAK7D,WAAW,OAAO;QAAEuD,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASG,oBAAoBrD,OAAe,EAAEsD,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC1E,MAAMwE,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAAC/E,aAAa+E,KAAKC,GAAG,GAAG,OAAOrE;YACpCiE,QAAQ;YACR,OAAO1E,MAAM+E,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAAS8D,UAAUC,OAAe;IAChC,OAAOA,QAAQ7D,KAAK,CAAC,KAAK,CAAC,EAAE,CAACT,OAAO,CAAC,SAAS;AACjD;AAEA,SAASuE,iBAAiBhE,OAAe,EAAE8C,EAAU,EAAES,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMzE,cAAciE,IAAI;QAAEmB,UAAU;IAAS;IACnD,MAAMC,UAAUZ,IAAIa,MAAM,CAAC5C,MAAM,CAAC,CAAC6C,MAAQ,CAAChF,oBAAoB4C,GAAG,CAACoC,IAAIC,IAAI;IAC5E,IAAIH,QAAQhB,MAAM,GAAG,GAAG;QACtB,MAAMoB,SAASJ,QAAQhB,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEgB,QAAQhB,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMqB,aAAa,GAAGT,UAAUI,OAAO,CAAC,EAAE,CAACH,OAAO,IAAIO,QAAQ;QAC9Df,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuE,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOlB,IAAImB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMG,aAAaT,UAAU,AAACM,IAAcL,OAAO;QACnDR,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuE,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASjF,WAAW,OAAO;QAAEiF,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBhB,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,CAAC,EAAEuE,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAlB,oBAAoBrD,SAASsD,KAAKC;IAClC,OAAO;QAAEiB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMnC,MAAMpE,aAAasG,KAAKlD,OAAO,EAAE;IACvC,MAAM4B,WAAqB,EAAE;IAE7B,MAAM,EAAET,EAAE,EAAEC,MAAMgC,OAAO,EAAE,GAAGrC,iBAAiBC;IAC/C,MAAM,EAAE6B,IAAI,EAAED,UAAU,EAAE,GAAGzB,OAAO,OAAO;QAAE0B,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIP,iBAAiBa,KAAK7E,OAAO,EAAE8C,IAAIS;IAC1I,MAAMyB,SAA0D,CAAC;IAEjE,KAAK,MAAMpB,OAAOqB,OAAOxD,IAAI,CAAC+C,MAAO;QACnC,IAAItF,iBAAiB8C,GAAG,CAAC4B,MAAM;YAC7BL,SAAStB,IAAI,CAAC,CAAC,SAAS,EAAE4C,KAAK7E,OAAO,CAAC,8BAA8B,EAAE4D,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAoB,MAAM,CAACpB,IAAI,GAAGxB,SAASoC,IAAI,CAACZ,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMsB,SAAS;QAAEC,OAAO9F,cAAcmF,KAAKW,KAAK;QAAGC,SAAS/F,cAAcmF,KAAKY,OAAO;QAAGC,MAAM1F,UAAUoF;IAAS;IAElH,OAAO;QACLzB,KAAK;YACHtD,SAAS6E,KAAK7E,OAAO;YACrBkC,SAAS2C,KAAK3C,OAAO;YACrBC,MAAM0C,KAAK1C,IAAI;YACftB,SAASgE,KAAKhE,OAAO;YACrB2D,MAAMQ;YACNT;YACAW;YACAI,WAAWL,OAAOM,WAAW,CAACT,WAAWvD,MAAM,CAAC,CAACiE,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAE7E,IAAI;qBAAE6E,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY7C,KAAKoC,SAASG;iBAAQ;;QAC1H;QACA3B;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { globSync, readFileSync, statSync } from 'node:fs';\nimport { join, sep } from 'node:path';\nimport removeMarkdown from 'remove-markdown';\nimport { isCollection, parseDocument, visit } from 'yaml';\nimport type { Config } from './config.ts';\nimport { embedEnabled, presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Frontmatter keys that would collide with table columns. Exported so db.ts's upsert can tell\n// a feature-owned column (`_rank`) from a parsed one and leave it alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', '_parse_error', 'content', 'links', 'sections']);\n\n// YAML error codes whose recovery is unambiguous, so the parse is accepted rather than\n// quarantined. Only one qualifies: YAML 1.2 reserves `@` and `` ` `` at the start of a plain\n// scalar for future use, so they can never be valid and the text can only be what was typed\n// (`aliases: [@handle]` -> [\"@handle\"]). Every other code has a second reading -- an unquoted\n// `:` swallows the keys after it, an unquoted `[..](..)` drops the URL, a duplicate key picks\n// one value in silence -- so it writes values nobody wrote. See plans/frontmatter-parse-policy.md.\nconst ACCEPTED_YAML_CODES = new Set(['BAD_SCALAR_START']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Keeps URL query strings, asset filenames, and HTML attributes out of the index: rare terms\n// carry high IDF, so they outrank prose. remove-markdown misses wikilinks and tables.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff a model is named and some covering preset has semantic on\n}\n\n// Presets are views, not partitions: they overlap freely, and a file's covering set (not one\n// owner) drives indexing. Globs resolve relative to baseDir; unmatched files are not indexed.\nexport function toPosixPath(relPath: string, separator: string = sep): string {\n  return separator === '\\\\' ? relPath.split(separator).join('/') : relPath;\n}\n\n// Every command pays listFiles before it answers (the freshness check stats each file), so\n// per-file work here is the hottest path in the package. Everything derivable from the config\n// alone is computed once, above the loop.\nconst NO_THROW = { throwIfNoEntry: false } as const;\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  const posixNeeded = sep === '\\\\';\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    for (const matched of globSync(preset.include, { cwd: baseDir, exclude: preset.exclude })) {\n      const relPath = posixNeeded ? toPosixPath(matched) : matched;\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  // Which presets want vectors is a property of the config, not of any file.\n  const embedding = embedEnabled(cfg);\n  const semanticPresets = embedding ? new Set(presetNames(cfg).filter((name) => presetSemanticEnabled(cfg, name))) : null;\n\n  const files: FileStat[] = [];\n  for (const relPath of [...coverage.keys()].sort()) {\n    const absPath = join(baseDir, relPath); // join re-applies the platform separator for fs calls\n    // node:fs glob matches directories and dangling symlinks; fast-glob returned neither, so\n    // one stat filters both back out (throwIfNoEntry keeps a dangling link from throwing).\n    const st = statSync(absPath, NO_THROW);\n    if (!st?.isFile()) continue;\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = semanticPresets !== null && presets.some((name) => semanticPresets.has(name));\n    files.push({ relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed });\n  }\n  return files;\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // NULL when the frontmatter parsed, the first YAML message otherwise. In the row rather than\n  // a side table so `SELECT *` and any `IS NULL` investigation trip over it without being asked.\n  parseError: string | null;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite's datetime() rejects a colonless offset (`-0800`) and a space separator, which ISO 8601\n// allows and producers emit. A rejected date is invisible, not excluded: every comparison is NULL.\nconst ISO_DATETIME = /^(\\d{4}-\\d{2}-\\d{2})[T ](\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?)(Z|[+-]\\d{2}(?::?\\d{2})?)?$/;\n\n// A value opening `YYYY-MM-DDT` was meant to be a datetime; prose never is. Reported when it\n// cannot be normalized, so a typo surfaces on the next crawl instead of at some later audit.\nconst MEANT_AS_DATETIME = /^\\d{4}-\\d{2}-\\d{2}[T ]\\d/;\n\nexport function looksLikeDatetime(value: string): boolean {\n  return MEANT_AS_DATETIME.test(value);\n}\n\n// Punctuation only, never a timezone conversion: the offset survives, so substr(d,1,10) is still\n// the local date. A shape that is not a real instant is left as written, and stays auditable.\nexport function normalizeDate(value: string): string {\n  const m = ISO_DATETIME.exec(value);\n  if (m === null) return value;\n  const [, date, time, zone] = m;\n  const digits = zone === undefined || zone === 'Z' ? '' : zone.replace(':', '');\n  const offset = digits === '' ? (zone ?? '') : digits.length === 3 ? `${digits}:00` : `${digits.slice(0, 3)}:${digits.slice(3)}`;\n  const normalized = `${date}T${time}${offset}`;\n  return Number.isNaN(Date.parse(normalized)) ? value : normalized;\n}\n\n// Storage class follows the YAML scalar. Booleans store as 1/0, so `WHERE flag = 1` matches\n// and `WHERE flag = 'true'` cannot; `map` prints observed types so the mismatch is visible.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return normalizeDate(value);\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// A well-formed document can still hold a value nobody meant: `created: {{date}}` is valid\n// YAML for a flow map used as a mapping key, so it raises no error and stores\n// {\"{ date }\": null}. No error code can catch that, but yaml notices the stringified key, so\n// this reports it with the path instead (yaml's own warning has none, fires once per document,\n// and is what trains readers to discard stderr).\nfunction warnStringifiedKeys(relPath: string, doc: ReturnType<typeof parseDocument>, warnings: string[]): void {\n  let found = false;\n  // Nested, not top level: `created: {{date}}` puts the collection key one level down, inside\n  // the flow map that `{{...}}` parses as.\n  visit(doc, {\n    Pair(_key, pair) {\n      if (!isCollection(pair.key)) return undefined;\n      found = true;\n      return visit.BREAK;\n    },\n  });\n  // One per file: a template repeats the same mistake on every field it stamps.\n  if (found) warnings.push(`warning: ${relPath} frontmatter has a key that is itself a list or mapping, stored as text; this is usually an unrendered template placeholder like {{date}}`);\n}\n\n// Accept a clean parse, and one whose every error is unambiguous (ACCEPTED_YAML_CODES).\n// Anything else is quarantined: no frontmatter columns at all, and `_parse_error` carries the\n// reason. Recovering it would write values nobody wrote, which is worse than absence because\n// no query can see it. The file is still indexed -- content, links and sections never touch\n// frontmatter -- so a broken note stays searchable while it is being hunted for.\n// yaml's message continues onto a source excerpt, so the first line is the sentence -- minus\n// the colon that introduced the part being dropped.\nfunction firstLine(message: string): string {\n  return message.split('\\n')[0].replace(/:\\s*$/, '');\n}\n\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): { data: Record<string, unknown>; parseError: string | null } {\n  // logLevel silences yaml's own pathless warnings; warnStringifiedKeys re-reports the one\n  // that carries information, with the file it came from.\n  const doc = parseDocument(fm, { logLevel: 'silent' });\n  const refused = doc.errors.filter((err) => !ACCEPTED_YAML_CODES.has(err.code));\n  if (refused.length > 0) {\n    const detail = refused.length > 1 ? ` (and ${refused.length - 1} more)` : '';\n    const parseError = `${firstLine(refused[0].message)}${detail}`;\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    // Reaches here with doc.errors empty: `title: **Bold**` parses, then opens an alias on\n    // materialisation. An empty error list is not a successful parse.\n    const parseError = firstLine((err as Error).message);\n    warnings.push(`warning: ${relPath} frontmatter did not parse, so none of it is indexed: ${parseError}`);\n    return { data: {}, parseError };\n  }\n\n  if (data === null || data === undefined) return { data: {}, parseError: null };\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    const parseError = 'frontmatter is not a key-value mapping';\n    warnings.push(`warning: ${relPath} ${parseError}; none of it is indexed`);\n    return { data: {}, parseError };\n  }\n  warnStringifiedKeys(relPath, doc, warnings);\n  return { data: data as Record<string, unknown>, parseError: null };\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const { data, parseError } = fm === null ? { data: {} as Record<string, unknown>, parseError: null } : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    const value = mapValue(data[key]);\n    if (typeof value === 'string' && looksLikeDatetime(value) && Number.isNaN(Date.parse(value))) {\n      warnings.push(`warning: ${file.relPath}: ${key} is not a valid date (${value}), so it is invisible to every date comparison`);\n    }\n    mapped[key] = value;\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      parseError,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["globSync","readFileSync","statSync","join","sep","removeMarkdown","isCollection","parseDocument","visit","embedEnabled","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","ACCEPTED_YAML_CODES","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","toPosixPath","relPath","separator","split","NO_THROW","throwIfNoEntry","listFiles","cfg","baseDir","coverage","Map","posixNeeded","name","preset","presets","matched","include","cwd","exclude","set","get","add","embedding","semanticPresets","filter","files","keys","sort","absPath","st","isFile","embed","some","has","push","mtimeMs","size","ISO_DATETIME","MEANT_AS_DATETIME","looksLikeDatetime","test","normalizeDate","m","exec","date","time","zone","digits","offset","length","slice","normalized","Number","isNaN","Date","parse","mapValue","BigInt","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","close","index","warnStringifiedKeys","doc","warnings","found","Pair","_key","pair","key","BREAK","firstLine","message","parseFrontmatter","logLevel","refused","errors","err","code","detail","parseError","data","toJS","Array","isArray","parseFile","file","extractors","content","mapped","Object","search","title","summary","text","extracted","fromEntries","f","extract","map"],"mappings":"AAAA,SAASA,QAAQ,EAAEC,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AAC3D,SAASC,IAAI,EAAEC,GAAG,QAAQ,YAAY;AACtC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,YAAY,EAAEC,aAAa,EAAEC,KAAK,QAAQ,OAAO;AAE1D,SAASC,YAAY,EAAEC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAG/E,6EAA6E;AAE7E,8FAA8F;AAC9F,oFAAoF;AACpF,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAgB;IAAW;IAAS;CAAW,EAAE;AAE9H,uFAAuF;AACvF,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,mGAAmG;AACnG,MAAMC,sBAAsB,IAAID,IAAI;IAAC;CAAmB;AAExD,SAASE,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBlB,eAAeiB;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAWA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,SAASC,YAAYC,OAAe,EAAEC,YAAoBvB,GAAG;IAClE,OAAOuB,cAAc,OAAOD,QAAQE,KAAK,CAACD,WAAWxB,IAAI,CAAC,OAAOuB;AACnE;AAEA,2FAA2F;AAC3F,8FAA8F;AAC9F,0CAA0C;AAC1C,MAAMG,WAAW;IAAEC,gBAAgB;AAAM;AAEzC,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,MAAMC,cAAchC,QAAQ;IAC5B,KAAK,MAAMiC,QAAQ3B,YAAYsB,KAAM;QACnC,MAAMM,SAASN,IAAIO,OAAO,CAACF,KAAK;QAChC,KAAK,MAAMG,WAAWxC,SAASsC,OAAOG,OAAO,EAAE;YAAEC,KAAKT;YAASU,SAASL,OAAOK,OAAO;QAAC,GAAI;gBAE7ET;YADZ,MAAMR,UAAUU,cAAcX,YAAYe,WAAWA;YACrD,MAAMI,OAAMV,gBAAAA,SAASW,GAAG,CAACnB,sBAAbQ,2BAAAA,gBAAyB,IAAIrB;YACzC+B,IAAIE,GAAG,CAACT;YACRH,SAASU,GAAG,CAAClB,SAASkB;QACxB;IACF;IAEA,2EAA2E;IAC3E,MAAMG,YAAYtC,aAAauB;IAC/B,MAAMgB,kBAAkBD,YAAY,IAAIlC,IAAIH,YAAYsB,KAAKiB,MAAM,CAAC,CAACZ,OAAS1B,sBAAsBqB,KAAKK,UAAU;IAEnH,MAAMa,QAAoB,EAAE;IAC5B,KAAK,MAAMxB,WAAW;WAAIQ,SAASiB,IAAI;KAAG,CAACC,IAAI,GAAI;QACjD,MAAMC,UAAUlD,KAAK8B,SAASP,UAAU,sDAAsD;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,MAAM4B,KAAKpD,SAASmD,SAASxB;QAC7B,IAAI,EAACyB,eAAAA,yBAAAA,GAAIC,MAAM,KAAI;QACnB,MAAMhB,UAAU;eAAKL,SAASW,GAAG,CAACnB;SAAyB,CAAC0B,IAAI;QAChE,MAAMI,QAAQR,oBAAoB,QAAQT,QAAQkB,IAAI,CAAC,CAACpB,OAASW,gBAAgBU,GAAG,CAACrB;QACrFa,MAAMS,IAAI,CAAC;YAAEjC;YAAS2B;YAASO,SAASN,GAAGM,OAAO;YAAEC,MAAMP,GAAGO,IAAI;YAAEtB;YAASiB;QAAM;IACpF;IACA,OAAON;AACT;AAiBA,iGAAiG;AACjG,mGAAmG;AACnG,MAAMY,eAAe;AAErB,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,oBAAoB;AAE1B,OAAO,SAASC,kBAAkBhD,KAAa;IAC7C,OAAO+C,kBAAkBE,IAAI,CAACjD;AAChC;AAEA,iGAAiG;AACjG,8FAA8F;AAC9F,OAAO,SAASkD,cAAclD,KAAa;IACzC,MAAMmD,IAAIL,aAAaM,IAAI,CAACpD;IAC5B,IAAImD,MAAM,MAAM,OAAOnD;IACvB,MAAM,GAAGqD,MAAMC,MAAMC,KAAK,GAAGJ;IAC7B,MAAMK,SAASD,SAAStD,aAAasD,SAAS,MAAM,KAAKA,KAAKpD,OAAO,CAAC,KAAK;IAC3E,MAAMsD,SAASD,WAAW,KAAMD,iBAAAA,kBAAAA,OAAQ,KAAMC,OAAOE,MAAM,KAAK,IAAI,GAAGF,OAAO,GAAG,CAAC,GAAG,GAAGA,OAAOG,KAAK,CAAC,GAAG,GAAG,CAAC,EAAEH,OAAOG,KAAK,CAAC,IAAI;IAC/H,MAAMC,aAAa,GAAGP,KAAK,CAAC,EAAEC,OAAOG,QAAQ;IAC7C,OAAOI,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAACJ,eAAe5D,QAAQ4D;AACxD;AAEA,4FAA4F;AAC5F,4FAA4F;AAC5F,SAASK,SAASjE,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOkE,OAAOlE,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO6D,OAAOM,aAAa,CAACnE,SAASkE,OAAOlE,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOkD,cAAclD;IACpD,OAAOoE,KAAKC,SAAS,CAACrE;AACxB;AAEA,gEAAgE;AAChE,SAASsE,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIZ,KAAK,CAACa,IAAI,CAAC,EAAE,CAACd,MAAM;IACrC,MAAMmB,QAAQD,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACI,SAASA,MAAMC,KAAK,KAAK7E,WAAW,OAAO;QAAEyE,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKjB,KAAK,CAAC,GAAGkB,MAAMC,KAAK;QAAGH,MAAMC,KAAKjB,KAAK,CAACkB,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACnB,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,8EAA8E;AAC9E,6FAA6F;AAC7F,+FAA+F;AAC/F,iDAAiD;AACjD,SAASqB,oBAAoBrE,OAAe,EAAEsE,GAAqC,EAAEC,QAAkB;IACrG,IAAIC,QAAQ;IACZ,4FAA4F;IAC5F,yCAAyC;IACzC1F,MAAMwF,KAAK;QACTG,MAAKC,IAAI,EAAEC,IAAI;YACb,IAAI,CAAC/F,aAAa+F,KAAKC,GAAG,GAAG,OAAOrF;YACpCiF,QAAQ;YACR,OAAO1F,MAAM+F,KAAK;QACpB;IACF;IACA,8EAA8E;IAC9E,IAAIL,OAAOD,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,yIAAyI,CAAC;AACzL;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,6FAA6F;AAC7F,4FAA4F;AAC5F,iFAAiF;AACjF,6FAA6F;AAC7F,oDAAoD;AACpD,SAAS8E,UAAUC,OAAe;IAChC,OAAOA,QAAQ7E,KAAK,CAAC,KAAK,CAAC,EAAE,CAACT,OAAO,CAAC,SAAS;AACjD;AAEA,SAASuF,iBAAiBhF,OAAe,EAAEgE,EAAU,EAAEO,QAAkB;IACvE,yFAAyF;IACzF,wDAAwD;IACxD,MAAMD,MAAMzF,cAAcmF,IAAI;QAAEiB,UAAU;IAAS;IACnD,MAAMC,UAAUZ,IAAIa,MAAM,CAAC5D,MAAM,CAAC,CAAC6D,MAAQ,CAAChG,oBAAoB4C,GAAG,CAACoD,IAAIC,IAAI;IAC5E,IAAIH,QAAQlC,MAAM,GAAG,GAAG;QACtB,MAAMsC,SAASJ,QAAQlC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAEkC,QAAQlC,MAAM,GAAG,EAAE,MAAM,CAAC,GAAG;QAC1E,MAAMuC,aAAa,GAAGT,UAAUI,OAAO,CAAC,EAAE,CAACH,OAAO,IAAIO,QAAQ;QAC9Df,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC;IACJ,IAAI;QACFA,OAAOlB,IAAImB,IAAI;IACjB,EAAE,OAAOL,KAAK;QACZ,uFAAuF;QACvF,kEAAkE;QAClE,MAAMG,aAAaT,UAAU,AAACM,IAAcL,OAAO;QACnDR,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,sDAAsD,EAAEuF,YAAY;QACtG,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IAEA,IAAIC,SAAS,QAAQA,SAASjG,WAAW,OAAO;QAAEiG,MAAM,CAAC;QAAGD,YAAY;IAAK;IAC7E,IAAI,OAAOC,SAAS,YAAYE,MAAMC,OAAO,CAACH,OAAO;QACnD,MAAMD,aAAa;QACnBhB,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAEjC,QAAQ,CAAC,EAAEuF,WAAW,uBAAuB,CAAC;QACxE,OAAO;YAAEC,MAAM,CAAC;YAAGD;QAAW;IAChC;IACAlB,oBAAoBrE,SAASsE,KAAKC;IAClC,OAAO;QAAEiB,MAAMA;QAAiCD,YAAY;IAAK;AACnE;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMjC,MAAMtF,aAAasH,KAAKlE,OAAO,EAAE;IACvC,MAAM4C,WAAqB,EAAE;IAE7B,MAAM,EAAEP,EAAE,EAAEC,MAAM8B,OAAO,EAAE,GAAGnC,iBAAiBC;IAC/C,MAAM,EAAE2B,IAAI,EAAED,UAAU,EAAE,GAAGvB,OAAO,OAAO;QAAEwB,MAAM,CAAC;QAA8BD,YAAY;IAAK,IAAIP,iBAAiBa,KAAK7F,OAAO,EAAEgE,IAAIO;IAC1I,MAAMyB,SAA0D,CAAC;IAEjE,KAAK,MAAMpB,OAAOqB,OAAOxE,IAAI,CAAC+D,MAAO;QACnC,IAAItG,iBAAiB8C,GAAG,CAAC4C,MAAM;YAC7BL,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAE4D,KAAK7F,OAAO,CAAC,8BAA8B,EAAE4E,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACA,MAAMtF,QAAQiE,SAASiC,IAAI,CAACZ,IAAI;QAChC,IAAI,OAAOtF,UAAU,YAAYgD,kBAAkBhD,UAAU6D,OAAOC,KAAK,CAACC,KAAKC,KAAK,CAAChE,SAAS;YAC5FiF,SAAStC,IAAI,CAAC,CAAC,SAAS,EAAE4D,KAAK7F,OAAO,CAAC,EAAE,EAAE4E,IAAI,sBAAsB,EAAEtF,MAAM,8CAA8C,CAAC;QAC9H;QACA0G,MAAM,CAACpB,IAAI,GAAGtF;IAChB;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAM4G,SAAS;QAAEC,OAAO9G,cAAcmG,KAAKW,KAAK;QAAGC,SAAS/G,cAAcmG,KAAKY,OAAO;QAAGC,MAAM1G,UAAUoG;IAAS;IAElH,OAAO;QACLzB,KAAK;YACHtE,SAAS6F,KAAK7F,OAAO;YACrBkC,SAAS2D,KAAK3D,OAAO;YACrBC,MAAM0D,KAAK1D,IAAI;YACftB,SAASgF,KAAKhF,OAAO;YACrB2E,MAAMQ;YACNT;YACAW;YACAI,WAAWL,OAAOM,WAAW,CAACT,WAAWvE,MAAM,CAAC,CAACiF,IAAMA,EAAEC,OAAO,EAAEC,GAAG,CAAC,CAACF;oBAAeA;uBAAT;oBAACA,EAAE7F,IAAI;qBAAE6F,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAY3C,KAAKkC,SAASG;iBAAQ;;QAC1H;QACA3B;IACF;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.11.2",
+  "version": "0.11.4",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",

package/skills/sense/SKILL.md

@@ -97,6 +97,7 @@ WHERE a.dst = ? AND b.dst IS NOT NULL AND b.dst <> a.dst;
 - **Dead links need the attachment filter.** `dst IS NULL` alone is not "broken link": a wikilink to anything that is not markdown (`[[Board.base]]`, `![[Pasted image.png]]`, `[[spec.pdf]]`) can never resolve, because sense indexes markdown and resolution only tries the exact path or `+.md`. Those are out of the index's universe, not broken. On a 1,400-note Obsidian vault the unfiltered query returns 143 rows where 14 are real. Exclude anything carrying a file extension, as in the recipe above, and widen the exclusion if your notes have dotted titles (`[[Node.js]]` carries one too, so a stricter list -- `'*.png'`, `'*.pdf'`, `'*.base'`, and whatever else your vault attaches -- is safer on a tree whose titles use dots). Scope it with `preset_files` as well: template and skill files are full of `[[Note Name]]` examples that are deliberately unresolved.
 - `has(field, value)`: array membership on JSON-array fields, substring on strings, false on NULL. This is the `includes()` convention. Substring means `has(f.status, 'active')` also matches `inactive`; exact scalar match is `f.status = ?`, deliberate substring is `LIKE`, exact array membership is `EXISTS (SELECT 1 FROM json_each(f.tags) WHERE value = ?)`. To aggregate per member instead, use `json_each(frontmatter.<field>)` (above) -- GROUP BY on the raw column splits `["a","b"]` and `["b","a"]` into separate buckets.
 - Date fields are stored as written. Compare through `datetime()`, which normalizes ISO 8601 timezone offsets to UTC: `WHERE datetime(created) >= datetime(?)`. Bare string comparison is only safe when every note uses the same offset.
+- Date spellings SQLite rejects (`-0800`, `-08`, a space separator) are normalized at index time, offset preserved. One it cannot fix is left as written and warned about by path: `datetime()` returns NULL there, so the row is invisible to a date comparison rather than excluded by it. List them with `WHERE d IS NOT NULL AND datetime(d) IS NULL`.
 - **SQLite's `now` is UTC, so any query about "today" needs `'localtime'`.** `date('now')` reads as tomorrow from mid-afternoon onward in the Americas, which silently flips "scheduled today" into "overdue" every evening: write `date('now','localtime')` and `datetime('now','start of day','localtime')`. This only matters where the boundary carries the meaning; a `'-90 day'` window is unaffected by a few hours of skew.
 - To bound what a query puts into context: `snippet()` excerpts just the matching text, `LIMIT` caps row counts, and selecting `path`/`title`/`summary` keeps rows small. `SELECT text FROM content` returns the tree's entire prose (sense warns past 50 KB). Aggregates (`COUNT`, `GROUP BY`) are already bounded. `SELECT * FROM frontmatter` is always safe: prose is not a frontmatter column.