sensemaking 0.6.0 → 0.7.0

package/dist/cjs/db.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["import { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile } from './scan.ts';\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  for (const file of toReparse) {\n    const { doc, warnings: fileWarnings } = parseFile(file, features);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n\n  const allColumns = [...seenColumns];\n  const insertSql = `INSERT OR REPLACE INTO frontmatter (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`;\n\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`.\n    const delBody = db.prepare(`DELETE FROM content WHERE \"path\" = ?`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        del.run(path);\n        delBody.run(path);\n        for (const feature of features) feature.remove?.(db, path);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (title, summary, text, \"path\") VALUES (?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = allColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        insert.run(...values);\n        // Delete-before-insert only for docs that have rows: an FTS5 DELETE by column\n        // cannot use an index and scans the whole table, so running it per doc on a\n        // cold build (empty table, nothing to delete) made the crawl quadratic --\n        // measured 4x time per note-count doubling at 13k/26k notes.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath);\n        }\n        insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name]);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, files);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile: the write transaction for 500 changed\n  // files measures ~5s at 26k notes, so 5s expired exactly at the boundary and queries\n  // racing the watcher got SQLITE_BUSY. 30s bounds the wait at ~3x the largest measured\n  // reconcile; a query that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    db.close();\n    rmSync(stateDir, { recursive: true, force: true });\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Manual reset for a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","docCount","getMeta","open","rebuild","reconcile","setMeta","quoteIdent","name","split","join","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","Set","map","r","ensureSchema","cfg","exec","activeFeatures","feature","schema","featureSignature","key","row","get","run","n","baseDir","files","listFiles","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","file","parseFile","doc","fileWarnings","push","Object","keys","data","add","allColumns","insertSql","col","delBody","del","remove","insert","insertBody","values","search","title","summary","text","store","extracted","afterReconcile","err","stateDir","STATE_DIR","mkdirSync","recursive","dbPath","DatabaseSync","version","wantFeatures","close","rmSync","force"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAEAC;eAAAA;;QAiEGC;eAAAA;;QAbAC;eAAAA;;QAwGAC;eAAAA;;QAkCAC;eAAAA;;QAxHAC;eAAAA;;QAbAC;eAAAA;;;sBAvEkB;wBACb;0BACQ;wBAEe;uBACb;sBAEM;;;;;;;;;;;;;;;;;;;;;;;;;;AAK9B,IAAMP,cAAc;AAEpB,IAAMC,iBAAiB;AAU9B,SAASO,WAAWC,IAAY;IAC9B,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEA,+FAA+F;AAC/F,SAASC,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,SAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,IAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,IAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,SAACC;mCAASR,OAAOQ,UAAUT;6BAAU,IAAI;oBAC9D;gBACF,EAAE,eAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,IAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAIC,IAAIH,KAAKI,GAAG,CAAC,SAACC;eAAMA,EAAE5B,IAAI;;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS6B,aAAazB,EAAgB,EAAE0B,GAAW;IACjD1B,GAAG2B,IAAI,CAAC;IACR3B,GAAG2B,IAAI,CAAC;QACH,kCAAA,2BAAA;;QAAL,QAAK,YAAiBC,IAAAA,uBAAc,EAACF,yBAAhC,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMG,UAAN;YAAsCA,QAAQC,MAAM,CAAC9B;;;QAArD;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAIV,QAAQU,IAAI,sBAAsB,MAAMN,QAAQM,IAAI,kBAAkBZ;IAC1E,IAAIE,QAAQU,IAAI,gBAAgB,MAAMN,QAAQM,IAAI,YAAY+B,IAAAA,0BAAgB,EAACL;AACjF;AAEO,SAASpC,QAAQU,EAAgB,EAAEgC,GAAW;IACnD,IAAMC,MAAMjC,GAAGoB,OAAO,CAAC,wCAAwCc,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI5B,KAAK,GAAG;AAC3B;AAEO,SAASX,QAAQM,EAAgB,EAAEgC,GAAW,EAAE3B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCe,GAAG,CAACH;QACjD;IACF;IACAhC,GAAGoB,OAAO,CAAC,qGAAqGe,GAAG,CAACH,KAAK3B;AAC3H;AAEO,SAAShB,SAASW,EAAgB;IACvC,IAAMiC,MAAMjC,GAAGoB,OAAO,CAAC,yCAAyCc,GAAG;IACnE,OAAOD,IAAIG,CAAC;AACd;AAEO,SAAS3C,UAAUO,EAAgB,EAAE0B,GAAW,EAAEW,OAAe;IACtE,IAAMC,QAAQC,IAAAA,iBAAS,EAACb,KAAKW;IAC7B,IAAMG,aAAa,IAAIlB,IAAIgB,MAAMf,GAAG,CAAC,SAACkB;eAAMA,EAAEC,OAAO;;IAErD,IAAMC,eAAe3C,GAAGoB,OAAO,CAAC,qDAAqDC,GAAG;IAKxF,IAAMuB,WAAW,IAAIC,IAAIF,aAAapB,GAAG,CAAC,SAACC;eAAM;YAACA,EAAEsB,IAAI;YAAEtB;SAAE;;IAC5D,IAAMuB,WAAWJ,aAAaK,MAAM,CAAC,SAACxB;eAAM,CAACgB,WAAWS,GAAG,CAACzB,EAAEsB,IAAI;OAAGvB,GAAG,CAAC,SAACC;eAAMA,EAAEsB,IAAI;;IAEtF,IAAMI,YAAYZ,MAAMU,MAAM,CAAC,SAACP;QAC9B,IAAMR,MAAMW,SAASV,GAAG,CAACO,EAAEC,OAAO;QAClC,OAAO,CAACT,OAAOA,IAAIkB,MAAM,KAAKV,EAAEW,OAAO,IAAInB,IAAIoB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE7C,QAAQ;QAAG8C,UAAU,EAAE;IAAC;IAEtF,IAAMC,WAAW7B,IAAAA,uBAAc,EAACF;IAChC,IAAMgC,cAAcxC,WAAWlB;IAC/B,IAAM2D,aAAuB,EAAE;IAC/B,IAAMC,aAA0B,EAAE;IAClC,IAAMJ,WAAqB,EAAE;QAExB,kCAAA,2BAAA;;QAAL,QAAK,YAAcN,8BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAyB;YAAzB,IAAMW,OAAN;gBAEHL;YADA,IAAwCM,aAAAA,IAAAA,iBAAS,EAACD,MAAMJ,WAAhDM,MAAgCD,WAAhCC,KAAKP,AAAUQ,eAAiBF,WAA3BN;YACbA,CAAAA,YAAAA,UAASS,IAAI,OAAbT,WAAc,qBAAGQ;gBACZ,mCAAA,4BAAA;;gBAAL,QAAK,aAAaE,OAAOC,IAAI,CAACJ,IAAIK,IAAI,sBAAjC,UAAA,8BAAA,SAAA,0BAAA,kCAAoC;oBAApC,IAAMpC,MAAN;oBACH,IAAI,CAAC0B,YAAYT,GAAG,CAACjB,MAAM;wBACzB0B,YAAYW,GAAG,CAACrC;wBAChB2B,WAAWM,IAAI,CAACjC;oBAClB;gBACF;;gBALK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAML4B,WAAWK,IAAI,CAACF;QAClB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,IAAMO,aAAc,qBAAGZ;IACvB,IAAMa,YAAY,AAAC,uCAAwFD,OAAlDA,WAAW/C,GAAG,CAAC5B,YAAYG,IAAI,CAAC,OAAM,cAAiD,OAArCwE,WAAW/C,GAAG,CAAC;eAAM;OAAKzB,IAAI,CAAC,OAAM;IAEhJE,GAAG2B,IAAI,CAAC;IACR,IAAI;YAmC8BE;YAlC3B,mCAAA,4BAAA;;YAAL,QAAK,aAAa8B,+BAAb,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAMa,MAAN;gBAAyBxE,GAAG2B,IAAI,CAAC,AAAC,sCAAqD,OAAhBhC,WAAW6E;;;YAAlF;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACL,8DAA8D;QAC9D,IAAMC,UAAUzE,GAAGoB,OAAO,CAAC;QAC3B,IAAI2B,SAASQ,MAAM,GAAG,GAAG;YACvB,IAAMmB,MAAM1E,GAAGoB,OAAO,CAAC;gBAClB,mCAAA,4BAAA;;gBAAL,QAAK,aAAc2B,6BAAd,UAAA,8BAAA,SAAA,0BAAA,kCAAwB;oBAAxB,IAAMD,OAAN;wBAG6BjB;oBAFhC6C,IAAIvC,GAAG,CAACW;oBACR2B,QAAQtC,GAAG,CAACW;wBACP,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBW,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,UAAN;6BAA2BA,kBAAAA,QAAQ8C,MAAM,cAAd9C,sCAAAA,qBAAAA,SAAiB7B,IAAI8C;;;wBAAhD;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;;gBAJK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAKP;QACA,IAAIc,WAAWL,MAAM,GAAG,GAAG;YACzB,IAAMqB,SAAS5E,GAAGoB,OAAO,CAACmD;YAC1B,IAAMM,aAAa7E,GAAGoB,OAAO,CAAC;gBACzB,mCAAA,4BAAA;;;oBAAA,IAAM2C,MAAN;wBAOHa;wBAUgC/C;oBAhBhC,IAAMiD,SAASR,WAAW/C,GAAG,CAAC,SAACiD;4BAItBT;wBAHP,IAAIS,QAAQ,QAAQ,OAAOT,IAAIrB,OAAO;wBACtC,IAAI8B,QAAQ,UAAU,OAAOT,IAAIX,OAAO;wBACxC,IAAIoB,QAAQ,SAAS,OAAOT,IAAIT,IAAI;wBACpC,QAAOS,gBAAAA,IAAIK,IAAI,CAACI,IAAI,cAAbT,2BAAAA,gBAAiB;oBAC1B;oBACAa,CAAAA,UAAAA,QAAOzC,GAAG,OAAVyC,SAAW,qBAAGE;oBACd,8EAA8E;oBAC9E,4EAA4E;oBAC5E,0EAA0E;oBAC1E,6DAA6D;oBAC7D,IAAIlC,SAASK,GAAG,CAACc,IAAIrB,OAAO,GAAG;4BAEGb;wBADhC4C,QAAQtC,GAAG,CAAC4B,IAAIrB,OAAO;4BAClB,kCAAA,2BAAA;;4BAAL,QAAK,YAAiBe,6BAAjB,SAAA,6BAAA,QAAA,yBAAA;gCAAA,IAAM5B,UAAN;iCAA2BA,kBAAAA,QAAQ8C,MAAM,cAAd9C,sCAAAA,qBAAAA,SAAiB7B,IAAI+D,IAAIrB,OAAO;;;4BAA3D;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;oBACP;oBACAmC,WAAW1C,GAAG,CAAC4B,IAAIgB,MAAM,CAACC,KAAK,EAAEjB,IAAIgB,MAAM,CAACE,OAAO,EAAElB,IAAIgB,MAAM,CAACG,IAAI,EAAEnB,IAAIrB,OAAO;wBAC5E,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBe,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,WAAN;6BAA2BA,iBAAAA,SAAQsD,KAAK,cAAbtD,qCAAAA,oBAAAA,UAAgB7B,IAAI+D,IAAIrB,OAAO,EAAEqB,IAAIqB,SAAS,CAACvD,SAAQjC,IAAI,CAAC;;;wBAAvF;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;gBAlBA,QAAK,aAAagE,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;gBAAA;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAmBP;YACK,mCAAA,4BAAA;;YAAL,QAAK,aAAiBH,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAM5B,WAAN;iBAA2BA,0BAAAA,SAAQwD,cAAc,cAAtBxD,8CAAAA,6BAAAA,UAAyB7B,IAAIsC;;;YAAxD;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACLtC,GAAG2B,IAAI,CAAC;IACV,EAAE,OAAO2D,KAAK;QACZtF,GAAG2B,IAAI,CAAC;QACR,MAAM2D;IACR;IAEA,OAAO;QAAE5E,QAAQkD,WAAWL,MAAM;QAAEC,UAAAA;IAAS;AAC/C;AAEO,SAASjE,KAAKmC,GAAmB;IACtC,IAAM6D,WAAWzF,IAAAA,cAAI,EAAC4B,IAAIW,OAAO,EAAEmD,mBAAS;IAC5CC,IAAAA,iBAAS,EAACF,UAAU;QAAEG,WAAW;IAAK;IACtC,IAAMC,SAAS7F,IAAAA,cAAI,EAACyF,UAAUpG;IAE9B,IAAMa,KAAK,IAAI4F,wBAAY,CAACD;IAC5B3F,GAAG2B,IAAI,CAAC;IACR,sFAAsF;IACtF,qFAAqF;IACrF,sFAAsF;IACtF,0DAA0D;IAC1D3B,GAAG2B,IAAI,CAAC;IACR5B,kBAAkBC;IAElBA,GAAG2B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,IAAMkE,UAAUvG,QAAQU,IAAI;IAC5B,IAAMyD,WAAWnE,QAAQU,IAAI;IAC7B,IAAM8F,eAAe/D,IAAAA,0BAAgB,EAACL;IACtC,IAAI,AAACmE,YAAY,QAAQA,YAAYzG,kBAAoBqE,aAAa,QAAQA,aAAaqC,cAAe;QACxG9F,GAAG+F,KAAK;QACRC,IAAAA,cAAM,EAACT,UAAU;YAAEG,WAAW;YAAMO,OAAO;QAAK;QAChD,OAAO1G,KAAKmC;IACd;IAEAD,aAAazB,IAAI0B;IACjB,IAA6BjC,aAAAA,UAAUO,IAAI0B,KAAKA,IAAIW,OAAO,GAAnD3B,SAAqBjB,WAArBiB,QAAQ8C,WAAa/D,WAAb+D;IAEhB,OAAO;QAAExD,IAAAA;QAAI0B,KAAAA;QAAKiE,QAAAA;QAAQjF,QAAAA;QAAQ8C,UAAAA;IAAS;AAC7C;AAGO,SAAShE,QAAQkC,GAAmB;IACzCsE,IAAAA,cAAM,EAAClG,IAAAA,cAAI,EAAC4B,IAAIW,OAAO,EAAEmD,mBAAS,GAAG;QAAEE,WAAW;QAAMO,OAAO;IAAK;IACpE,OAAO1G,KAAKmC;AACd"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db.ts"],"sourcesContent":["import { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from './config.ts';\nimport { featureSignature, STATE_DIR } from './config.ts';\nimport { activeFeatures } from './features/index.ts';\nimport type { ParsedDoc } from './scan.ts';\nimport { listFiles, parseFile } from './scan.ts';\n\n// Rows -> SQLite: core schema, reconcile loop, has(). Parsing lives in scan.ts;\n// everything beyond frontmatter + content lives in src/features/.\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`.\nexport const SCHEMA_VERSION = '4'; // 4: frontmatter scalars store as INTEGER/REAL/TEXT by YAML type (was: all numbers REAL)\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nfunction quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nfunction registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n}\n\nfunction getColumns(db: DatabaseSync): Set<string> {\n  const rows = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_size\" INTEGER)`);\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, tokenize = 'porter unicode61')`);\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function getMeta(db: DatabaseSync, key: string): string | null {\n  const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport function setMeta(db: DatabaseSync, key: string, value: string | null): void {\n  if (value === null) {\n    db.prepare('DELETE FROM meta WHERE key = ?').run(key);\n    return;\n  }\n  db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, value);\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  for (const file of toReparse) {\n    const { doc, warnings: fileWarnings } = parseFile(file, features);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n\n  const allColumns = [...seenColumns];\n  const insertSql = `INSERT OR REPLACE INTO frontmatter (${allColumns.map(quoteIdent).join(', ')}) VALUES (${allColumns.map(() => '?').join(', ')})`;\n\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`.\n    const delBody = db.prepare(`DELETE FROM content WHERE \"path\" = ?`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        del.run(path);\n        delBody.run(path);\n        for (const feature of features) feature.remove?.(db, path);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(`INSERT INTO content (title, summary, text, \"path\") VALUES (?, ?, ?, ?)`);\n      for (const doc of parsedDocs) {\n        const values = allColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_size') return doc.size;\n          return doc.data[col] ?? null;\n        });\n        insert.run(...values);\n        // Delete-before-insert only for docs that have rows: an FTS5 DELETE by column\n        // cannot use an index and scans the whole table, so running it per doc on a\n        // cold build (empty table, nothing to delete) made the crawl quadratic --\n        // measured 4x time per note-count doubling at 13k/26k notes.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath);\n        }\n        insertBody.run(doc.search.title, doc.search.summary, doc.search.text, doc.relPath);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name]);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, files);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile: the write transaction for 500 changed\n  // files measures ~5s at 26k notes, so 5s expired exactly at the boundary and queries\n  // racing the watcher got SQLITE_BUSY. 30s bounds the wait at ~3x the largest measured\n  // reconcile; a query that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    db.close();\n    rmSync(stateDir, { recursive: true, force: true });\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg);\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings };\n}\n\n// Manual reset for a doubted cache.\nexport function rebuild(cfg: ResolvedConfig): OpenResult {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n  return open(cfg);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","docCount","getMeta","open","rebuild","reconcile","setMeta","quoteIdent","name","split","join","registerFunctions","db","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","getColumns","rows","prepare","all","Set","map","r","ensureSchema","cfg","exec","activeFeatures","feature","schema","featureSignature","key","row","get","run","n","baseDir","files","listFiles","currentSet","f","relPath","existingRows","existing","Map","path","vanished","filter","has","toReparse","_mtime","mtimeMs","_size","size","length","warnings","features","seenColumns","newColumns","parsedDocs","file","parseFile","doc","fileWarnings","push","Object","keys","data","add","allColumns","insertSql","col","delBody","del","remove","insert","insertBody","values","search","title","summary","text","store","extracted","afterReconcile","err","stateDir","STATE_DIR","mkdirSync","recursive","dbPath","DatabaseSync","version","wantFeatures","close","rmSync","force"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAEAC;eAAAA;;QAiEGC;eAAAA;;QAbAC;eAAAA;;QAwGAC;eAAAA;;QAkCAC;eAAAA;;QAxHAC;eAAAA;;QAbAC;eAAAA;;;sBAvEkB;wBACb;0BACQ;wBAEe;uBACb;sBAEM;;;;;;;;;;;;;;;;;;;;;;;;;;AAK9B,IAAMP,cAAc;AAEpB,IAAMC,iBAAiB,KAAK,yFAAyF;AAU5H,SAASO,WAAWC,IAAY;IAC9B,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEA,+FAA+F;AAC/F,SAASC,kBAAkBC,EAAgB;IACzCA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,SAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,IAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,IAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,SAACC;mCAASR,OAAOQ,UAAUT;6BAAU,IAAI;oBAC9D;gBACF,EAAE,eAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;AACF;AAEA,SAASW,WAAWlB,EAAgB;IAClC,IAAMmB,OAAOnB,GAAGoB,OAAO,CAAC,kCAAkCC,GAAG;IAC7D,OAAO,IAAIC,IAAIH,KAAKI,GAAG,CAAC,SAACC;eAAMA,EAAE5B,IAAI;;AACvC;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAS6B,aAAazB,EAAgB,EAAE0B,GAAW;IACjD1B,GAAG2B,IAAI,CAAC;IACR3B,GAAG2B,IAAI,CAAC;QACH,kCAAA,2BAAA;;QAAL,QAAK,YAAiBC,IAAAA,uBAAc,EAACF,yBAAhC,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMG,UAAN;YAAsCA,QAAQC,MAAM,CAAC9B;;;QAArD;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAIV,QAAQU,IAAI,sBAAsB,MAAMN,QAAQM,IAAI,kBAAkBZ;IAC1E,IAAIE,QAAQU,IAAI,gBAAgB,MAAMN,QAAQM,IAAI,YAAY+B,IAAAA,0BAAgB,EAACL;AACjF;AAEO,SAASpC,QAAQU,EAAgB,EAAEgC,GAAW;IACnD,IAAMC,MAAMjC,GAAGoB,OAAO,CAAC,wCAAwCc,GAAG,CAACF;IACnE,OAAOC,MAAMA,IAAI5B,KAAK,GAAG;AAC3B;AAEO,SAASX,QAAQM,EAAgB,EAAEgC,GAAW,EAAE3B,KAAoB;IACzE,IAAIA,UAAU,MAAM;QAClBL,GAAGoB,OAAO,CAAC,kCAAkCe,GAAG,CAACH;QACjD;IACF;IACAhC,GAAGoB,OAAO,CAAC,qGAAqGe,GAAG,CAACH,KAAK3B;AAC3H;AAEO,SAAShB,SAASW,EAAgB;IACvC,IAAMiC,MAAMjC,GAAGoB,OAAO,CAAC,yCAAyCc,GAAG;IACnE,OAAOD,IAAIG,CAAC;AACd;AAEO,SAAS3C,UAAUO,EAAgB,EAAE0B,GAAW,EAAEW,OAAe;IACtE,IAAMC,QAAQC,IAAAA,iBAAS,EAACb,KAAKW;IAC7B,IAAMG,aAAa,IAAIlB,IAAIgB,MAAMf,GAAG,CAAC,SAACkB;eAAMA,EAAEC,OAAO;;IAErD,IAAMC,eAAe3C,GAAGoB,OAAO,CAAC,qDAAqDC,GAAG;IAKxF,IAAMuB,WAAW,IAAIC,IAAIF,aAAapB,GAAG,CAAC,SAACC;eAAM;YAACA,EAAEsB,IAAI;YAAEtB;SAAE;;IAC5D,IAAMuB,WAAWJ,aAAaK,MAAM,CAAC,SAACxB;eAAM,CAACgB,WAAWS,GAAG,CAACzB,EAAEsB,IAAI;OAAGvB,GAAG,CAAC,SAACC;eAAMA,EAAEsB,IAAI;;IAEtF,IAAMI,YAAYZ,MAAMU,MAAM,CAAC,SAACP;QAC9B,IAAMR,MAAMW,SAASV,GAAG,CAACO,EAAEC,OAAO;QAClC,OAAO,CAACT,OAAOA,IAAIkB,MAAM,KAAKV,EAAEW,OAAO,IAAInB,IAAIoB,KAAK,KAAKZ,EAAEa,IAAI;IACjE;IAEA,IAAIP,SAASQ,MAAM,KAAK,KAAKL,UAAUK,MAAM,KAAK,GAAG,OAAO;QAAE7C,QAAQ;QAAG8C,UAAU,EAAE;IAAC;IAEtF,IAAMC,WAAW7B,IAAAA,uBAAc,EAACF;IAChC,IAAMgC,cAAcxC,WAAWlB;IAC/B,IAAM2D,aAAuB,EAAE;IAC/B,IAAMC,aAA0B,EAAE;IAClC,IAAMJ,WAAqB,EAAE;QAExB,kCAAA,2BAAA;;QAAL,QAAK,YAAcN,8BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAyB;YAAzB,IAAMW,OAAN;gBAEHL;YADA,IAAwCM,aAAAA,IAAAA,iBAAS,EAACD,MAAMJ,WAAhDM,MAAgCD,WAAhCC,KAAKP,AAAUQ,eAAiBF,WAA3BN;YACbA,CAAAA,YAAAA,UAASS,IAAI,OAAbT,WAAc,qBAAGQ;gBACZ,mCAAA,4BAAA;;gBAAL,QAAK,aAAaE,OAAOC,IAAI,CAACJ,IAAIK,IAAI,sBAAjC,UAAA,8BAAA,SAAA,0BAAA,kCAAoC;oBAApC,IAAMpC,MAAN;oBACH,IAAI,CAAC0B,YAAYT,GAAG,CAACjB,MAAM;wBACzB0B,YAAYW,GAAG,CAACrC;wBAChB2B,WAAWM,IAAI,CAACjC;oBAClB;gBACF;;gBALK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAML4B,WAAWK,IAAI,CAACF;QAClB;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAYL,IAAMO,aAAc,qBAAGZ;IACvB,IAAMa,YAAY,AAAC,uCAAwFD,OAAlDA,WAAW/C,GAAG,CAAC5B,YAAYG,IAAI,CAAC,OAAM,cAAiD,OAArCwE,WAAW/C,GAAG,CAAC;eAAM;OAAKzB,IAAI,CAAC,OAAM;IAEhJE,GAAG2B,IAAI,CAAC;IACR,IAAI;YAmC8BE;YAlC3B,mCAAA,4BAAA;;YAAL,QAAK,aAAa8B,+BAAb,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAMa,MAAN;gBAAyBxE,GAAG2B,IAAI,CAAC,AAAC,sCAAqD,OAAhBhC,WAAW6E;;;YAAlF;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACL,8DAA8D;QAC9D,IAAMC,UAAUzE,GAAGoB,OAAO,CAAC;QAC3B,IAAI2B,SAASQ,MAAM,GAAG,GAAG;YACvB,IAAMmB,MAAM1E,GAAGoB,OAAO,CAAC;gBAClB,mCAAA,4BAAA;;gBAAL,QAAK,aAAc2B,6BAAd,UAAA,8BAAA,SAAA,0BAAA,kCAAwB;oBAAxB,IAAMD,OAAN;wBAG6BjB;oBAFhC6C,IAAIvC,GAAG,CAACW;oBACR2B,QAAQtC,GAAG,CAACW;wBACP,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBW,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,UAAN;6BAA2BA,kBAAAA,QAAQ8C,MAAM,cAAd9C,sCAAAA,qBAAAA,SAAiB7B,IAAI8C;;;wBAAhD;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;;gBAJK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAKP;QACA,IAAIc,WAAWL,MAAM,GAAG,GAAG;YACzB,IAAMqB,SAAS5E,GAAGoB,OAAO,CAACmD;YAC1B,IAAMM,aAAa7E,GAAGoB,OAAO,CAAC;gBACzB,mCAAA,4BAAA;;;oBAAA,IAAM2C,MAAN;wBAOHa;wBAUgC/C;oBAhBhC,IAAMiD,SAASR,WAAW/C,GAAG,CAAC,SAACiD;4BAItBT;wBAHP,IAAIS,QAAQ,QAAQ,OAAOT,IAAIrB,OAAO;wBACtC,IAAI8B,QAAQ,UAAU,OAAOT,IAAIX,OAAO;wBACxC,IAAIoB,QAAQ,SAAS,OAAOT,IAAIT,IAAI;wBACpC,QAAOS,gBAAAA,IAAIK,IAAI,CAACI,IAAI,cAAbT,2BAAAA,gBAAiB;oBAC1B;oBACAa,CAAAA,UAAAA,QAAOzC,GAAG,OAAVyC,SAAW,qBAAGE;oBACd,8EAA8E;oBAC9E,4EAA4E;oBAC5E,0EAA0E;oBAC1E,6DAA6D;oBAC7D,IAAIlC,SAASK,GAAG,CAACc,IAAIrB,OAAO,GAAG;4BAEGb;wBADhC4C,QAAQtC,GAAG,CAAC4B,IAAIrB,OAAO;4BAClB,kCAAA,2BAAA;;4BAAL,QAAK,YAAiBe,6BAAjB,SAAA,6BAAA,QAAA,yBAAA;gCAAA,IAAM5B,UAAN;iCAA2BA,kBAAAA,QAAQ8C,MAAM,cAAd9C,sCAAAA,qBAAAA,SAAiB7B,IAAI+D,IAAIrB,OAAO;;;4BAA3D;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;oBACP;oBACAmC,WAAW1C,GAAG,CAAC4B,IAAIgB,MAAM,CAACC,KAAK,EAAEjB,IAAIgB,MAAM,CAACE,OAAO,EAAElB,IAAIgB,MAAM,CAACG,IAAI,EAAEnB,IAAIrB,OAAO;wBAC5E,mCAAA,4BAAA;;wBAAL,QAAK,aAAiBe,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;4BAAA,IAAM5B,WAAN;6BAA2BA,iBAAAA,SAAQsD,KAAK,cAAbtD,qCAAAA,oBAAAA,UAAgB7B,IAAI+D,IAAIrB,OAAO,EAAEqB,IAAIqB,SAAS,CAACvD,SAAQjC,IAAI,CAAC;;;wBAAvF;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;gBACP;gBAlBA,QAAK,aAAagE,+BAAb,UAAA,8BAAA,SAAA,0BAAA;;gBAAA;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAmBP;YACK,mCAAA,4BAAA;;YAAL,QAAK,aAAiBH,6BAAjB,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAM5B,WAAN;iBAA2BA,0BAAAA,SAAQwD,cAAc,cAAtBxD,8CAAAA,6BAAAA,UAAyB7B,IAAIsC;;;YAAxD;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;QACLtC,GAAG2B,IAAI,CAAC;IACV,EAAE,OAAO2D,KAAK;QACZtF,GAAG2B,IAAI,CAAC;QACR,MAAM2D;IACR;IAEA,OAAO;QAAE5E,QAAQkD,WAAWL,MAAM;QAAEC,UAAAA;IAAS;AAC/C;AAEO,SAASjE,KAAKmC,GAAmB;IACtC,IAAM6D,WAAWzF,IAAAA,cAAI,EAAC4B,IAAIW,OAAO,EAAEmD,mBAAS;IAC5CC,IAAAA,iBAAS,EAACF,UAAU;QAAEG,WAAW;IAAK;IACtC,IAAMC,SAAS7F,IAAAA,cAAI,EAACyF,UAAUpG;IAE9B,IAAMa,KAAK,IAAI4F,wBAAY,CAACD;IAC5B3F,GAAG2B,IAAI,CAAC;IACR,sFAAsF;IACtF,qFAAqF;IACrF,sFAAsF;IACtF,0DAA0D;IAC1D3B,GAAG2B,IAAI,CAAC;IACR5B,kBAAkBC;IAElBA,GAAG2B,IAAI,CAAC;IAER,uFAAuF;IACvF,qGAAqG;IACrG,IAAMkE,UAAUvG,QAAQU,IAAI;IAC5B,IAAMyD,WAAWnE,QAAQU,IAAI;IAC7B,IAAM8F,eAAe/D,IAAAA,0BAAgB,EAACL;IACtC,IAAI,AAACmE,YAAY,QAAQA,YAAYzG,kBAAoBqE,aAAa,QAAQA,aAAaqC,cAAe;QACxG9F,GAAG+F,KAAK;QACRC,IAAAA,cAAM,EAACT,UAAU;YAAEG,WAAW;YAAMO,OAAO;QAAK;QAChD,OAAO1G,KAAKmC;IACd;IAEAD,aAAazB,IAAI0B;IACjB,IAA6BjC,aAAAA,UAAUO,IAAI0B,KAAKA,IAAIW,OAAO,GAAnD3B,SAAqBjB,WAArBiB,QAAQ8C,WAAa/D,WAAb+D;IAEhB,OAAO;QAAExD,IAAAA;QAAI0B,KAAAA;QAAKiE,QAAAA;QAAQjF,QAAAA;QAAQ8C,UAAAA;IAAS;AAC7C;AAGO,SAAShE,QAAQkC,GAAmB;IACzCsE,IAAAA,cAAM,EAAClG,IAAAA,cAAI,EAAC4B,IAAIW,OAAO,EAAEmD,mBAAS,GAAG;QAAEE,WAAW;QAAMO,OAAO;IAAK;IACpE,OAAO1G,KAAKmC;AACd"}

package/dist/cjs/errors.d.cts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'SEARCH_SYNTAX';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.d.ts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'SEARCH_SYNTAX';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAGxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'SEARCH_SYNTAX';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAGxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}

package/dist/cjs/features/embed.d.cts

@@ -11,4 +11,5 @@ export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: str
 export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{
     path: string;
     lines: string;
+    similarity: number;
 }>>;

package/dist/cjs/features/embed.d.ts

@@ -11,4 +11,5 @@ export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: str
 export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{
     path: string;
     lines: string;
+    similarity: number;
 }>>;

package/dist/cjs/features/embed.js

@@ -366,10 +366,13 @@ function staticProvider(model) {
     return _async_to_generator(function() {
         var _ref, _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1, dir, _i, _iter, file, raw, headerLen, header, entry, spec, dims, dataStart, matrix, tokenizerJson, Tokenizer, tok, unkId;
         function one(text) {
+            // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an
+            // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the
+            // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.
             var ids = tok.encode(text, {
                 add_special_tokens: false
             }).ids.filter(function(id) {
-                return id !== unkId;
+                return Number.isInteger(id) && id !== unkId;
             });
             var v = new Float32Array(dims);
             if (ids.length === 0) return v;
@@ -829,7 +832,8 @@ function semanticCandidates(db, cfg, terms, fetch1) {
                             var _param = _sliced_to_array(param, 2), path = _param[0], b = _param[1];
                             return {
                                 path: path,
-                                lines: b.lines
+                                lines: b.lines,
+                                similarity: Math.round(b.score * 1000) / 1000
                             };
                         })
                     ];

package/dist/cjs/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic expansion needs features.embed in sense.config.json (e.g. \"features\": { \"embed\": true })');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0 }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n  }\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines }));\n}\n"],"names":["embed","embedPending","semanticCandidates","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","join","chunks","forEach","start","end","text","slice","trim","startLine","endLine","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","fetchToFile","url","dest","fetch","then","res","ok","SenseError","status","Buffer","from","arrayBuffer","writeFileSync","renameSync","staticProvider","model","tokenizerJson","dir","file","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","ids","encode","add_special_tokens","id","v","Float32Array","off","d","norm","Math","sqrt","existsSync","homedir","replace","mkdirSync","recursive","console","error","readFileSync","Number","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","vocab","unk_token","texts","map","apiProvider","keyEnv","base","headers","key","post","body","method","stringify","input","json","data","embedding","process","env","undefined","authorization","providers","Map","getProvider","cfg","e","embedConfig","sig","type","p","get","set","toStore","full","int8","scale","max","abs","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","chunkIdxs","update","batch","vectors","j","job","q","Int8Array","round","chunk","err","all","min","parseFile","relPath","absPath","mtimeMs","size","doc","terms","qv","rows","best","dot","score","existing","match","t","includes","vector","byteLength","start_line","end_line","sort","a","b"],"mappings":";;;;;;;;;;;QA6DaA;eAAAA;;QAgISC;eAAAA;;QA+CAC;eAAAA;;;sBA5OyD;sBACvD;wBACH;wBAGO;wBACD;sBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG1B,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,IAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,IAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,IAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,IAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;KAAa,CAAd,OAAI,qBAAGA,WAAUA;IAC5E,IAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,IAAMC,SAAkB,EAAE;IAC1BP,OAAOQ,OAAO,CAAC,SAACC,OAAOb;QACrB,IAAMc,MAAMd,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,IAAMc,OAAOnB,MACVoB,KAAK,CAACH,QAAQ,GAAGC,KACjBJ,IAAI,CAAC,MACLO,IAAI;QACP,IAAIF,KAAKd,MAAM,GAAG,GAAGU,OAAOR,IAAI,CAAC;YAAEe,WAAWL;YAAOM,SAASL;YAAKC,MAAMV,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAS,OAALU,QAASA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEO,IAAMvB,QAAiB;IAC5BgC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;IACV;IACAC,SAAAA,SAAAA,QAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAAA,SAAAA,OAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACAG,OAAAA,SAAAA,MAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBnB,OAAO,CAAC,SAACqB,GAAGC;mBAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEf,SAAS,EAAEe,EAAEd,OAAO;;IACzF;AACF;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASgB,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,SAAOC;;;;;;wBAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,0BAAwCF,OAAfJ,KAAI,aAAsB,OAAXI,IAAIG,MAAM;;4BACrF,GAAO,OAALN,MAAK;;4BAAQO,OAAOC,IAAI;wBAAC;;4BAAML,IAAIM,WAAW;;;wBAA/DC,qBAAa;4BAAiBH,QAAAA;gCAAY;;;wBAC1CI,IAAAA,kBAAU,EAAC,AAAC,GAAO,OAALX,MAAK,UAAQA;;;;;;QAC7B;;AACF;AAEA,SAAeY,eAAeC,KAAa;;kBA2BEC,sBAA7BA,4BAAAA,uBA1BVC,SAIiB,OAARC,MAQP3D,KACA4D,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAT,eAEEU,WACFC,KACAC;QAEN,SAASC,IAAIjD,IAAY;YACvB,IAAMkD,MAAM,AAACH,IAAII,MAAM,CAACnD,MAAM;gBAAEoD,oBAAoB;YAAM,GAAGF,GAAG,CAAczD,MAAM,CAAC,SAAC4D;uBAAOA,OAAOL;;YACpG,IAAMM,IAAI,IAAIC,aAAaZ;YAC3B,IAAIO,IAAIhE,MAAM,KAAK,GAAG,OAAOoE;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYJ,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMG,KAAN;oBACH,IAAMG,MAAMH,KAAKV;oBACjB,IAAK,IAAIc,IAAI,GAAGA,IAAId,MAAMc,IAAKH,CAAC,CAACG,EAAE,IAAIZ,MAAM,CAACW,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAId,MAAMc,KAAK;gBAC7BH,CAAC,CAACG,GAAE,IAAIP,IAAIhE,MAAM;gBAClBwE,QAAQJ,CAAC,CAACG,GAAE,GAAGH,CAAC,CAACG,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAId,MAAMc,KAAKH,CAAC,CAACG,GAAE,IAAIC;YACvC,OAAOJ;QACT;;;;oBA5CIjB,MAAMF;yBACN,CAAC0B,IAAAA,kBAAU,EAAClE,IAAAA,cAAI,EAAC0C,KAAK,uBAAtB;;;;oBACFA,MAAM1C,IAAAA,cAAI,EAACmE,IAAAA,eAAO,KAAI,UAAU,eAAe,UAAU3B,MAAM4B,OAAO,CAAC,OAAO;oBAC9EC,IAAAA,iBAAS,EAAC3B,KAAK;wBAAE4B,WAAW;oBAAK;4BACd;wBAAC;wBAAqB;;;;+BAAtB;;;;oBAAR3B,OAAQ;yBACb,CAACuB,IAAAA,kBAAU,EAAClE,IAAAA,cAAI,EAAC0C,KAAKC,QAAtB;;;;oBACF4B,QAAQC,KAAK,CAAC,AAAC,YAAoB7B,OAATH,OAAM,KAAgBE,OAAbC,MAAK,UAAY,OAAJD,KAAI;oBACpD;;wBAAMjB,YAAY,AAAC,0BAA+CkB,OAAtBH,OAAM,kBAAqB,OAALG,OAAQ3C,IAAAA,cAAI,EAAC0C,KAAKC;;;oBAApF;;;;;;;;;oBAKA3D,MAAMyF,IAAAA,oBAAY,EAACzE,IAAAA,cAAI,EAAC0C,KAAK;oBAC7BE,YAAY8B,OAAO1F,IAAI2F,eAAe,CAAC;oBACvC9B,SAAS+B,KAAKC,KAAK,CAAC7F,IAAI8F,QAAQ,CAAC,GAAG,IAAIlC,WAAWmC,QAAQ,CAAC;oBAC5DjC,QAAQkC,OAAOC,OAAO,CAACpC,QAAQqC,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAACrC,SAASA,KAAK,CAAC,EAAE,CAACsC,KAAK,KAAK,OAAO,MAAM,IAAIpD,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAANQ,OAAM;oBAC/EO,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKsC,KAAK,CAAC,EAAE;oBACpBpC,YAAYjE,IAAIsG,UAAU,GAAG,IAAI1C,YAAYG,KAAKwC,YAAY,CAAC,EAAE;oBACjErC,SAASD,YAAY,MAAM,IAAI,IAAIW,aAAa5E,IAAIwG,MAAM,EAAEvC,WAAWF,KAAKsC,KAAK,CAAC,EAAE,GAAGrC,QAAQ,IAAIY,aAAa5E,IAAIwG,MAAM,CAAClF,KAAK,CAAC2C,WAAWA,YAAYF,KAAKsC,KAAK,CAAC,EAAE,GAAGrC,OAAO;oBAE/KP,gBAAgBmC,KAAKC,KAAK,CAACJ,IAAAA,oBAAY,EAACzE,IAAAA,cAAI,EAAC0C,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BS,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUV,eAAe,CAAC;oBACpCY,iBAAQZ,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBgD,KAAK,cAA1BhD,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBiD,SAAS,CAAC,uCAAI,CAAC;oBAoB/E;;wBAAO;4BAAEhC,IAAI,AAAC,UAAe,OAANlB;4BAASQ,MAAAA;4BAAMtE,OAAO,SAAPA,MAAciH;;;;;4CAAUA,MAAMC,GAAG,CAACtC;;;;;wBAAK;;;;IAC/E;;AAEA,gFAAgF;AAEhF,SAAeuC,YAAYrD,KAAa,EAAEd,GAAuB,EAAEoE,MAA0B;;YAErFC,MACAC,SACAC,KAUAjD;QAPN,SAAekD,KAAKP,KAAe;;oBAC3B7D,KAEAqE;;;;4BAFM;;gCAAMvE,MAAM,AAAC,GAAO,OAALmE,MAAK,gBAAc;oCAAEK,QAAQ;oCAAQJ,SAAAA;oCAASG,MAAMvB,KAAKyB,SAAS,CAAC;wCAAE7D,OAAAA;wCAAO8D,OAAOX;oCAAM;gCAAG;;;4BAAjH7D,MAAM;4BACZ,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BF,OAA3BiE,MAAK,wBAAiC,OAAXjE,IAAIG,MAAM;4BAC3E;;gCAAMH,IAAIyE,IAAI;;;4BAAtBJ,OAAQ;4BACd;;gCAAOA,KAAKK,IAAI,CAACZ,GAAG,CAAC,SAAC9B;2CAAMF,aAAazB,IAAI,CAAC2B,EAAE2C,SAAS;;;;;YAC3D;;;;;oBAXA,IAAI,CAAC/E,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe;oBACxC+D,OAAOrE,IAAI0C,OAAO,CAAC,QAAQ;oBAC3B4B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAASY,QAAQC,GAAG,CAACb,OAAO,GAAGc;oBAC3C,IAAIX,KAAKD,QAAQa,aAAa,GAAG,AAAC,UAAa,OAAJZ;oBAS7B;;wBAAMC;4BAAM;;;;oBAApBlD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACzD,MAAM;oBACxD;;wBAAO;4BAAEmE,IAAI,AAAC,OAAclB,OAARuD,MAAK,KAAS,OAANvD;4BAASQ,MAAAA;4BAAMtE,OAAOwH;wBAAK;;;;IACzD;;AAEA,IAAMY,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,IAAMA,IAAIC,IAAAA,qBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIlF,oBAAU,CAAC,kBAAkB;IAC/C,IAAMoF,MAAM,AArKd,AAqKe,GAAYF,OAAVA,EAAEG,IAAI,EAAC,YAAGH,EAAE1E,KAAK,EAAC,KAAe,QAAZ0E,SAAAA,EAAExF,GAAG,cAALwF,oBAAAA,SAAS;IAC7C,IAAII,IAAIR,UAAUS,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIJ,EAAEG,IAAI,KAAK,QAAQxB,YAAYqB,EAAE1E,KAAK,EAAE0E,EAAExF,GAAG,EAAEwF,EAAEjB,GAAG,IAAI1D,eAAe2E,EAAE1E,KAAK;QAClFsE,UAAUU,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAE1E,IAAY,EAAE2E,IAAa;IAC9D,IAAMhE,IAAI,IAAIC,aAAaZ;IAC3B,IAAIe,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAId,MAAMc,IAAKC,QAAQ2D,IAAI,CAAC5D,EAAE,GAAG4D,IAAI,CAAC5D,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,KAAI,GAAGA,KAAId,MAAMc,KAAKH,CAAC,CAACG,GAAE,GAAG4D,IAAI,CAAC5D,GAAE,GAAGC;IAChD,IAAI,CAAC4D,MAAM,OAAO;QAAEhE,GAAAA;QAAGiE,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAI/D,KAAI,GAAGA,KAAId,MAAMc,KAAK+D,MAAM7D,KAAK6D,GAAG,CAACA,KAAK7D,KAAK8D,GAAG,CAACnE,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGiE,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAelJ,aAAaiC,EAAgB,EAAEqG,GAAW,EAAEc,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOrH,MAAMsH,WACZtI,QAMC,4BAAA,oBAAA,iBAAA,YAAA,QAAMuB,KAGPgH,QACGlJ;;;;;4BACDmJ,OACAC;;;;oCADAD,QAAQH,KAAKhI,KAAK,CAAChB,GAAGA,IAAIR;oCAChB;;wCAAMkJ,SAAStJ,KAAK,CAAC+J,MAAM7C,GAAG,CAAC,SAAC+C;mDAAMA,EAAEtI,IAAI;;;;oCAAtDqI,UAAU;oCAChB9H,GAAGC,IAAI,CAAC;oCACR,IAAI;wCACF4H,MAAMvI,OAAO,CAAC,SAAC0I,KAAKD;4CAClB,IAAqBlB,WAAAA,QAAQiB,OAAO,CAACC,EAAE,EAAET,WAAW,OAA5CvE,IAAa8D,SAAb9D,GAAGiE,QAAUH,SAAVG;4CACX,IAAMiB,IAAI,IAAIC,UAAUZ;4CACxB,IAAK,IAAIpE,IAAI,GAAGA,IAAIoE,WAAWpE,IAAK+E,CAAC,CAAC/E,EAAE,GAAGE,KAAK+E,KAAK,CAACpF,CAAC,CAACG,EAAE,GAAG8D;4CAC7DY,OAAOrH,GAAG,CAACyG,OAAO1F,OAAOC,IAAI,CAAC0G,EAAErD,MAAM,GAAGoD,IAAI3H,IAAI,EAAE2H,IAAII,KAAK;wCAC9D;wCACApI,GAAGC,IAAI,CAAC;oCACV,EAAE,OAAOoI,KAAK;wCACZrI,GAAGC,IAAI,CAAC;wCACR,MAAMoI;oCACR;;;;;;oBACF;oBAxCiB;;wBAAMjC,YAAYC;;;oBAA7Be,WAAW;oBACXC,QAAQrH,GAAGM,OAAO,CAAC,oFAAoFgI,GAAG;oBAChH,IAAIjB,MAAM1I,MAAM,KAAK,GAAG;;;oBAClB2I,YAAYlE,KAAKmF,GAAG,CAACtK,YAAYmJ,SAAShF,IAAI;oBAE9CmF,SAAS,IAAIpB;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAakB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOZ,GAAG,CAACa,IAAInH,IAAI,eAAnBkH,yBAAAA;4BACbE,KAAK5I,IAAI,CAAC2I,IAAIY,KAAK;4BACnBb,OAAOX,GAAG,CAACY,IAAInH,IAAI,EAAEoH;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOlH,uBAAMsH;4BACZtI,SAAAA,KAAAA;4BACJ,IAAI;gCACFA,SAASmJ,IAAAA,iBAAS,EAAC;oCAAEC,SAASpI;oCAAMqI,SAAStJ,IAAAA,cAAI,EAAC+H,SAAS9G;oCAAOsI,SAAS;oCAAGC,MAAM;gCAAE;oCAAI9K;mCAAQ+K,GAAG,CAACpI,SAAS,CAAC3C,KAAK;4BACvH,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa6J,gCAAb,8BAAA,SAAA,0BAAA;oCAAM/G,MAAN;oCAAwB,IAAIvB,MAAM,CAACuB,IAAI,EAAE8G,KAAK7I,IAAI,CAAC;wCAAEwB,MAAAA;wCAAM+H,OAAOxH;wCAAKnB,MAAMJ,MAAM,CAACuB,IAAI,CAACnB,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBARK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAUCmI,SAAS5H,GAAGM,OAAO,CAAC;oBACjB5B,IAAI;;;yBAAGA,CAAAA,IAAIgJ,KAAK/I,MAAM,AAAD;;;;;;;;;;;;oBAAGD,KAAKR;;;;;;;;;;;IAiBxC;;AAKO,SAAeF,mBAAmBgC,EAAgB,EAAEqG,GAAW,EAAEyC,KAAa,EAAE9H,MAAa;;YAOpF8H,cANR3B,SAIAC,UACAE,WACA7H,MACYoH,UAAPkC,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMzB,KACHS,GACFiB,KACKhG,GACHiG,OACAC;;;;oBAvBFjC,UAAU,AAACd,IAAgCc,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAI/F,oBAAU,CAAC,eAAe;oBAClD;;wBAAMrD,aAAaiC,IAAIqG,KAAKc;;;oBAA5B;oBAEiB;;wBAAMf,YAAYC;;;oBAA7Be,WAAW;oBACXE,YAAYlE,KAAKmF,GAAG,CAACtK,YAAYmJ,SAAShF,IAAI;oBAC9C3C,OAAO,EAACqJ,eAAAA,MAAMO,KAAK,CAAC,2naAAZP,0BAAAA,mBAAsC5J,MAAM,CAAC,SAACoK;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIlK,IAAI,CAAC;oBACvF;;wBAAMgI,SAAStJ,KAAK;4BAAE2B;;;;oBAA/BoH,WAAAA;wBAAS,aAA6B,CAAC,EAAE;wBAAES;wBAAW;wBAA7DyB,KAAOlC,SAAV9D;oBAEFiG,OAAOhJ,GAAGM,OAAO,CAAC,+FAA+FgI,GAAG;oBAQpHW,OAAO,IAAI9C;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAa6C,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbxB,MAAN;4BACGS,IAAI,IAAIC,UAAUV,IAAIgC,MAAM,CAAC5E,MAAM,EAAE4C,IAAIgC,MAAM,CAAC9E,UAAU,EAAEtB,KAAKmF,GAAG,CAACjB,WAAWE,IAAIgC,MAAM,CAACC,UAAU;4BACvGP,MAAM;4BACV,IAAShG,IAAI,GAAGA,IAAI+E,EAAEtJ,MAAM,EAAEuE,IAAKgG,OAAOjB,CAAC,CAAC/E,EAAE,GAAG6F,EAAE,CAAC7F,EAAE;4BAChDiG,QAAQD,MAAM1B,IAAIR,KAAK;4BACvBoC,WAAWH,KAAKtC,GAAG,CAACa,IAAInH,IAAI;4BAClC,IAAI,CAAC+I,YAAYD,QAAQC,SAASD,KAAK,EAAEF,KAAKrC,GAAG,CAACY,IAAInH,IAAI,EAAE;gCAAE8I,OAAAA;gCAAO7K,OAAO,AAAC,IAAqBkJ,OAAlBA,IAAIkC,UAAU,EAAC,KAAgB,OAAblC,IAAImC,QAAQ;4BAAG;wBACnH;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAQL;;wBAAQ,qBAAGV,KAAK5E,OAAO,IACpBuF,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACX,KAAK,GAAGU,CAAC,CAAC,EAAE,CAACV,KAAK;2BACtCzJ,KAAK,CAAC,GAAGsB,QACTgE,GAAG,CAAC;qEAAE3E,kBAAMyJ;mCAAQ;gCAAEzJ,MAAAA;gCAAM/B,OAAOwL,EAAExL,KAAK;4BAAC;;;;;IAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic expansion needs features.embed in sense.config.json (e.g. \"features\": { \"embed\": true })');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0 }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n  }\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\n// Returns the cosine similarity alongside each candidate: stored and query vectors are both\n// L2-normalised before quantisation, so the dot product is cosine in [-1, 1]. `find` surfaces\n// it because the fused rank score cannot express match quality -- nearest-neighbour search\n// always returns a nearest neighbour, so without a magnitude an agent cannot tell a real hit\n// from the best of a bad lot.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: Math.round(b.score * 1000) / 1000 }));\n}\n"],"names":["embed","embedPending","semanticCandidates","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","join","chunks","forEach","start","end","text","slice","trim","startLine","endLine","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","fetchToFile","url","dest","fetch","then","res","ok","SenseError","status","Buffer","from","arrayBuffer","writeFileSync","renameSync","staticProvider","model","tokenizerJson","dir","file","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","ids","encode","add_special_tokens","id","Number","isInteger","v","Float32Array","off","d","norm","Math","sqrt","existsSync","homedir","replace","mkdirSync","recursive","console","error","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","vocab","unk_token","texts","map","apiProvider","keyEnv","base","headers","key","post","body","method","stringify","input","json","data","embedding","process","env","undefined","authorization","providers","Map","getProvider","cfg","e","embedConfig","sig","type","p","get","set","toStore","full","int8","scale","max","abs","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","chunkIdxs","update","batch","vectors","j","job","q","Int8Array","round","chunk","err","all","min","parseFile","relPath","absPath","mtimeMs","size","doc","terms","qv","rows","best","dot","score","existing","match","t","includes","vector","byteLength","start_line","end_line","sort","a","b","similarity"],"mappings":";;;;;;;;;;;QA6DaA;eAAAA;;QAmISC;eAAAA;;QAoDAC;eAAAA;;;sBApPyD;sBACvD;wBACH;wBAGO;wBACD;sBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG1B,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,IAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,IAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,IAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,IAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;KAAa,CAAd,OAAI,qBAAGA,WAAUA;IAC5E,IAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,IAAMC,SAAkB,EAAE;IAC1BP,OAAOQ,OAAO,CAAC,SAACC,OAAOb;QACrB,IAAMc,MAAMd,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,IAAMc,OAAOnB,MACVoB,KAAK,CAACH,QAAQ,GAAGC,KACjBJ,IAAI,CAAC,MACLO,IAAI;QACP,IAAIF,KAAKd,MAAM,GAAG,GAAGU,OAAOR,IAAI,CAAC;YAAEe,WAAWL;YAAOM,SAASL;YAAKC,MAAMV,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAS,OAALU,QAASA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEO,IAAMvB,QAAiB;IAC5BgC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;IACV;IACAC,SAAAA,SAAAA,QAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAAA,SAAAA,OAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACAG,OAAAA,SAAAA,MAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBnB,OAAO,CAAC,SAACqB,GAAGC;mBAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEf,SAAS,EAAEe,EAAEd,OAAO;;IACzF;AACF;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASgB,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,SAAOC;;;;;;wBAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,0BAAwCF,OAAfJ,KAAI,aAAsB,OAAXI,IAAIG,MAAM;;4BACrF,GAAO,OAALN,MAAK;;4BAAQO,OAAOC,IAAI;wBAAC;;4BAAML,IAAIM,WAAW;;;wBAA/DC,qBAAa;4BAAiBH,QAAAA;gCAAY;;;wBAC1CI,IAAAA,kBAAU,EAAC,AAAC,GAAO,OAALX,MAAK,UAAQA;;;;;;QAC7B;;AACF;AAEA,SAAeY,eAAeC,KAAa;;kBA2BEC,sBAA7BA,4BAAAA,uBA1BVC,SAIiB,OAARC,MAQP3D,KACA4D,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAT,eAEEU,WACFC,KACAC;QAEN,SAASC,IAAIjD,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMkD,MAAM,AAACH,IAAII,MAAM,CAACnD,MAAM;gBAAEoD,oBAAoB;YAAM,GAAGF,GAAG,CAAczD,MAAM,CAAC,SAAC4D;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOL;;YAC5H,IAAMQ,IAAI,IAAIC,aAAad;YAC3B,IAAIO,IAAIhE,MAAM,KAAK,GAAG,OAAOsE;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYN,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMG,KAAN;oBACH,IAAMK,MAAML,KAAKV;oBACjB,IAAK,IAAIgB,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKH,CAAC,CAACG,EAAE,IAAId,MAAM,CAACa,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK;gBAC7BH,CAAC,CAACG,GAAE,IAAIT,IAAIhE,MAAM;gBAClB0E,QAAQJ,CAAC,CAACG,GAAE,GAAGH,CAAC,CAACG,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,IAAIC;YACvC,OAAOJ;QACT;;;;oBA/CInB,MAAMF;yBACN,CAAC4B,IAAAA,kBAAU,EAACpE,IAAAA,cAAI,EAAC0C,KAAK,uBAAtB;;;;oBACFA,MAAM1C,IAAAA,cAAI,EAACqE,IAAAA,eAAO,KAAI,UAAU,eAAe,UAAU7B,MAAM8B,OAAO,CAAC,OAAO;oBAC9EC,IAAAA,iBAAS,EAAC7B,KAAK;wBAAE8B,WAAW;oBAAK;4BACd;wBAAC;wBAAqB;;;;+BAAtB;;;;oBAAR7B,OAAQ;yBACb,CAACyB,IAAAA,kBAAU,EAACpE,IAAAA,cAAI,EAAC0C,KAAKC,QAAtB;;;;oBACF8B,QAAQC,KAAK,CAAC,AAAC,YAAoB/B,OAATH,OAAM,KAAgBE,OAAbC,MAAK,UAAY,OAAJD,KAAI;oBACpD;;wBAAMjB,YAAY,AAAC,0BAA+CkB,OAAtBH,OAAM,kBAAqB,OAALG,OAAQ3C,IAAAA,cAAI,EAAC0C,KAAKC;;;oBAApF;;;;;;;;;oBAKA3D,MAAM2F,IAAAA,oBAAY,EAAC3E,IAAAA,cAAI,EAAC0C,KAAK;oBAC7BE,YAAYe,OAAO3E,IAAI4F,eAAe,CAAC;oBACvC/B,SAASgC,KAAKC,KAAK,CAAC9F,IAAI+F,QAAQ,CAAC,GAAG,IAAInC,WAAWoC,QAAQ,CAAC;oBAC5DlC,QAAQmC,OAAOC,OAAO,CAACrC,QAAQsC,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAACtC,SAASA,KAAK,CAAC,EAAE,CAACuC,KAAK,KAAK,OAAO,MAAM,IAAIrD,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAANQ,OAAM;oBAC/EO,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKuC,KAAK,CAAC,EAAE;oBACpBrC,YAAYjE,IAAIuG,UAAU,GAAG,IAAI3C,YAAYG,KAAKyC,YAAY,CAAC,EAAE;oBACjEtC,SAASD,YAAY,MAAM,IAAI,IAAIa,aAAa9E,IAAIyG,MAAM,EAAExC,WAAWF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,QAAQ,IAAIc,aAAa9E,IAAIyG,MAAM,CAACnF,KAAK,CAAC2C,WAAWA,YAAYF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,OAAO;oBAE/KP,gBAAgBoC,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAAC3E,IAAAA,cAAI,EAAC0C,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BS,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUV,eAAe,CAAC;oBACpCY,iBAAQZ,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBiD,KAAK,cAA1BjD,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBkD,SAAS,CAAC,uCAAI,CAAC;oBAuB/E;;wBAAO;4BAAEjC,IAAI,AAAC,UAAe,OAANlB;4BAASQ,MAAAA;4BAAMtE,OAAO,SAAPA,MAAckH;;;;;4CAAUA,MAAMC,GAAG,CAACvC;;;;;wBAAK;;;;IAC/E;;AAEA,gFAAgF;AAEhF,SAAewC,YAAYtD,KAAa,EAAEd,GAAuB,EAAEqE,MAA0B;;YAErFC,MACAC,SACAC,KAUAlD;QAPN,SAAemD,KAAKP,KAAe;;oBAC3B9D,KAEAsE;;;;4BAFM;;gCAAMxE,MAAM,AAAC,GAAO,OAALoE,MAAK,gBAAc;oCAAEK,QAAQ;oCAAQJ,SAAAA;oCAASG,MAAMvB,KAAKyB,SAAS,CAAC;wCAAE9D,OAAAA;wCAAO+D,OAAOX;oCAAM;gCAAG;;;4BAAjH9D,MAAM;4BACZ,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BF,OAA3BkE,MAAK,wBAAiC,OAAXlE,IAAIG,MAAM;4BAC3E;;gCAAMH,IAAI0E,IAAI;;;4BAAtBJ,OAAQ;4BACd;;gCAAOA,KAAKK,IAAI,CAACZ,GAAG,CAAC,SAAC7B;2CAAMF,aAAa3B,IAAI,CAAC6B,EAAE0C,SAAS;;;;;YAC3D;;;;;oBAXA,IAAI,CAAChF,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe;oBACxCgE,OAAOtE,IAAI4C,OAAO,CAAC,QAAQ;oBAC3B2B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAASY,QAAQC,GAAG,CAACb,OAAO,GAAGc;oBAC3C,IAAIX,KAAKD,QAAQa,aAAa,GAAG,AAAC,UAAa,OAAJZ;oBAS7B;;wBAAMC;4BAAM;;;;oBAApBnD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACzD,MAAM;oBACxD;;wBAAO;4BAAEmE,IAAI,AAAC,OAAclB,OAARwD,MAAK,KAAS,OAANxD;4BAASQ,MAAAA;4BAAMtE,OAAOyH;wBAAK;;;;IACzD;;AAEA,IAAMY,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,IAAMA,IAAIC,IAAAA,qBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAInF,oBAAU,CAAC,kBAAkB;IAC/C,IAAMqF,MAAM,AAxKd,AAwKe,GAAYF,OAAVA,EAAEG,IAAI,EAAC,YAAGH,EAAE3E,KAAK,EAAC,KAAe,QAAZ2E,SAAAA,EAAEzF,GAAG,cAALyF,oBAAAA,SAAS;IAC7C,IAAII,IAAIR,UAAUS,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIJ,EAAEG,IAAI,KAAK,QAAQxB,YAAYqB,EAAE3E,KAAK,EAAE2E,EAAEzF,GAAG,EAAEyF,EAAEjB,GAAG,IAAI3D,eAAe4E,EAAE3E,KAAK;QAClFuE,UAAUU,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAE3E,IAAY,EAAE4E,IAAa;IAC9D,IAAM/D,IAAI,IAAIC,aAAad;IAC3B,IAAIiB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKC,QAAQ0D,IAAI,CAAC3D,EAAE,GAAG2D,IAAI,CAAC3D,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,GAAG2D,IAAI,CAAC3D,GAAE,GAAGC;IAChD,IAAI,CAAC2D,MAAM,OAAO;QAAE/D,GAAAA;QAAGgE,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAI9D,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK8D,MAAM5D,KAAK4D,GAAG,CAACA,KAAK5D,KAAK6D,GAAG,CAAClE,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGgE,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAenJ,aAAaiC,EAAgB,EAAEsG,GAAW,EAAEc,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOtH,MAAMuH,WACZvI,QAMC,4BAAA,oBAAA,iBAAA,YAAA,QAAMuB,KAGPiH,QACGnJ;;;;;4BACDoJ,OACAC;;;;oCADAD,QAAQH,KAAKjI,KAAK,CAAChB,GAAGA,IAAIR;oCAChB;;wCAAMmJ,SAASvJ,KAAK,CAACgK,MAAM7C,GAAG,CAAC,SAAC+C;mDAAMA,EAAEvI,IAAI;;;;oCAAtDsI,UAAU;oCAChB/H,GAAGC,IAAI,CAAC;oCACR,IAAI;wCACF6H,MAAMxI,OAAO,CAAC,SAAC2I,KAAKD;4CAClB,IAAqBlB,WAAAA,QAAQiB,OAAO,CAACC,EAAE,EAAET,WAAW,OAA5CtE,IAAa6D,SAAb7D,GAAGgE,QAAUH,SAAVG;4CACX,IAAMiB,IAAI,IAAIC,UAAUZ;4CACxB,IAAK,IAAInE,IAAI,GAAGA,IAAImE,WAAWnE,IAAK8E,CAAC,CAAC9E,EAAE,GAAGE,KAAK8E,KAAK,CAACnF,CAAC,CAACG,EAAE,GAAG6D;4CAC7DY,OAAOtH,GAAG,CAAC0G,OAAO3F,OAAOC,IAAI,CAAC2G,EAAErD,MAAM,GAAGoD,IAAI5H,IAAI,EAAE4H,IAAII,KAAK;wCAC9D;wCACArI,GAAGC,IAAI,CAAC;oCACV,EAAE,OAAOqI,KAAK;wCACZtI,GAAGC,IAAI,CAAC;wCACR,MAAMqI;oCACR;;;;;;oBACF;oBAxCiB;;wBAAMjC,YAAYC;;;oBAA7Be,WAAW;oBACXC,QAAQtH,GAAGM,OAAO,CAAC,oFAAoFiI,GAAG;oBAChH,IAAIjB,MAAM3I,MAAM,KAAK,GAAG;;;oBAClB4I,YAAYjE,KAAKkF,GAAG,CAACvK,YAAYoJ,SAASjF,IAAI;oBAE9CoF,SAAS,IAAIpB;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAakB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOZ,GAAG,CAACa,IAAIpH,IAAI,eAAnBmH,yBAAAA;4BACbE,KAAK7I,IAAI,CAAC4I,IAAIY,KAAK;4BACnBb,OAAOX,GAAG,CAACY,IAAIpH,IAAI,EAAEqH;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOnH,uBAAMuH;4BACZvI,SAAAA,KAAAA;4BACJ,IAAI;gCACFA,SAASoJ,IAAAA,iBAAS,EAAC;oCAAEC,SAASrI;oCAAMsI,SAASvJ,IAAAA,cAAI,EAACgI,SAAS/G;oCAAOuI,SAAS;oCAAGC,MAAM;gCAAE;oCAAI/K;mCAAQgL,GAAG,CAACrI,SAAS,CAAC3C,KAAK;4BACvH,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa8J,gCAAb,8BAAA,SAAA,0BAAA;oCAAMhH,MAAN;oCAAwB,IAAIvB,MAAM,CAACuB,IAAI,EAAE+G,KAAK9I,IAAI,CAAC;wCAAEwB,MAAAA;wCAAMgI,OAAOzH;wCAAKnB,MAAMJ,MAAM,CAACuB,IAAI,CAACnB,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBARK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAUCoI,SAAS7H,GAAGM,OAAO,CAAC;oBACjB5B,IAAI;;;yBAAGA,CAAAA,IAAIiJ,KAAKhJ,MAAM,AAAD;;;;;;;;;;;;oBAAGD,KAAKR;;;;;;;;;;;IAiBxC;;AAUO,SAAeF,mBAAmBgC,EAAgB,EAAEsG,GAAW,EAAEyC,KAAa,EAAE/H,MAAa;;YAOpF+H,cANR3B,SAIAC,UACAE,WACA9H,MACYqH,UAAPkC,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMzB,KACHS,GACFiB,KACK/F,GACHgG,OACAC;;;;oBAvBFjC,UAAU,AAACd,IAAgCc,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIhG,oBAAU,CAAC,eAAe;oBAClD;;wBAAMrD,aAAaiC,IAAIsG,KAAKc;;;oBAA5B;oBAEiB;;wBAAMf,YAAYC;;;oBAA7Be,WAAW;oBACXE,YAAYjE,KAAKkF,GAAG,CAACvK,YAAYoJ,SAASjF,IAAI;oBAC9C3C,OAAO,EAACsJ,eAAAA,MAAMO,KAAK,CAAC,2naAAZP,0BAAAA,mBAAsC7J,MAAM,CAAC,SAACqK;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAInK,IAAI,CAAC;oBACvF;;wBAAMiI,SAASvJ,KAAK;4BAAE2B;;;;oBAA/BqH,WAAAA;wBAAS,aAA6B,CAAC,EAAE;wBAAES;wBAAW;wBAA7DyB,KAAOlC,SAAV7D;oBAEFgG,OAAOjJ,GAAGM,OAAO,CAAC,+FAA+FiI,GAAG;oBAQpHW,OAAO,IAAI9C;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAa6C,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbxB,MAAN;4BACGS,IAAI,IAAIC,UAAUV,IAAIgC,MAAM,CAAC5E,MAAM,EAAE4C,IAAIgC,MAAM,CAAC9E,UAAU,EAAErB,KAAKkF,GAAG,CAACjB,WAAWE,IAAIgC,MAAM,CAACC,UAAU;4BACvGP,MAAM;4BACV,IAAS/F,IAAI,GAAGA,IAAI8E,EAAEvJ,MAAM,EAAEyE,IAAK+F,OAAOjB,CAAC,CAAC9E,EAAE,GAAG4F,EAAE,CAAC5F,EAAE;4BAChDgG,QAAQD,MAAM1B,IAAIR,KAAK;4BACvBoC,WAAWH,KAAKtC,GAAG,CAACa,IAAIpH,IAAI;4BAClC,IAAI,CAACgJ,YAAYD,QAAQC,SAASD,KAAK,EAAEF,KAAKrC,GAAG,CAACY,IAAIpH,IAAI,EAAE;gCAAE+I,OAAAA;gCAAO9K,OAAO,AAAC,IAAqBmJ,OAAlBA,IAAIkC,UAAU,EAAC,KAAgB,OAAblC,IAAImC,QAAQ;4BAAG;wBACnH;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAQL;;wBAAQ,qBAAGV,KAAK5E,OAAO,IACpBuF,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACX,KAAK,GAAGU,CAAC,CAAC,EAAE,CAACV,KAAK;2BACtC1J,KAAK,CAAC,GAAGsB,QACTiE,GAAG,CAAC;qEAAE5E,kBAAM0J;mCAAQ;gCAAE1J,MAAAA;gCAAM/B,OAAOyL,EAAEzL,KAAK;gCAAE0L,YAAY1G,KAAK8E,KAAK,CAAC2B,EAAEX,KAAK,GAAG,QAAQ;4BAAK;;;;;IAC/F"}

package/dist/cjs/output.js

@@ -85,13 +85,35 @@ function cell(value) {
     return String(value !== null && value !== void 0 ? value : '').replace(/\s*\n\s*/g, ' ');
 }
 function printRows(rows, format) {
-    var rendered = render(rows, format);
+    var rendered = renderRows(rows, format);
     console.log(rendered);
     if (rendered.length > OVERSIZE_BYTES) {
         console.warn("warning: result is ".concat(Math.round(rendered.length / 1000), " KB across ").concat(rows.length, " row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values"));
     }
 }
-function render(rows, format) {
+// Table output is for reading; a row wider than the terminal wraps into an unreadable
+// block (a `find` header measured 330 chars once `summary` is present, which drove one
+// user to pipe --format json through a script for every interactive query). Widest
+// columns give up space first, down to a floor, and only as far as the terminal needs.
+// --format json is never truncated: it is the machine-readable surface.
+var MIN_COLUMN = 12;
+var TRUNCATED = '…';
+function fitWidths(widths, limit) {
+    var gaps = (widths.length - 1) * 2;
+    var fitted = _to_consumable_array(widths);
+    for(;;){
+        var _Math;
+        var total = fitted.reduce(function(a, b) {
+            return a + b;
+        }, 0) + gaps;
+        if (total <= limit) return fitted;
+        var widest = fitted.indexOf((_Math = Math).max.apply(_Math, _to_consumable_array(fitted)));
+        if (fitted[widest] <= MIN_COLUMN) return fitted;
+        fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));
+    }
+}
+function renderRows(rows, format) {
+    var width = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : process.stdout.columns;
     var _Math;
     if (format === 'json') return JSON.stringify(rows, null, 2);
     if (rows.length === 0) return '(0 rows)';
@@ -101,17 +123,21 @@ function render(rows, format) {
             return cell(row[col]);
         });
     });
-    var widths = columns.map(function(col, i) {
+    var natural = columns.map(function(col, i) {
         return (_Math = Math).max.apply(_Math, [
             col.length
         ].concat(_to_consumable_array(cells.map(function(row) {
             return row[i].length;
         }))));
     });
+    var widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;
+    var clip = function clip(value, w) {
+        return value.length <= w ? value.padEnd(w) : "".concat(value.slice(0, w - 1)).concat(TRUNCATED);
+    };
     var formatRow = function formatRow(values) {
         return values.map(function(value, i) {
-            return value.padEnd(widths[i]);
-        }).join('  ');
+            return clip(value, widths[i]);
+        }).join('  ').trimEnd();
     };
     return [
         formatRow(columns),
@@ -131,12 +157,12 @@ function renderMap(result) {
     var parts = [
         "docs: ".concat(result.docs.count, " (").concat(Math.round(result.docs.bytes / 1024), " KB)"),
         "".concat(featuresLine(result.features), "\n"),
-        render(result.fields, 'table')
+        renderRows(result.fields, 'table')
     ];
     if (result.fieldsTotal > result.fields.length) parts.push("(+".concat(result.fieldsTotal - result.fields.length, " more fields)"));
-    if (result.hubs.length > 0) parts.push('\nhubs (by link rank):', render(result.hubs, 'table'));
+    if (result.hubs.length > 0) parts.push('\nhubs (by link rank):', renderRows(result.hubs, 'table'));
     else if (result.features.off.includes('rank')) parts.push('\nhubs: off (features.rank)');
-    parts.push('\nrecent:', render(result.recent, 'table'));
+    parts.push('\nrecent:', renderRows(result.recent, 'table'));
     return parts.join('\n');
 }
 function renderPeek(result) {

package/dist/cjs/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = render(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\nfunction render(rows: Row[], format: 'table' | 'json'): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const widths = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n\n  const formatRow = (values: string[]) => values.map((value, i) => value.padEnd(widths[i])).join('  ');\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nfunction featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, `${featuresLine(result.features)}\\n`, render(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', render(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push('\\nhubs: off (features.rank)');\n  parts.push('\\nrecent:', render(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["printRows","renderMap","renderPeek","OVERSIZE_BYTES","cell","value","String","replace","rows","format","rendered","render","console","log","length","warn","Math","round","JSON","stringify","columns","Object","keys","cells","map","row","col","widths","i","max","formatRow","values","padEnd","join","w","repeat","featuresLine","features","off","name","on","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","lines","path","tokens","entries","frontmatter","key","sections","s","level","heading","start_line","end_line","linkLine","label","shown","total","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;;;;;;;;;;;;QAYjBA;eAAAA;;QA4BAC;eAAAA;;QASAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA7ChB,yFAAyF;AACzF,IAAMC,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEO,SAASP,UAAUQ,IAAW,EAAEC,MAAwB;IAC7D,IAAMC,WAAWC,OAAOH,MAAMC;IAC9BG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGX,gBAAgB;QACpCS,QAAQG,IAAI,CAAC,AAAC,sBAAqEP,OAAhDQ,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,OAAM,eAAyB,OAAZN,KAAKM,MAAM,EAAC;IACjG;AACF;AAEA,SAASH,OAAOH,IAAW,EAAEC,MAAwB;QAMZO;IALvC,IAAIP,WAAW,QAAQ,OAAOS,KAAKC,SAAS,CAACX,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,IAAMM,UAAUC,OAAOC,IAAI,CAACd,IAAI,CAAC,EAAE;IACnC,IAAMe,QAAQf,KAAKgB,GAAG,CAAC,SAACC;eAAQL,QAAQI,GAAG,CAAC,SAACE;mBAAQtB,KAAKqB,GAAG,CAACC,IAAI;;;IAClE,IAAMC,SAASP,QAAQI,GAAG,CAAC,SAACE,KAAKE;eAAMZ,CAAAA,QAAAA,MAAKa,GAAG,OAARb,OAAAA;YAASU,IAAIZ,MAAM;SAAuC,CAA1DE,OAAqB,qBAAGO,MAAMC,GAAG,CAAC,SAACC;mBAAQA,GAAG,CAACG,EAAE,CAACd,MAAM;;;IAE/F,IAAMgB,YAAY,mBAACC;eAAqBA,OAAOP,GAAG,CAAC,SAACnB,OAAOuB;mBAAMvB,MAAM2B,MAAM,CAACL,MAAM,CAACC,EAAE;WAAGK,IAAI,CAAC;;IAE/F,OAAO;QAACH,UAAUV;QAAUO,OAAOH,GAAG,CAAC,SAACU;mBAAM,IAAIC,MAAM,CAACD;WAAID,IAAI,CAAC;KAA+B,CAA1F,OAAkE,qBAAGV,MAAMC,GAAG,CAACM,aAAYG,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,SAASG,aAAaC,QAAyC;IAC7D,IAAMC,MAAMD,SAASC,GAAG,CAACxB,MAAM,GAAG,IAAI,AAAC,cAA8E,OAApEuB,SAASC,GAAG,CAACd,GAAG,CAAC,SAACe;eAAS,AAAC,GAAoBA,OAAlBA,MAAK,eAAkB,OAALA,MAAK;OAAIN,IAAI,CAAC,SAAU;IACzH,OAAO,AAAC,aAAqCK,OAAzBD,SAASG,EAAE,CAACP,IAAI,CAAC,OAAY,OAAJK;AAC/C;AAEO,SAASrC,UAAUwC,MAA6J;IACrL,IAAMC,QAAQ;QAAE,SAA8B1B,OAAtByB,OAAOE,IAAI,CAACC,KAAK,EAAC,MAAyC,OAArC5B,KAAKC,KAAK,CAACwB,OAAOE,IAAI,CAACE,KAAK,GAAG,OAAM;QAAQ,GAAgC,OAA9BT,aAAaK,OAAOJ,QAAQ,GAAE;QAAK1B,OAAO8B,OAAOK,MAAM,EAAE;KAAS;IAC/J,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAChC,MAAM,EAAE4B,MAAMM,IAAI,CAAC,AAAC,KAA8C,OAA1CP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAChC,MAAM,EAAC;IACzG,IAAI2B,OAAOQ,IAAI,CAACnC,MAAM,GAAG,GAAG4B,MAAMM,IAAI,CAAC,0BAA0BrC,OAAO8B,OAAOQ,IAAI,EAAE;SAChF,IAAIR,OAAOJ,QAAQ,CAACC,GAAG,CAACY,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC;IAC1DN,MAAMM,IAAI,CAAC,aAAarC,OAAO8B,OAAOU,MAAM,EAAE;IAC9C,OAAOT,MAAMT,IAAI,CAAC;AACpB;AAEO,SAAS/B,WAAWuC,MAAiO;IAC1P,IAAMW,QAAQ;QAAE,GAAoBX,OAAlBA,OAAOY,IAAI,EAAC,QAAoB,OAAdZ,OAAOa,MAAM,EAAC;KAAU;QACvD,kCAAA,2BAAA;;QAAL,QAAK,YAAsBjC,OAAOkC,OAAO,CAACd,OAAOe,WAAW,sBAAvD,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOC,sBAAKpD;YAA8C+C,MAAMJ,IAAI,CAAC,AAAC,KAAY3C,OAARoD,KAAI,MAAU,OAANpD;;;QAAlF;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAIoC,OAAOiB,QAAQ,CAAC5C,MAAM,GAAG,GAAG;QAC9BsC,MAAMJ,IAAI,CAAC,IAAI;YACV,mCAAA,4BAAA;;YAAL,QAAK,aAAWP,OAAOiB,QAAQ,qBAA1B,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAMC,IAAN;gBAA4BP,MAAMJ,IAAI,CAAC,AAAC,KAAqCW,OAAjC,IAAIxB,MAAM,CAACwB,EAAEC,KAAK,GAAY,KAAmBD,OAAhBA,EAAEE,OAAO,EAAC,QAAsBF,OAAhBA,EAAEG,UAAU,EAAC,KAAmBH,OAAhBA,EAAEI,QAAQ,EAAC,OAAc,OAATJ,EAAEL,MAAM,EAAC;;;YAAtI;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;IACP,OAAO,IAAIb,OAAOH,GAAG,CAACY,QAAQ,CAAC,aAAaE,MAAMJ,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOH,GAAG,CAACY,QAAQ,CAAC,UAAU;QAChCE,MAAMJ,IAAI,CAAC,IAAI;QACf,OAAOI,MAAMnB,IAAI,CAAC;IACpB;IACA,IAAM+B,WAAW,kBAACC,OAAeC,OAAiBC;QAChD,IAAIA,QAAQ,GAAGf,MAAMJ,IAAI,CAAC,AAAC,GAAYmB,OAAVF,OAAM,MAAeC,OAAXC,OAAM,OAAwBA,OAAnBD,MAAMjC,IAAI,CAAC,OAAsE,OAA9DkC,QAAQD,MAAMpD,MAAM,GAAG,AAAC,MAA0B,OAArBqD,QAAQD,MAAMpD,MAAM,EAAC,WAAS;IAClI;IACA,IAAI2B,OAAO2B,aAAa,GAAG3B,OAAO4B,eAAe,GAAG5B,OAAO6B,cAAc,GAAG,GAAGlB,MAAMJ,IAAI,CAAC;IAC1FgB,SAAS,aAAavB,OAAO8B,QAAQ,EAAE9B,OAAO2B,aAAa;IAC3DJ,SAAS,cAAcvB,OAAO+B,UAAU,EAAE/B,OAAO4B,eAAe;IAChEL,SAAS,aAAavB,OAAOgC,SAAS,EAAEhC,OAAO6B,cAAc;IAC7D,OAAOlB,MAAMnB,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\n// Table output is for reading; a row wider than the terminal wraps into an unreadable\n// block (a `find` header measured 330 chars once `summary` is present, which drove one\n// user to pipe --format json through a script for every interactive query). Widest\n// columns give up space first, down to a floor, and only as far as the terminal needs.\n// --format json is never truncated: it is the machine-readable surface.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: 'table' | 'json', width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer verbs; cli.ts prints what these return.\n\nfunction featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, `${featuresLine(result.features)}\\n`, renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push('\\nhubs: off (features.rank)');\n  parts.push('\\nrecent:', renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["printRows","renderMap","renderPeek","OVERSIZE_BYTES","cell","value","String","replace","rows","format","rendered","renderRows","console","log","length","warn","Math","round","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","process","stdout","columns","JSON","stringify","Object","keys","cells","map","row","col","natural","i","clip","w","padEnd","slice","formatRow","values","join","trimEnd","repeat","featuresLine","features","off","name","on","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","lines","path","tokens","entries","frontmatter","key","sections","s","level","heading","start_line","end_line","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;;;;;;;;;;;;QAYjBA;eAAAA;;QAsDAC;eAAAA;;QASAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAvEhB,yFAAyF;AACzF,IAAMC,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEO,SAASP,UAAUQ,IAAW,EAAEC,MAAwB;IAC7D,IAAMC,WAAWC,WAAWH,MAAMC;IAClCG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGX,gBAAgB;QACpCS,QAAQG,IAAI,CAAC,AAAC,sBAAqEP,OAAhDQ,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,OAAM,eAAyB,OAAZN,KAAKM,MAAM,EAAC;IACjG;AACF;AAEA,sFAAsF;AACtF,uFAAuF;AACvF,mFAAmF;AACnF,uFAAuF;AACvF,wEAAwE;AACxE,IAAMI,aAAa;AACnB,IAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,IAAMC,OAAO,AAACF,CAAAA,OAAOP,MAAM,GAAG,CAAA,IAAK;IACnC,IAAMU,SAAU,qBAAGH;IACnB,OAAS;YAGuBL;QAF9B,IAAMS,QAAQD,OAAOE,MAAM,CAAC,SAACC,GAAGC;mBAAMD,IAAIC;WAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,IAAMK,SAASL,OAAOM,OAAO,CAACd,CAAAA,QAAAA,MAAKe,GAAG,OAARf,OAAS,qBAAGQ;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASX,WAAWH,IAAW,EAAEC,MAAwB;QAAEuB,QAAAA,iEAAQC,QAAQC,MAAM,CAACC,OAAO;QAM/CnB;IALxC,IAAIP,WAAW,QAAQ,OAAO2B,KAAKC,SAAS,CAAC7B,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,IAAMqB,UAAUG,OAAOC,IAAI,CAAC/B,IAAI,CAAC,EAAE;IACnC,IAAMgC,QAAQhC,KAAKiC,GAAG,CAAC,SAACC;eAAQP,QAAQM,GAAG,CAAC,SAACE;mBAAQvC,KAAKsC,GAAG,CAACC,IAAI;;;IAClE,IAAMC,UAAUT,QAAQM,GAAG,CAAC,SAACE,KAAKE;eAAM7B,CAAAA,QAAAA,MAAKe,GAAG,OAARf,OAAAA;YAAS2B,IAAI7B,MAAM;SAAuC,CAA1DE,OAAqB,qBAAGwB,MAAMC,GAAG,CAAC,SAACC;mBAAQA,GAAG,CAACG,EAAE,CAAC/B,MAAM;;;IAChG,IAAMO,SAASW,SAASA,QAAQd,aAAaE,UAAUwB,SAASZ,SAASY;IAEzE,IAAME,OAAO,cAACzC,OAAe0C;eAAe1C,MAAMS,MAAM,IAAIiC,IAAI1C,MAAM2C,MAAM,CAACD,KAAK,AAAC,GAA0B5B,OAAxBd,MAAM4C,KAAK,CAAC,GAAGF,IAAI,IAAe,OAAV5B;;IAC7G,IAAM+B,YAAY,mBAACC;eACjBA,OACGV,GAAG,CAAC,SAACpC,OAAOwC;mBAAMC,KAAKzC,OAAOgB,MAAM,CAACwB,EAAE;WACvCO,IAAI,CAAC,MACLC,OAAO;;IAEZ,OAAO;QAACH,UAAUf;QAAUd,OAAOoB,GAAG,CAAC,SAACM;mBAAM,IAAIO,MAAM,CAACP;WAAIK,IAAI,CAAC;KAA+B,CAA1F,OAAkE,qBAAGZ,MAAMC,GAAG,CAACS,aAAYE,IAAI,CAAC;AACzG;AAEA,uEAAuE;AAEvE,SAASG,aAAaC,QAAyC;IAC7D,IAAMC,MAAMD,SAASC,GAAG,CAAC3C,MAAM,GAAG,IAAI,AAAC,cAA8E,OAApE0C,SAASC,GAAG,CAAChB,GAAG,CAAC,SAACiB;eAAS,AAAC,GAAoBA,OAAlBA,MAAK,eAAkB,OAALA,MAAK;OAAIN,IAAI,CAAC,SAAU;IACzH,OAAO,AAAC,aAAqCK,OAAzBD,SAASG,EAAE,CAACP,IAAI,CAAC,OAAY,OAAJK;AAC/C;AAEO,SAASxD,UAAU2D,MAA6J;IACrL,IAAMC,QAAQ;QAAE,SAA8B7C,OAAtB4C,OAAOE,IAAI,CAACC,KAAK,EAAC,MAAyC,OAArC/C,KAAKC,KAAK,CAAC2C,OAAOE,IAAI,CAACE,KAAK,GAAG,OAAM;QAAQ,GAAgC,OAA9BT,aAAaK,OAAOJ,QAAQ,GAAE;QAAK7C,WAAWiD,OAAOK,MAAM,EAAE;KAAS;IACnK,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACnD,MAAM,EAAE+C,MAAMM,IAAI,CAAC,AAAC,KAA8C,OAA1CP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACnD,MAAM,EAAC;IACzG,IAAI8C,OAAOQ,IAAI,CAACtD,MAAM,GAAG,GAAG+C,MAAMM,IAAI,CAAC,0BAA0BxD,WAAWiD,OAAOQ,IAAI,EAAE;SACpF,IAAIR,OAAOJ,QAAQ,CAACC,GAAG,CAACY,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC;IAC1DN,MAAMM,IAAI,CAAC,aAAaxD,WAAWiD,OAAOU,MAAM,EAAE;IAClD,OAAOT,MAAMT,IAAI,CAAC;AACpB;AAEO,SAASlD,WAAW0D,MAAiO;IAC1P,IAAMW,QAAQ;QAAE,GAAoBX,OAAlBA,OAAOY,IAAI,EAAC,QAAoB,OAAdZ,OAAOa,MAAM,EAAC;KAAU;QACvD,kCAAA,2BAAA;;QAAL,QAAK,YAAsBnC,OAAOoC,OAAO,CAACd,OAAOe,WAAW,sBAAvD,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOC,sBAAKvE;YAA8CkE,MAAMJ,IAAI,CAAC,AAAC,KAAY9D,OAARuE,KAAI,MAAU,OAANvE;;;QAAlF;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,IAAIuD,OAAOiB,QAAQ,CAAC/D,MAAM,GAAG,GAAG;QAC9ByD,MAAMJ,IAAI,CAAC,IAAI;YACV,mCAAA,4BAAA;;YAAL,QAAK,aAAWP,OAAOiB,QAAQ,qBAA1B,UAAA,8BAAA,SAAA,0BAAA;gBAAA,IAAMC,IAAN;gBAA4BP,MAAMJ,IAAI,CAAC,AAAC,KAAqCW,OAAjC,IAAIxB,MAAM,CAACwB,EAAEC,KAAK,GAAY,KAAmBD,OAAhBA,EAAEE,OAAO,EAAC,QAAsBF,OAAhBA,EAAEG,UAAU,EAAC,KAAmBH,OAAhBA,EAAEI,QAAQ,EAAC,OAAc,OAATJ,EAAEL,MAAM,EAAC;;;YAAtI;YAAA;;;qBAAA,8BAAA;oBAAA;;;oBAAA;0BAAA;;;;IACP,OAAO,IAAIb,OAAOH,GAAG,CAACY,QAAQ,CAAC,aAAaE,MAAMJ,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOH,GAAG,CAACY,QAAQ,CAAC,UAAU;QAChCE,MAAMJ,IAAI,CAAC,IAAI;QACf,OAAOI,MAAMnB,IAAI,CAAC;IACpB;IACA,IAAM+B,WAAW,kBAACC,OAAeC,OAAiB5D;QAChD,IAAIA,QAAQ,GAAG8C,MAAMJ,IAAI,CAAC,AAAC,GAAY1C,OAAV2D,OAAM,MAAeC,OAAX5D,OAAM,OAAwBA,OAAnB4D,MAAMjC,IAAI,CAAC,OAAsE,OAA9D3B,QAAQ4D,MAAMvE,MAAM,GAAG,AAAC,MAA0B,OAArBW,QAAQ4D,MAAMvE,MAAM,EAAC,WAAS;IAClI;IACA,IAAI8C,OAAO0B,aAAa,GAAG1B,OAAO2B,eAAe,GAAG3B,OAAO4B,cAAc,GAAG,GAAGjB,MAAMJ,IAAI,CAAC;IAC1FgB,SAAS,aAAavB,OAAO6B,QAAQ,EAAE7B,OAAO0B,aAAa;IAC3DH,SAAS,cAAcvB,OAAO8B,UAAU,EAAE9B,OAAO2B,eAAe;IAChEJ,SAAS,aAAavB,OAAO+B,SAAS,EAAE/B,OAAO4B,cAAc;IAC7D,OAAOjB,MAAMnB,IAAI,CAAC;AACpB"}

package/dist/cjs/scan.d.cts

@@ -11,7 +11,7 @@ export interface ParsedDoc {
     relPath: string;
     mtimeMs: number;
     size: number;
-    data: Record<string, string | number | null>;
+    data: Record<string, string | number | bigint | null>;
     search: {
         title: string;
         summary: string;

package/dist/cjs/scan.d.ts

@@ -11,7 +11,7 @@ export interface ParsedDoc {
     relPath: string;
     mtimeMs: number;
     size: number;
-    data: Record<string, string | number | null>;
+    data: Record<string, string | number | bigint | null>;
     search: {
         title: string;
         summary: string;

package/dist/cjs/scan.js

@@ -68,10 +68,16 @@ function listFiles(cfg, baseDir) {
         };
     });
 }
+// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole
+// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.
+// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`
+// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so
+// that mismatch is visible rather than a silent zero-row answer.
 function mapValue(value) {
     if (value === null || value === undefined) return null;
-    if (typeof value === 'boolean') return value ? 1 : 0;
-    if (typeof value === 'string' || typeof value === 'number') return value;
+    if (typeof value === 'boolean') return BigInt(value ? 1 : 0);
+    if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;
+    if (typeof value === 'string') return value;
     return JSON.stringify(value);
 }
 // The delimiter split is all this package used gray-matter for.

package/dist/cjs/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\nfunction mapValue(value: unknown): string | number | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return value ? 1 : 0;\n  if (typeof value === 'string' || typeof value === 'number') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["listFiles","parseFile","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","cfg","baseDir","relPaths","fastGlob","sync","scan","include","cwd","sort","map","relPath","absPath","join","st","statSync","mtimeMs","size","mapValue","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","parseDocument","errors","push","message","split","data","toJS","err","Array","isArray","file","extractors","readFileSync","content","mapped","Object","keys","key","has","search","title","summary","text","extracted","fromEntries","filter","f","extract","name"],"mappings":";;;;;;;;;;;QAkCgBA;eAAAA;;QA2DAC;eAAAA;;;sBA7FuB;wBAClB;+DACA;qEACM;oBACG;;;;;;;;;;AAI9B,6EAA6E;AAE7E,8IAA8I;AAC9I,IAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAErG,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AASO,SAASd,UAAUe,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAWC,iBAAQ,CAACC,IAAI,CAACJ,IAAIK,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKN;IAAQ,GAAGO,IAAI;IACvE,OAAON,SAASO,GAAG,CAAC,SAACC;QACnB,IAAMC,UAAUC,IAAAA,cAAI,EAACX,SAASS;QAC9B,IAAMG,KAAKC,IAAAA,gBAAQ,EAACH;QACpB,OAAO;YAAED,SAAAA;YAASC,SAAAA;YAASI,SAASF,GAAGE,OAAO;YAAEC,MAAMH,GAAGG,IAAI;QAAC;IAChE;AACF;AAaA,SAASC,SAAS3B,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOA,QAAQ,IAAI;IACnD,IAAI,OAAOA,UAAU,YAAY,OAAOA,UAAU,UAAU,OAAOA;IACnE,OAAO4B,KAAKC,SAAS,CAAC7B;AACxB;AAEA,gEAAgE;AAChE,SAAS8B,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,IAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKvC,WAAW,OAAO;QAAEiC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBrB,OAAe,EAAEc,EAAU,EAAEQ,QAAkB;IACvE,IAAMC,MAAMC,IAAAA,mBAAa,EAACV;IAC1B,IAAIS,IAAIE,MAAM,CAACP,MAAM,GAAG,GAAG;QACzBI,SAASI,IAAI,CAAC,AAAC,YAAsCH,OAA3BvB,SAAQ,qBAAyDuB,OAAtCA,IAAIE,MAAM,CAACP,MAAM,EAAC,sBAAyD,OAArCK,IAAIE,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;IAClI;IACA,IAAIC;IACJ,IAAI;QACFA,OAAON,IAAIO,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZT,SAASI,IAAI,CAAC,AAAC,YAAmD,OAAxC1B,SAAQ,kCAAsE,OAAtC,AAAC+B,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;QACxG,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAShD,WAAW,OAAO,CAAC;IACjD,IAAI,CAAA,OAAOgD,qCAAP,SAAOA,KAAG,MAAM,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDP,SAASI,IAAI,CAAC,AAAC,YAAmB,OAAR1B,SAAQ;QAClC,OAAO,CAAC;IACV;IACA,OAAO6B;AACT;AAEO,SAASrD,UAAU0D,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMxB,MAAMyB,IAAAA,oBAAY,EAACF,KAAKjC,OAAO,EAAE;IACvC,IAAMqB,WAAqB,EAAE;IAE7B,IAA8BZ,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMsB,UAAY3B,kBAAlBK;IACZ,IAAMc,OAAOf,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAKlC,OAAO,EAAEc,IAAIQ;IACnE,IAAMgB,SAAiD,CAAC;QAEnD,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAOC,IAAI,CAACX,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMY,MAAN;YACH,IAAIhE,iBAAiBiE,GAAG,CAACD,MAAM;gBAC7BnB,SAASI,IAAI,CAAC,AAAC,YAAwDe,OAA7CP,KAAKlC,OAAO,EAAC,kCAAoC,OAAJyC,KAAI;gBAC3E;YACF;YACAH,MAAM,CAACG,IAAI,GAAGlC,SAASsB,IAAI,CAACY,IAAI;QAClC;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAME,SAAS;QAAEC,OAAOjE,cAAckD,KAAKe,KAAK;QAAGC,SAASlE,cAAckD,KAAKgB,OAAO;QAAGC,MAAM7D,UAAUoD;IAAS;IAElH,OAAO;QACLd,KAAK;YACHvB,SAASkC,KAAKlC,OAAO;YACrBK,SAAS6B,KAAK7B,OAAO;YACrBC,MAAM4B,KAAK5B,IAAI;YACfuB,MAAMS;YACNK,QAAAA;YACAI,WAAWR,OAAOS,WAAW,CAACb,WAAWc,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAEpD,GAAG,CAAC,SAACmD;oBAAeA;uBAAT;oBAACA,EAAEE,IAAI;qBAAEF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYvC,KAAK0B,SAASM;iBAAQ;;QAC1H;QACArB,UAAAA;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or the other tables (`content`, `links`, `sections`).\nconst RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["listFiles","parseFile","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","removeMarkdown","withoutTables","cfg","baseDir","relPaths","fastGlob","sync","scan","include","cwd","sort","map","relPath","absPath","join","st","statSync","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","parseDocument","errors","push","message","split","data","toJS","err","Array","isArray","file","extractors","readFileSync","content","mapped","Object","keys","key","has","search","title","summary","text","extracted","fromEntries","filter","f","extract","name"],"mappings":";;;;;;;;;;;QAkCgBA;eAAAA;;QAiEAC;eAAAA;;;sBAnGuB;wBAClB;+DACA;qEACM;oBACG;;;;;;;;;;AAI9B,6EAA6E;AAE7E,8IAA8I;AAC9I,IAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW;AAErG,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,IAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,IAAMI,kBAAkBC,IAAAA,uBAAc,EAACF;IACvC,IAAMG,gBAAgBF,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcU;AACvB;AASO,SAASd,UAAUe,GAAW,EAAEC,OAAe;IACpD,IAAMC,WAAWC,iBAAQ,CAACC,IAAI,CAACJ,IAAIK,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKN;IAAQ,GAAGO,IAAI;IACvE,OAAON,SAASO,GAAG,CAAC,SAACC;QACnB,IAAMC,UAAUC,IAAAA,cAAI,EAACX,SAASS;QAC9B,IAAMG,KAAKC,IAAAA,gBAAQ,EAACH;QACpB,OAAO;YAAED,SAAAA;YAASC,SAAAA;YAASI,SAASF,GAAGE,OAAO;YAAEC,MAAMH,GAAGG,IAAI;QAAC;IAChE;AACF;AAaA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASC,SAAS3B,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAO4B,OAAO5B,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO6B,OAAOC,aAAa,CAAC9B,SAAS4B,OAAO5B,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAO+B,KAAKC,SAAS,CAAChC;AACxB;AAEA,gEAAgE;AAChE,SAASiC,iBAAiBC,GAAW;IACnC,IAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,IAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,IAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAK1C,WAAW,OAAO;QAAEoC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBxB,OAAe,EAAEiB,EAAU,EAAEQ,QAAkB;IACvE,IAAMC,MAAMC,IAAAA,mBAAa,EAACV;IAC1B,IAAIS,IAAIE,MAAM,CAACP,MAAM,GAAG,GAAG;QACzBI,SAASI,IAAI,CAAC,AAAC,YAAsCH,OAA3B1B,SAAQ,qBAAyD0B,OAAtCA,IAAIE,MAAM,CAACP,MAAM,EAAC,sBAAyD,OAArCK,IAAIE,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;IAClI;IACA,IAAIC;IACJ,IAAI;QACFA,OAAON,IAAIO,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZT,SAASI,IAAI,CAAC,AAAC,YAAmD,OAAxC7B,SAAQ,kCAAsE,OAAtC,AAACkC,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAC;QACxG,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAASnD,WAAW,OAAO,CAAC;IACjD,IAAI,CAAA,OAAOmD,qCAAP,SAAOA,KAAG,MAAM,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDP,SAASI,IAAI,CAAC,AAAC,YAAmB,OAAR7B,SAAQ;QAClC,OAAO,CAAC;IACV;IACA,OAAOgC;AACT;AAEO,SAASxD,UAAU6D,IAAc;QAAEC,aAAAA,iEAAwB,EAAE;IAClE,IAAMxB,MAAMyB,IAAAA,oBAAY,EAACF,KAAKpC,OAAO,EAAE;IACvC,IAAMwB,WAAqB,EAAE;IAE7B,IAA8BZ,oBAAAA,iBAAiBC,MAAvCG,KAAsBJ,kBAAtBI,IAAIC,AAAMsB,UAAY3B,kBAAlBK;IACZ,IAAMc,OAAOf,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAKrC,OAAO,EAAEiB,IAAIQ;IACnE,IAAMgB,SAA0D,CAAC;QAE5D,kCAAA,2BAAA;;QAAL,QAAK,YAAaC,OAAOC,IAAI,CAACX,0BAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAAgC;YAAhC,IAAMY,MAAN;YACH,IAAInE,iBAAiBoE,GAAG,CAACD,MAAM;gBAC7BnB,SAASI,IAAI,CAAC,AAAC,YAAwDe,OAA7CP,KAAKrC,OAAO,EAAC,kCAAoC,OAAJ4C,KAAI;gBAC3E;YACF;YACAH,MAAM,CAACG,IAAI,GAAGrC,SAASyB,IAAI,CAACY,IAAI;QAClC;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAQL,oEAAoE;IACpE,0CAA0C;IAC1C,IAAME,SAAS;QAAEC,OAAOpE,cAAcqD,KAAKe,KAAK;QAAGC,SAASrE,cAAcqD,KAAKgB,OAAO;QAAGC,MAAMhE,UAAUuD;IAAS;IAElH,OAAO;QACLd,KAAK;YACH1B,SAASqC,KAAKrC,OAAO;YACrBK,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACf0B,MAAMS;YACNK,QAAAA;YACAI,WAAWR,OAAOS,WAAW,CAACb,WAAWc,MAAM,CAAC,SAACC;uBAAMA,EAAEC,OAAO;eAAEvD,GAAG,CAAC,SAACsD;oBAAeA;uBAAT;oBAACA,EAAEE,IAAI;qBAAEF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYvC,KAAK0B,SAASM;iBAAQ;;QAC1H;QACArB,UAAAA;IACF;AACF"}