tinywebui 0.1.0

package/src/retrieval/embedding.js

@@ -0,0 +1,298 @@
+import { createHash } from 'node:crypto';
+import { createReadStream, existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+/**
+ * Local ONNX sentence embeddings for document retrieval, done the way
+ * TinySearch and TinyContext do it: the same model bundles (ONNX graph plus
+ * tokenizer files from the same Hugging Face repos), CPU inference, the same
+ * pooling and normalisation per model, and a model key stored with every
+ * vector so a model change re-embeds instead of mixing vectors.
+ *
+ * One deliberate difference: those services fetch a missing bundle on first
+ * start. The TinyWebUI server never downloads anything -- a bundle is a
+ * deployment artifact, fetched by `tinywebui models pull` / `models ensure`
+ * (which `npm start` runs first) or baked into the image, and optionally
+ * pinned by checksum.
+ *
+ * onnxruntime-node and @huggingface/tokenizers are optional dependencies,
+ * imported only when an embedder is loaded. Without them (or without the
+ * bundle) the default mode 'auto' runs lexical -- see resolveRetrieval.
+ */
+
+// fast/balanced/quality mirror TinySearch's _PRESET_MODELS
+// (services/embedding_service.py); all three are English-only.
+export const PRESETS = {
+  fast: {
+    repoId: 'onnx-models/all-MiniLM-L6-v2-onnx',
+    localDir: 'all-minilm-l6-v2-onnx',
+    onnxPaths: ['model.onnx'],
+    pooling: 'auto',
+    normalize: false,
+    maxLength: 256,
+    files: ['model.onnx', 'tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.txt']
+  },
+  balanced: {
+    repoId: 'BAAI/bge-small-en-v1.5',
+    localDir: 'bge-small-en-v1.5-onnx',
+    onnxPaths: ['onnx/model.onnx', 'model.onnx'],
+    pooling: 'cls',
+    normalize: true,
+    maxLength: 512,
+    files: ['onnx/model.onnx', 'tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.txt', 'config.json']
+  },
+  quality: {
+    repoId: 'BAAI/bge-base-en-v1.5',
+    localDir: 'bge-base-en-v1.5-onnx',
+    onnxPaths: ['onnx/model.onnx', 'model.onnx'],
+    pooling: 'cls',
+    normalize: true,
+    maxLength: 512,
+    files: ['onnx/model.onnx', 'tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'vocab.txt', 'config.json']
+  },
+  // About as fast as `fast` (same 384 dims), but matches across languages:
+  // an English question finds a German passage. No query/passage prefixes.
+  // CLS pooling per the model card -- the export's other 2-D output is the
+  // tanh pooler, not the sentence embedding.
+  multilingual: {
+    repoId: 'ibm-granite/granite-embedding-107m-multilingual',
+    localDir: 'granite-embedding-107m-multilingual-onnx',
+    onnxPaths: ['model.onnx'],
+    pooling: 'cls',
+    normalize: true,
+    maxLength: 512,
+    files: ['model.onnx', 'tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json', 'config.json']
+  }
+};
+
+/** Where bundles live unless retrieval.modelDir says otherwise. */
+export function defaultModelsDir(configPath) {
+  if (process.env.TINYWEBUI_MODELS_DIR) return resolve(process.env.TINYWEBUI_MODELS_DIR);
+  return resolve(configPath ? join(configPath, '..') : process.cwd(), 'models');
+}
+
+/**
+ * The model a retrieval config names: a preset (by name) with its bundle
+ * directory, or a custom bundle given by modelDir (pooling 'auto', no
+ * normalisation, 512 tokens -- TinySearch's defaults for custom models).
+ */
+export function resolveModel(retrieval, modelsDir) {
+  const name = retrieval.model || 'fast';
+  const preset = PRESETS[name];
+  if (preset) return { name, ...preset, dir: retrieval.modelDir ? resolve(retrieval.modelDir) : join(modelsDir, preset.localDir) };
+  if (!retrieval.modelDir) throw new Error(`retrieval.model "${name}" is not a preset (${Object.keys(PRESETS).join(', ')}); set retrieval.modelDir for a custom bundle`);
+  return {
+    name, repoId: name, localDir: null, dir: resolve(retrieval.modelDir),
+    onnxPaths: ['model.onnx', 'onnx/model.onnx'], pooling: 'auto', normalize: false, maxLength: 512, files: []
+  };
+}
+
+export function findOnnxFile(spec) {
+  for (const rel of spec.onnxPaths) {
+    const p = join(spec.dir, rel);
+    if (existsSync(p) && statSync(p).isFile()) return p;
+  }
+  return null;
+}
+
+export async function sha256File(path) {
+  const hash = createHash('sha256');
+  for await (const chunk of createReadStream(path)) hash.update(chunk);
+  return hash.digest('hex');
+}
+
+/**
+ * Identifies the vectors a model produces. Like TinyContext's
+ * embedding_model_key: model identity plus a digest of the document prefix,
+ * and here the file checksum too, so a swapped model file never reuses old
+ * vectors.
+ */
+export function modelKey(spec, onnxSha256, documentPrefix = '', chunkOverlap = 32) {
+  const prefix = createHash('sha256').update(documentPrefix).digest('hex').slice(0, 12);
+  // Chunking is part of what a stored vector means: a different overlap
+  // cuts different chunks, so it re-embeds rather than mixing.
+  return `onnx:${spec.repoId}:${onnxSha256.slice(0, 16)}:document-prefix:${prefix}:chunks:${spec.maxLength}/${chunkOverlap}`;
+}
+
+// Marks "not installed here" (packages or bundle), as opposed to a broken or
+// mismatched install: only the former lets mode 'auto' fall back to lexical.
+const UNAVAILABLE = 'EMBEDDING_UNAVAILABLE';
+const unavailable = (message) => Object.assign(new Error(message), { code: UNAVAILABLE });
+
+async function importPeer(name) {
+  try { return await import(name); } catch (err) {
+    if (err?.code === 'ERR_MODULE_NOT_FOUND' || /Cannot find (package|module)/.test(err?.message || '')) {
+      throw unavailable(`dense/hybrid retrieval needs the optional packages onnxruntime-node and @huggingface/tokenizers (missing ${name}; npm install without --omit=optional)`);
+    }
+    throw err;
+  }
+}
+
+/**
+ * What retrieval actually runs as: { mode, embedder, fallback }. 'auto' (the
+ * default) is hybrid when the model and packages are installed, and lexical
+ * otherwise, with `fallback` saying why. An explicit 'dense' or 'hybrid'
+ * never falls back -- a missing model stops startup, as asked. Anything
+ * wrong with an installed model (a checksum mismatch, a broken file) fails
+ * in every mode rather than being papered over.
+ */
+export async function resolveRetrieval(retrieval, modelsDir, load = loadEmbedder) {
+  if (retrieval.mode === 'lexical') return { mode: 'lexical', embedder: null, fallback: null };
+  const mode = retrieval.mode === 'auto' ? 'hybrid' : retrieval.mode;
+  try {
+    return { mode, embedder: await load(retrieval, modelsDir), fallback: null };
+  } catch (err) {
+    if (retrieval.mode !== 'auto' || err.code !== UNAVAILABLE) throw err;
+    return { mode: 'lexical', embedder: null, fallback: err.message };
+  }
+}
+
+const l2normalize = (v) => {
+  let n = 0;
+  for (const x of v) n += x * x;
+  n = Math.sqrt(n) || 1e-12;
+  for (let i = 0; i < v.length; i++) v[i] /= n;
+  return v;
+};
+
+/**
+ * Loads a bundle and returns { key, dim, embed(texts) -> Float32Array[] }.
+ * Fails clearly -- missing packages, missing bundle, checksum mismatch --
+ * and never reaches the network.
+ */
+export async function loadEmbedder(retrieval, modelsDir) {
+  const spec = resolveModel(retrieval, modelsDir);
+  const onnxPath = findOnnxFile(spec);
+  const tokenizerPath = join(spec.dir, 'tokenizer.json');
+  if (!onnxPath || !existsSync(tokenizerPath)) {
+    const hint = PRESETS[spec.name] ? `run: tinywebui models pull ${spec.name}` : 'the bundle needs an ONNX model and tokenizer.json';
+    throw unavailable(`no embedding bundle at ${spec.dir} (${hint}); the server never downloads models itself`);
+  }
+  const actual = await sha256File(onnxPath);
+  if (retrieval.modelSha256 && retrieval.modelSha256.toLowerCase() !== actual) {
+    throw new Error(`embedding model checksum mismatch for ${onnxPath}: expected ${retrieval.modelSha256}, got ${actual}`);
+  }
+
+  // ONNX Runtime ships telemetry; a local, private deployment keeps it off
+  // unless the operator explicitly says otherwise.
+  process.env.ORT_DISABLE_TELEMETRY ??= '1';
+  const ort = await importPeer('onnxruntime-node');
+  const { Tokenizer } = await importPeer('@huggingface/tokenizers');
+  const configPath = join(spec.dir, 'tokenizer_config.json');
+  const tokenizer = new Tokenizer(
+    JSON.parse(readFileSync(tokenizerPath, 'utf8')),
+    existsSync(configPath) ? JSON.parse(readFileSync(configPath, 'utf8')) : {}
+  );
+  const session = await ort.InferenceSession.create(onnxPath, { executionProviders: ['cpu'] });
+  const inputNames = new Set(session.inputNames);
+  const pooledOutput = session.outputNames.includes('sentence_embedding') ? 'sentence_embedding' : null;
+
+  const encode = (text) => {
+    let { ids } = tokenizer.encode(text);
+    // Truncate like TinySearch's enable_truncation: keep the closing special token.
+    if (ids.length > spec.maxLength) ids = [...ids.slice(0, spec.maxLength - 1), ids[ids.length - 1]];
+    return ids;
+  };
+
+  // The model's own wrapper tokens ([CLS] … [SEP] for the BERT-family presets).
+  const wrapper = tokenizer.encode('').ids;
+  const open = wrapper.length >= 1 ? wrapper.slice(0, 1) : [];
+  const close = wrapper.length >= 2 ? wrapper.slice(-1) : [];
+  const chunkOverlap = retrieval.chunkOverlap ?? 32;
+  /**
+   * Small-to-big: a passage (what read_document returns) is cut into chunks
+   * (what gets embedded). Chunk length is the model's own token limit --
+   * 256 for `fast`, 512 for the others -- so every token of a passage
+   * is embedded somewhere; neighbouring chunks overlap by chunkOverlap
+   * tokens so a sentence cut at a boundary still lands whole in one of them.
+   */
+  const chunksOf = (text) => {
+    const body = tokenizer.encode(text, { add_special_tokens: false }).ids;
+    const room = spec.maxLength - open.length - close.length;
+    if (body.length <= room) return [[...open, ...body, ...close]];
+    const out = [];
+    for (let start = 0; start < body.length; start += room - chunkOverlap) {
+      out.push([...open, ...body.slice(start, start + room), ...close]);
+      if (start + room >= body.length) break;
+    }
+    return out;
+  };
+
+  // One run at a time, like TinySearch's _EMBED_LOCK: ORT sessions are not
+  // meant for overlapping runs, and CPU inference gains nothing from it.
+  let queue = Promise.resolve();
+  const runLocked = (fn) => { const next = queue.then(fn, fn); queue = next.catch(() => {}); return next; };
+
+  async function embedBatch(encoded) {
+    const texts = encoded;
+    const maxLen = Math.max(1, ...encoded.map((ids) => ids.length));
+    const n = texts.length;
+    const ids = new BigInt64Array(n * maxLen);
+    const mask = new BigInt64Array(n * maxLen);
+    encoded.forEach((row, r) => row.forEach((id, c) => { ids[r * maxLen + c] = BigInt(id); mask[r * maxLen + c] = 1n; }));
+    const feeds = {};
+    const dims = [n, maxLen];
+    if (inputNames.has('input_ids')) feeds.input_ids = new ort.Tensor('int64', ids, dims);
+    if (inputNames.has('attention_mask')) feeds.attention_mask = new ort.Tensor('int64', mask, dims);
+    if (inputNames.has('token_type_ids')) feeds.token_type_ids = new ort.Tensor('int64', new BigInt64Array(n * maxLen), dims);
+
+    const results = await session.run(feeds);
+    let rows;
+    const pick = (rank) => Object.values(results).find((t) => t.dims.length === rank);
+    const pooled = pooledOutput ? results[pooledOutput] : spec.pooling === 'cls' ? null : pick(2);
+    if (pooled) {
+      const dim = pooled.dims[1];
+      rows = Array.from({ length: n }, (_, r) => Float32Array.from(pooled.data.subarray(r * dim, (r + 1) * dim)));
+    } else {
+      // CLS pooling: the first token's hidden state.
+      const tokens = pick(3);
+      if (!tokens) throw new Error(`ONNX model ${spec.repoId} has no usable embedding output`);
+      const [, seq, dim] = tokens.dims;
+      rows = Array.from({ length: n }, (_, r) => Float32Array.from(tokens.data.subarray(r * seq * dim, r * seq * dim + dim)));
+    }
+    return spec.normalize ? rows.map(l2normalize) : rows;
+  }
+
+  async function embedIds(idLists) {
+    const out = [];
+    for (let i = 0; i < idLists.length; i += 32) out.push(...await runLocked(() => embedBatch(idLists.slice(i, i + 32))));
+    return out;
+  }
+
+  /** One vector per text, truncated to the model's length (queries). */
+  const embed = (texts) => embedIds(texts.map(encode));
+
+  /** Each passage as its chunks' vectors: Float32Array[][] (passage -> chunks). */
+  async function embedPassages(texts) {
+    const chunks = texts.map(chunksOf);
+    const flat = await embedIds(chunks.flat());
+    let i = 0;
+    return chunks.map((c) => c.map(() => flat[i++]));
+  }
+
+  const [probe] = await embed(['probe']);
+  return {
+    key: modelKey(spec, actual, retrieval.documentPrefix || '', chunkOverlap),
+    chunkTokens: spec.maxLength,
+    dim: probe.length,
+    spec,
+    sha256: actual,
+    embed,
+    embedPassages,
+    close: () => session.release?.()
+  };
+}
+
+/** Files a preset bundle consists of, for `tinywebui models pull`. */
+export function bundleListing(dir) {
+  const out = [];
+  const walk = (d, rel = '') => {
+    for (const name of readdirSync(d)) {
+      const p = join(d, name);
+      if (statSync(p).isDirectory()) walk(p, `${rel}${name}/`);
+      else out.push(`${rel}${name}`);
+    }
+  };
+  if (existsSync(dir)) walk(dir);
+  return out.sort();
+}

package/src/retrieval/retrieval.js

@@ -0,0 +1,250 @@
+import { createHash } from 'node:crypto';
+import { CORPUS } from '../store/index.js';
+
+/**
+ * Search for everything the model can look things up in: lexical (FTS5 BM25,
+ * the default), dense (local embeddings), or hybrid (both, fused). One engine,
+ * any number of CORPORA -- documents and past chat turns ship built in.
+ *
+ * A corpus is a set of searchable UNITS (a document passage, a chat turn),
+ * each belonging to an OWNER (the document, the chat). It tells the engine:
+ *
+ *   units(owner)                  [{ id, text }] -- what to embed for an owner
+ *   ownersMissing(modelKey)       owners with units not yet embedded (backfill)
+ *   candidates(scope)             unit ids one search ranks
+ *   lexical(scope, query, {any})  Map(unit id -> BM25, higher is better)
+ *   hydrate(hits, scope)          the result rows, from [{ id, rank, dense, bm25 }]
+ *   ownersOf?(scope)              owners to embed before searching (else rely on ingest/backfill)
+ *   mutable?                      true when a unit's text can change under the same id
+ *   maintain?(engine, log)        one-time upkeep at backfill (e.g. re-splitting)
+ *
+ * Small-to-big: a unit is cut into CHUNKS sized to the embedding model, every
+ * chunk is embedded, a unit scores by its best chunk, and the caller gets the
+ * whole unit. BM25 scores units directly.
+ *
+ * The fusion is TinySearch's weighted Reciprocal Rank Fusion
+ * (hybrid_embed_search_service.py): each side ranks every candidate, which
+ * scores
+ *
+ *     (1 - denseWeight) / (k + bm25Rank) + denseWeight / (k + denseRank)
+ *
+ * with k = 60 and denseWeight = 0.5 by default, and ties break on the dense
+ * score, then the BM25 score, then candidate order -- fully deterministic.
+ *
+ * Vectors are computed once per unit, stored in SQLite (`embeddings`), and
+ * reused by every query; only the query itself is embedded at search time.
+ */
+
+export const RETRIEVAL_MODES = ['lexical', 'dense', 'hybrid'];
+
+const cosine = (a, b) => {
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
+  return na && nb ? dot / Math.sqrt(na * nb) : 0;
+};
+
+const digestOf = (text) => createHash('sha256').update(text).digest('hex').slice(0, 32);
+
+/**
+ * 1-based competition ranks by descending score: equal scores share a rank
+ * (1, 2, 2, 4). One deliberate departure from TinySearch's _rank_by_score,
+ * which numbers ties in input order -- there, a query with no BM25 match
+ * at all still hands the first candidate a lexical lead. Shared ranks make a
+ * side that cannot tell candidates apart stay neutral.
+ */
+function ranksOf(scores) {
+  const order = scores.map((s, i) => i).sort((a, b) => scores[b] - scores[a]);
+  const ranks = new Array(scores.length);
+  order.forEach((idx, r) => {
+    const prev = order[r - 1];
+    ranks[idx] = r > 0 && scores[prev] === scores[idx] ? ranks[prev] : r + 1;
+  });
+  return ranks;
+}
+
+/**
+ * Weighted RRF over parallel score lists. Returns candidate indices, best
+ * first, with the scores that decided them.
+ */
+export function fuse(bm25Scores, denseScores, { denseWeight = 0.5, k = 60 } = {}) {
+  const sparseWeight = 1 - denseWeight;
+  const bm25Ranks = ranksOf(bm25Scores);
+  const denseRanks = ranksOf(denseScores);
+  const rows = bm25Scores.map((_, i) => ({
+    index: i,
+    bm25: bm25Scores[i],
+    dense: denseScores[i],
+    rrf: (sparseWeight > 0 ? sparseWeight / (k + bm25Ranks[i]) : 0) + (denseWeight > 0 ? denseWeight / (k + denseRanks[i]) : 0)
+  }));
+  rows.sort((a, b) => b.rrf - a.rrf || b.dense - a.dense || b.bm25 - a.bm25 || a.index - b.index);
+  return rows;
+}
+
+/** Attached documents: one unit per passage. Scope: { docId }. */
+export function documentsCorpus(store) {
+  return {
+    units: (docId) => store.documents.passages(docId).map((p) => ({ id: p.rowid, text: p.body })),
+    ownersMissing: (key) => store.documents.missingVectors(key),
+    ownersOf: ({ docId }) => [docId],
+    candidates: ({ docId }) => store.documents.passages(docId).map((p) => p.rowid),
+    lexical: ({ docId }, query, { any }) => store.documents.lexicalScores(docId, query, { any }),
+    hydrate(hits, { docId }) {
+      const byId = new Map(store.documents.passages(docId).map((p) => [p.rowid, p]));
+      return hits.map((h) => {
+        const p = byId.get(h.id);
+        return { passageIdx: p.passageIdx, charStart: p.charStart, body: p.body, rank: h.rank, dense: h.dense, bm25: h.bm25 };
+      });
+    },
+    maintain(engine, log) {
+      const { passageSize, passageOverlap } = engine.passageSettings();
+      const stale = store.documents.withOtherPassages(passageSize, passageOverlap);
+      for (const id of stale) store.documents.repassage(id, passageSize, passageOverlap);
+      if (stale.length) log(`re-split ${stale.length} document(s) into ${passageSize}-character passages`);
+    }
+  };
+}
+
+// What of a turn gets embedded: the question and the start of the answer.
+const TURN_QUESTION_CHARS = 2000;
+const TURN_ANSWER_CHARS = 6000;
+
+/**
+ * Past conversations: one unit per turn -- a question and the answer it
+ * finally got. Scope: { userId, excludeChatId }. A turn's answer can still
+ * change (the run was going when it was last embedded), so it is mutable.
+ */
+export function chatsCorpus(store) {
+  const turnText = (t) => `${t.question.slice(0, TURN_QUESTION_CHARS)}\n\n${t.answer.slice(0, TURN_ANSWER_CHARS)}`.trim();
+  return {
+    mutable: true,
+    units: (chatId) => store.turns.forChat(chatId).map((t) => ({ id: t.id, text: turnText(t) })).filter((u) => u.text),
+    ownersMissing: (key) => store.turns.chatsMissingVectors(key),
+    candidates: ({ userId, excludeChatId }) => store.turns.ids(userId, { excludeChatId }),
+    lexical: ({ userId, excludeChatId }, query, { any }) => store.turns.lexicalScores(userId, query, { excludeChatId, any }),
+    hydrate(hits) {
+      const turns = store.turns.byIds(hits.map((h) => h.id));
+      return hits.filter((h) => turns.has(h.id)).map((h) => ({ ...turns.get(h.id), rank: h.rank, dense: h.dense, bm25: h.bm25 }));
+    }
+  };
+}
+
+/**
+ * The retrieval engine for one running instance. `embedder` is null in
+ * lexical mode; otherwise it is what loadEmbedder() returned (or a stand-in
+ * with the same { key, embed } shape, which is how the tests stay offline).
+ * Documents and chats are registered; `register` adds more.
+ */
+export class Retrieval {
+  constructor(store, retrieval, embedder = null) {
+    this.store = store;
+    this.cfg = retrieval;
+    // 'auto' means whatever the embedder allows (resolveRetrieval decides at startup).
+    const mode = retrieval.mode || 'lexical';
+    this.mode = mode === 'auto' ? (embedder ? 'hybrid' : 'lexical') : mode;
+    this.embedder = this.mode === 'lexical' ? null : embedder;
+    if (this.mode !== 'lexical' && !this.embedder) throw new Error(`retrieval.mode '${this.mode}' needs an embedding model`);
+    this.corpora = new Map();
+    this.pending = new Map(); // "corpus:owner" -> in-flight ingestion, so a query can wait for it
+    this.register(CORPUS.DOCUMENTS, documentsCorpus(store));
+    this.register(CORPUS.CHATS, chatsCorpus(store));
+  }
+
+  register(name, corpus) {
+    this.corpora.set(name, corpus);
+    return this;
+  }
+
+  #corpus(name) {
+    const c = this.corpora.get(name);
+    if (!c) throw new Error(`retrieval: no corpus "${name}"`);
+    return c;
+  }
+
+  /**
+   * Embeds whatever of one owner's units has no vectors for the current model
+   * (or, for a mutable corpus, vectors of text that has since changed), and
+   * drops vectors of units the owner no longer has. Idempotent: an owner that
+   * is up to date costs nothing to ingest again.
+   */
+  ingest(name, ownerId) {
+    if (!this.embedder) return Promise.resolve(0);
+    const corpus = this.#corpus(name);
+    const tag = `${name}:${ownerId}`;
+    if (this.pending.has(tag)) return this.pending.get(tag);
+    const job = (async () => {
+      const key = this.embedder.key;
+      const units = corpus.units(ownerId).map((u) => ({ ...u, digest: corpus.mutable ? digestOf(u.text) : '' }));
+      const have = this.store.embeddings.digests(name, ownerId, key);
+      if ([...have.keys()].some((id) => !units.some((u) => u.id === id))) this.store.embeddings.dropStale(name, ownerId, units.map((u) => u.id));
+      const missing = units.filter((u) => have.get(u.id) !== u.digest);
+      if (!missing.length) return 0;
+      const prefix = this.cfg.documentPrefix || '';
+      const texts = missing.map((u) => prefix + u.text);
+      const vecs = this.embedder.embedPassages
+        ? await this.embedder.embedPassages(texts)
+        : (await this.embedder.embed(texts)).map((v) => [v]);
+      // The owner may have been deleted while it was being embedded.
+      const still = new Set(corpus.units(ownerId).map((u) => u.id));
+      const rows = missing.map((u, i) => ({ unitId: u.id, digest: u.digest, vecs: vecs[i] })).filter((r) => still.has(r.unitId));
+      if (rows.length) this.store.embeddings.put(name, ownerId, key, rows);
+      return rows.length;
+    })().finally(() => this.pending.delete(tag));
+    this.pending.set(tag, job);
+    return job;
+  }
+
+  /**
+   * Brings every corpus up to the current settings: each corpus's own upkeep
+   * (every mode), then embeds whatever the current model has no vectors for,
+   * dropping vectors from old models.
+   */
+  async backfill(log = () => {}) {
+    for (const corpus of this.corpora.values()) corpus.maintain?.(this, log);
+    if (!this.embedder) return 0;
+    const pruned = this.store.embeddings.prune(this.embedder.key);
+    if (pruned) log(`dropped ${pruned} chunk vector(s) from a previous model or chunking`);
+    let total = 0;
+    for (const [name, corpus] of this.corpora) {
+      let n = 0;
+      for (const owner of corpus.ownersMissing(this.embedder.key)) n += await this.ingest(name, owner);
+      if (n) log(`embedded ${n} ${name} unit(s) for ${this.mode} retrieval`);
+      total += n;
+    }
+    return total;
+  }
+
+  /** The passage split new documents get. */
+  passageSettings() {
+    return { passageSize: this.cfg.passageSize ?? 1800, passageOverlap: this.cfg.passageOverlap ?? 200 };
+  }
+
+  /**
+   * The best units of a corpus for a query, as that corpus hydrates them,
+   * each with the scores that ranked it. Same call in every mode.
+   */
+  async search(name, scope, query, limit = 5) {
+    const corpus = this.#corpus(name);
+    if (!String(query ?? '').trim()) return [];
+    if (this.mode === 'lexical') {
+      // Every query word must match, like a search box; best BM25 first.
+      const hits = [...corpus.lexical(scope, query, { any: false })]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, limit)
+        .map(([id, s]) => ({ id, rank: s, dense: null, bm25: s }));
+      return corpus.hydrate(hits, scope);
+    }
+    for (const owner of corpus.ownersOf?.(scope) || []) await this.ingest(name, owner);
+    const ids = corpus.candidates(scope);
+    if (!ids.length) return [];
+    const vectors = this.store.embeddings.vectors(name, this.embedder.key, ids);
+    const [q] = await this.embedder.embed([(this.cfg.queryPrefix || '') + query]);
+    const dense = ids.map((id) => (vectors.has(id) ? Math.max(...vectors.get(id).map((v) => cosine(q, v))) : -1));
+    const lexical = corpus.lexical(scope, query, { any: true });
+    const bm25 = ids.map((id) => lexical.get(id) ?? 0);
+    const denseWeight = this.mode === 'dense' ? 1 : (this.cfg.denseWeight ?? 0.5);
+    const hits = fuse(bm25, dense, { denseWeight, k: this.cfg.rrfK ?? 60 })
+      .slice(0, limit)
+      .map(({ index, rrf, dense: d, bm25: b }) => ({ id: ids[index], rank: rrf, dense: d, bm25: b }));
+    return corpus.hydrate(hits, scope);
+  }
+}

package/src/routes/admin.js

@@ -0,0 +1,112 @@
+import { json, readJson } from '../http.js';
+import { ALL_USERS, toView } from '../store/index.js';
+import { validateConfig, fingerprint, resolveAccess } from '../access/policy.js';
+import { destroyUserSessions } from '../access/auth.js';
+import { audit } from '../audit.js';
+import { FEATURE } from '../../public/shared/features.js';
+
+function adminUserView(u) {
+  return {
+    id: u.id, email: u.email, name: u.name ?? null, role: u.role, status: u.status,
+    createdAt: u.created_at, lastLoginAt: u.last_login_at, chatCount: u.chat_count ?? null
+  };
+}
+
+/**
+ * User management and oversight. They only mean something when there is more
+ * than one person; 'none' and 'single' are just you, and get admin_only.
+ */
+export function adminRoutes({ config, multiUser, refreshConfig, runs, source, store }) {
+
+  // Wraps a handler so it refuses outside multi-user deployments.
+  const multi = (handle) => (ctx) => (multiUser() ? handle(ctx) : json(ctx.res, 403, { error: 'admin_only' }));
+
+  return [
+    { method: 'GET', path: /^\/api\/admin\/users$/, feature: FEATURE.ADMIN, handle: multi(({ res }) => {
+      const pins = resolveAccess(config());
+      const code = source.codeAccessUsers();
+      const pinnedBy = (u) => (pins.bootstrapAdmins.includes(u.external_id) ? 'bootstrapAdmins'
+        : code[u.external_id] ? 'access.users' : null);
+      return json(res, 200, {
+        fingerprint: fingerprint(config(), source.loadMcpServers()),
+        users: store.users.list().map((u) => ({ ...adminUserView(u), pinnedInCode: pinnedBy(u) }))
+      });
+    }) },
+
+    { method: 'PATCH', path: /^\/api\/admin\/users\/([\w-]+)$/, feature: FEATURE.ADMIN, handle: multi(async ({ req, res, auth, params: [id] }) => {
+      const target = store.users.get(id);
+      if (!target) return json(res, 404, { error: 'no such user' });
+      const { role, status } = await readJson(req);
+      if (role !== undefined && !['admin', 'user'].includes(role)) return json(res, 400, { error: 'role must be admin or user' });
+      if (status !== undefined && !['pending', 'approved', 'disabled'].includes(status)) {
+        return json(res, 400, { error: 'status must be pending, approved or disabled' });
+      }
+      if (target.id === auth.userId && ((role && role !== 'admin') || (status && status !== 'approved'))) {
+        return json(res, 400, { error: 'you cannot demote or disable your own account' });
+      }
+      const pins = resolveAccess(config());
+      if (target.external_id && pins.bootstrapAdmins.includes(target.external_id)) {
+        return json(res, 400, { error: 'this admin is declared in code (access.bootstrapAdmins)' });
+      }
+      // The file is the record: the decision goes into config.json's
+      // access.users first, and only then into the database.
+      if (target.external_id) {
+        const file = source.readFile();
+        const access = file.access && typeof file.access === 'object' ? file.access : {};
+        const users = { ...(access.users || {}) };
+        users[target.external_id] = {
+          ...(users[target.external_id] || {}),
+          ...(role !== undefined ? { role } : {}),
+          ...(status !== undefined ? { status } : {})
+        };
+        const merged = { ...file, access: { ...access, users } };
+        const problems = validateConfig({ ...config(), access: merged.access });
+        if (problems.length) return json(res, 400, { error: problems.join('; ') });
+        if (source.codeAccessUsers()[target.external_id]) return json(res, 409, { error: 'this user is pinned in code (access.users)' });
+        source.writeFile(merged);
+        refreshConfig();
+      }
+      const updated = store.users.update(target.id, { role, status });
+      if (role !== undefined && role !== target.role) {
+        audit('user.role_changed', { userId: target.id, from: target.role, to: role, by: auth.userId });
+      }
+      if (status !== undefined && status !== target.status) {
+        audit('user.status_changed', { userId: target.id, from: target.status, to: status, by: auth.userId });
+        if (status !== 'approved') destroyUserSessions(store, target.id);
+      }
+      return json(res, 200, { user: adminUserView(updated) });
+    }) },
+
+    // Oversight: read-only, every view audited. Reached through ALL_USERS on
+    // purpose -- this is the one place a user's scope is crossed, and it only
+    // exists when there are users to cross between.
+    { method: 'GET', path: /^\/api\/admin\/users\/([\w-]+)\/chats$/, feature: FEATURE.OVERSIGHT, handle: multi(({ res, auth, params: [id] }) => {
+      const target = store.users.get(id);
+      if (!target) return json(res, 404, { error: 'no such user' });
+      audit('admin.view_user_chats', { by: auth.userId, userId: target.id });
+      const chats = store.chats.list(500, target.id).map((c) => ({ ...c, running: runs.isRunning(c.id) }));
+      return json(res, 200, { user: adminUserView(target), chats, summary: store.usage.statistics(target.id).summary });
+    }) },
+
+    { method: 'GET', path: /^\/api\/admin\/chats\/([\w.-]+)$/, feature: FEATURE.OVERSIGHT, handle: multi(({ res, auth, params: [id] }) => {
+      const found = store.chats.get(id, ALL_USERS);
+      if (!found) return json(res, 404, { error: 'no such chat' });
+      audit('admin.view_chat', { by: auth.userId, chatId: found.id, owner: found.user_id });
+      const owner = found.user_id ? store.users.get(found.user_id) : null;
+      return json(res, 200, {
+        id: found.id, title: found.title, updatedAt: found.updated_at,
+        owner: owner ? adminUserView(owner) : null,
+        running: runs.isRunning(found.id),
+        messages: store.messages.list(found.id).map(toView),
+        documents: store.documents.list(found.id)
+      });
+    }) },
+
+    { method: 'GET', path: /^\/api\/admin\/documents\/([\w.-]+)$/, feature: FEATURE.OVERSIGHT, handle: multi(({ res, auth, params: [id] }) => {
+      const doc = store.documents.get(id, ALL_USERS);
+      if (!doc) return json(res, 404, { error: 'no such document' });
+      audit('admin.view_document', { by: auth.userId, documentId: doc.id, chatId: doc.chat_id });
+      return json(res, 200, { filename: doc.filename, mime: doc.mime, content: doc.content });
+    }) }
+  ];
+}