tinywebui 0.1.4 → 0.1.5

package/src/cli.js

@@ -5,485 +5,587 @@ import { dirname, join } from 'node:path';
 import { PRESETS, resolveModel, findOnnxFile, sha256File, resolveRetrieval, defaultModelsDir } from './retrieval/embedding.js';
 import { createConfigSource, DEFAULTS, WRITABLE, configProblems } from './config/config.js';
 import { REASONING_REPLAY, MODEL_TAGS } from './config/models.js';
-import {
-  validateConfig, fingerprint, featuresFor, modelsFor, resolveAccess, keyClass,
-  FEATURES, ROLES, STATUSES, FILE_ONLY, SECRET_KEYS, CUSTOMIZABLE
-} from './access/policy.js';
+import { validateConfig, fingerprint, featuresFor, modelsFor, resolveAccess, keyClass, FEATURES, ROLES, STATUSES, FILE_ONLY, SECRET_KEYS, CUSTOMIZABLE } from './access/policy.js';
 import { Store, SCHEMA_VERSION, MigrationRequiredError } from './store/index.js';
-
 /**
  * The scriptable side of TinyWebUI: everything an agent (or a person) needs
  * to check, explain and change a deployment from files, without the UI.
  * Each command reads the same layers start() does and never writes a
  * generated sessionSecret as a side effect.
  */
-
 const out = (v) => process.stdout.write(typeof v === 'string' ? `${v}\n` : `${JSON.stringify(v, null, 2)}\n`);
 const flag = (args, name) => { const i = args.indexOf(`--${name}`); return i === -1 ? undefined : args[i + 1]; };
-
 function loadAll(opts) {
-  const source = createConfigSource(opts);
-  const cfg = source.load({ persistSecret: false });
-  let servers = null;
-  let mcpError = null;
-  try { servers = source.strictMcpServers(); } catch (err) { mcpError = `mcp.json: ${err.message}`; }
-  return { source, cfg, servers: servers || {}, mcpError };
+    const source = createConfigSource(opts);
+    const cfg = source.load({ persistSecret: false });
+    let servers = null;
+    let mcpError = null;
+    try {
+        servers = source.strictMcpServers();
+    }
+    catch (err) {
+        mcpError = `mcp.json: ${err.message}`;
+    }
+    return { source, cfg, servers: servers || {}, mcpError };
 }
-
 const mb = (n) => (n < 1048576 ? `${Math.ceil(n / 1024)} KB` : `${(n / 1048576).toFixed(1)} MB`);
-
 /**
  * Streams a response body to disk. On a terminal it redraws one progress
  * line (the model file is ~90 MB and otherwise looks like a hang); piped
  * output gets just the finished line so logs stay clean.
  */
 async function download(res, target, label) {
-  const total = Number(res.headers.get('content-length')) || 0;
-  const tty = process.stderr.isTTY;
-  let done = 0;
-  let last = 0;
-  const line = () => `  ${label}  ${mb(done)}${total ? ` / ${mb(total)} (${Math.floor((done / total) * 100)}%)` : ''}`;
-  const counter = new Transform({
-    transform(chunk, _enc, cb) {
-      done += chunk.length;
-      const now = Date.now();
-      if (tty && now - last > 100) { last = now; process.stderr.write(`\r${line()}\x1b[K`); }
-      cb(null, chunk);
-    }
-  });
-  await pipeline(Readable.fromWeb(res.body), counter, createWriteStream(target));
-  process.stderr.write(tty ? `\r${line()}\x1b[K\n` : `${line()}\n`);
+    const total = Number(res.headers.get('content-length')) || 0;
+    const tty = process.stderr.isTTY;
+    let done = 0;
+    let last = 0;
+    const line = () => `  ${label}  ${mb(done)}${total ? ` / ${mb(total)} (${Math.floor((done / total) * 100)}%)` : ''}`;
+    const counter = new Transform({
+        transform(chunk, _enc, cb) {
+            done += chunk.length;
+            const now = Date.now();
+            if (tty && now - last > 100) {
+                last = now;
+                process.stderr.write(`\r${line()}\x1b[K`);
+            }
+            cb(null, chunk);
+        }
+    });
+    await pipeline(Readable.fromWeb(res.body), counter, createWriteStream(target));
+    process.stderr.write(tty ? `\r${line()}\x1b[K\n` : `${line()}\n`);
 }
-
 /**
  * Downloads a preset bundle into `dest`, via a .partial folder so an
  * interrupted fetch never leaves a half bundle that looks complete.
  */
 async function pullPreset(name, dest) {
-  const preset = PRESETS[name];
-  const tmp = `${dest}.partial`;
-  rmSync(tmp, { recursive: true, force: true });
-  process.stderr.write(`[tinywebui] fetching ${preset.repoId} into ${dest} (see its Hugging Face model card for the license)\n`);
-  try {
-    for (const file of preset.files) {
-      const url = `https://huggingface.co/${preset.repoId}/resolve/main/${file}`;
-      const res = await fetch(url);
-      if (res.status === 404 && file !== preset.onnxPaths[0] && file !== 'tokenizer.json') continue; // companion files are optional
-      if (!res.ok) throw new Error(`${res.status} fetching ${url}`);
-      const target = join(tmp, file);
-      mkdirSync(dirname(target), { recursive: true });
-      await download(res, target, file);
-    }
-  } catch (err) {
+    const preset = PRESETS[name];
+    const tmp = `${dest}.partial`;
     rmSync(tmp, { recursive: true, force: true });
-    throw err;
-  }
-  rmSync(dest, { recursive: true, force: true });
-  mkdirSync(dirname(dest), { recursive: true });
-  renameSync(tmp, dest);
-  const onnx = findOnnxFile({ ...preset, dir: dest });
-  return { model: name, dir: dest, onnx, sha256: await sha256File(onnx) };
+    process.stderr.write(`[tinywebui] fetching ${preset.repoId} into ${dest} (see its Hugging Face model card for the license)\n`);
+    try {
+        for (const file of preset.files) {
+            const url = `https://huggingface.co/${preset.repoId}/resolve/main/${file}`;
+            const res = await fetch(url);
+            if (res.status === 404 && file !== preset.onnxPaths[0] && file !== 'tokenizer.json')
+                continue; // companion files are optional
+            if (!res.ok)
+                throw new Error(`${res.status} fetching ${url}`);
+            const target = join(tmp, file);
+            mkdirSync(dirname(target), { recursive: true });
+            await download(res, target, file);
+        }
+    }
+    catch (err) {
+        rmSync(tmp, { recursive: true, force: true });
+        throw err;
+    }
+    rmSync(dest, { recursive: true, force: true });
+    mkdirSync(dirname(dest), { recursive: true });
+    renameSync(tmp, dest);
+    const onnx = findOnnxFile({ ...preset, dir: dest });
+    return { model: name, dir: dest, onnx, sha256: await sha256File(onnx) };
 }
-
 function mcpProblems(servers) {
-  const problems = [];
-  for (const [name, spec] of Object.entries(servers)) {
-    if (!spec || (!spec.command && !spec.url)) problems.push(`mcp server "${name}" needs either "command" or "url"`);
-  }
-  return problems;
+    const problems = [];
+    for (const [name, spec] of Object.entries(servers)) {
+        if (!spec || (!spec.command && !spec.url))
+            problems.push(`mcp server "${name}" needs either "command" or "url"`);
+    }
+    return problems;
 }
-
 /** Server specs with credentials blanked: env values and header values. */
 function redactServers(servers) {
-  const clean = {};
-  for (const [name, spec] of Object.entries(servers)) {
-    const s = { ...spec };
-    if (s.env) s.env = Object.fromEntries(Object.keys(s.env).map((k) => [k, '<redacted>']));
-    if (s.headers) s.headers = Object.fromEntries(Object.keys(s.headers).map((k) => [k, '<redacted>']));
-    clean[name] = s;
-  }
-  return clean;
+    const clean = {};
+    for (const [name, spec] of Object.entries(servers)) {
+        const s = { ...spec };
+        if (s.env)
+            s.env = Object.fromEntries(Object.keys(s.env).map((k) => [k, '<redacted>']));
+        if (s.headers)
+            s.headers = Object.fromEntries(Object.keys(s.headers).map((k) => [k, '<redacted>']));
+        clean[name] = s;
+    }
+    return clean;
 }
-
 const commands = {
-  /** Exit 0 with the fingerprint when the files are valid; exit 1 listing every problem. */
-  validate(args, opts) {
-    const { source, cfg, servers, mcpError } = loadAll(opts);
-    // The merged config can be valid while config.json holds a mistake that
-    // code happens to shadow today; flag that too, before it surfaces later.
-    const raw = source.readFile();
-    const shadowed = raw.access === undefined ? []
-      : validateConfig({ authMode: 'none', access: raw.access }).map((p) => `config.json: ${p}`);
-    const problems = [...configProblems(cfg), ...shadowed, ...(mcpError ? [mcpError] : mcpProblems(servers))];
-    if (problems.length) {
-      for (const p of problems) process.stderr.write(`error: ${p}\n`);
-      return 1;
-    }
-    out(`ok ${fingerprint(cfg, servers)}`);
-    return 0;
-  },
-
-  fingerprint(args, opts) {
-    const { cfg, servers } = loadAll(opts);
-    out(fingerprint(cfg, servers));
-    return 0;
-  },
-
-  /**
-   * What is actually in effect, and how each setting can change. With
-   * --role, just what that role gets. Secrets show only as set/unset.
-   */
-  effective(args, opts) {
-    const { source, cfg, servers } = loadAll(opts);
-    const locked = source.lockedKeys();
-    const roleFor = (role) => ({
-      features: [...featuresFor(cfg, role)],
-      models: modelsFor(cfg, role)
-    });
-    const role = flag(args, 'role');
-    if (role) {
-      if (!ROLES.includes(role)) { process.stderr.write(`error: unknown role "${role}" (known: ${ROLES.join(', ')})\n`); return 1; }
-      out({ role, authMode: cfg.authMode, ...roleFor(role) });
-      return 0;
-    }
-    const settings = {};
-    for (const key of Object.keys(cfg).sort()) {
-      if (key === 'access') continue;
-      const value = SECRET_KEYS.has(key) ? (cfg[key] ? '<set>' : '<unset>') : cfg[key];
-      settings[key] = { value, change: WRITABLE.has(key) || FILE_ONLY.has(key) ? keyClass(key, locked) : 'file-only' };
-    }
-    const access = resolveAccess(cfg);
-    out({
-      fingerprint: fingerprint(cfg, servers),
-      files: { config: source.path(), mcp: source.mcpPath() || '(in code)' },
-      authMode: cfg.authMode,
-      roles: cfg.authMode === 'trusted-header'
-        ? Object.fromEntries(ROLES.map((r) => [r, roleFor(r)]))
-        : { you: roleFor(null) },
-      access: cfg.authMode === 'trusted-header'
-        ? { bootstrapAdmins: access.bootstrapAdmins, newUsers: access.newUsers, users: access.users }
-        : undefined,
-      mcpServers: { locked: source.mcpLocked, servers: redactServers(servers) },
-      settings
-    });
-    return 0;
-  },
-
-  /**
-   * users list            provisioned users, with any decision pinned in the files
-   * users set <id> [--role admin|user] [--status pending|approved|disabled] [--clear]
-   *                       records the decision in config.json (a running
-   *                       server picks it up through its file watch)
-   */
-  users(args, opts) {
-    const [sub, externalId] = args;
-    const { source, cfg } = loadAll(opts);
-    const access = resolveAccess(cfg);
-    if (sub === 'list' || !sub) {
-      const dbFile = source.dbPath(cfg);
-      const rows = dbFile !== ':memory:' && existsSync(dbFile) ? (() => {
-        const store = new Store(dbFile);
-        try { return store.users.list(); } finally { store.close(); }
-      })() : [];
-      const seen = new Set();
-      const users = rows.map((u) => {
-        seen.add(u.external_id);
-        return {
-          externalId: u.external_id, email: u.email, name: u.name, role: u.role, status: u.status,
-          chats: u.chat_count, lastLoginAt: u.last_login_at ? new Date(u.last_login_at).toISOString() : null,
-          pinned: access.bootstrapAdmins.includes(u.external_id) ? 'bootstrapAdmins' : access.users[u.external_id] || null
-        };
-      });
-      // Decisions for people who have not signed in yet still belong in the list.
-      for (const [id, pin] of Object.entries(access.users)) if (!seen.has(id)) users.push({ externalId: id, pinned: pin, provisioned: false });
-      for (const id of access.bootstrapAdmins) if (!seen.has(id)) users.push({ externalId: id, pinned: 'bootstrapAdmins', provisioned: false });
-      out(users);
-      return 0;
-    }
-    if (sub === 'set') {
-      if (!externalId) { process.stderr.write('usage: tinywebui users set <gateway user id> [--role r] [--status s] [--clear]\n'); return 1; }
-      if (!source.path()) { process.stderr.write('error: no config file to write to\n'); return 1; }
-      if (source.codeAccessUsers()[externalId] || access.bootstrapAdmins.includes(externalId)) {
-        process.stderr.write(`error: "${externalId}" is declared in tinywebui.config.js; change it there\n`);
-        return 1;
-      }
-      const role = flag(args, 'role');
-      const status = flag(args, 'status');
-      const clear = args.includes('--clear');
-      if (!clear && role === undefined && status === undefined) { process.stderr.write('error: nothing to set (use --role, --status or --clear)\n'); return 1; }
-      if (role !== undefined && !ROLES.includes(role)) { process.stderr.write(`error: role must be one of ${ROLES.join(', ')}\n`); return 1; }
-      if (status !== undefined && !STATUSES.includes(status)) { process.stderr.write(`error: status must be one of ${STATUSES.join(', ')}\n`); return 1; }
-      const file = source.readFile();
-      const fileAccess = file.access && typeof file.access === 'object' ? file.access : {};
-      const users = { ...(fileAccess.users || {}) };
-      if (clear) delete users[externalId];
-      else users[externalId] = { ...(users[externalId] || {}), ...(role ? { role } : {}), ...(status ? { status } : {}) };
-      source.writeFile({ ...file, access: { ...fileAccess, users } });
-      out(clear ? `cleared ${externalId}` : `${externalId}: ${JSON.stringify(users[externalId])}`);
-      return 0;
-    }
-    process.stderr.write(`error: unknown users command "${sub}" (list, set)\n`);
-    return 1;
-  },
-
-  schema() {
-    out(configSchema());
-    return 0;
-  },
-
-  /**
-   * models pull <fast|balanced|quality|multilingual> [--dir path]
-   *   Fetches a preset embedding bundle from Hugging Face -- the same repos
-   *   and files TinySearch uses (multilingual is TinyWebUI's own) -- into the models folder, then prints the
-   *   model's sha256 to pin as retrieval.modelSha256.
-   * models ensure
-   *   Pulls the configured preset only if it is not there yet; `npm start`
-   *   runs it first. Nothing to do under mode 'lexical' or with a custom
-   *   modelDir. Under 'auto' a failed pull is a warning and exit 0 -- the
-   *   server then starts lexical -- so being offline never blocks a start.
-   * models verify
-   *   Loads the configured bundle exactly as startup would and embeds a probe.
-   *
-   * pull and ensure are the only code that downloads a model; the server never does.
-   */
-  async models(args, opts) {
-    const [sub, name] = args;
-    const { source, cfg } = loadAll(opts);
-    const modelsDir = defaultModelsDir(source.path());
-    if (sub === 'pull') {
-      const preset = PRESETS[name];
-      if (!preset) { process.stderr.write(`usage: tinywebui models pull <${Object.keys(PRESETS).join('|')}> [--dir path]\n`); return 1; }
-      const dest = flag(args, 'dir') ? join(flag(args, 'dir')) : join(modelsDir, preset.localDir);
-      try { out(await pullPreset(name, dest)); return 0; } catch (err) { process.stderr.write(`error: ${err.message}\n`); return 1; }
-    }
-    if (sub === 'ensure') {
-      const { mode, model, modelDir } = cfg.retrieval;
-      if (mode === 'lexical' || modelDir || !PRESETS[model]) return 0;
-      const spec = resolveModel(cfg.retrieval, modelsDir);
-      if (findOnnxFile(spec) && existsSync(join(spec.dir, 'tokenizer.json'))) return 0;
-      try {
-        const pulled = await pullPreset(model, spec.dir);
-        process.stderr.write(`[tinywebui] embedding model ${model} ready (sha256 ${pulled.sha256})\n`);
+    /** Exit 0 with the fingerprint when the files are valid; exit 1 listing every problem. */
+    validate(args, opts) {
+        const { source, cfg, servers, mcpError } = loadAll(opts);
+        // The merged config can be valid while config.json holds a mistake that
+        // code happens to shadow today; flag that too, before it surfaces later.
+        const raw = source.readFile();
+        const shadowed = raw.access === undefined ? []
+            : validateConfig({ authMode: 'local', access: raw.access }).map((p) => `config.json: ${p}`);
+        const problems = [...configProblems(cfg), ...shadowed, ...(mcpError ? [mcpError] : mcpProblems(servers))];
+        if (problems.length) {
+            for (const p of problems)
+                process.stderr.write(`error: ${p}\n`);
+            return 1;
+        }
+        const inFile = source.secretsInFile();
+        if (inFile.length)
+            process.stderr.write(`note: ${source.path()} holds ${inFile.join(', ')}; the server moves them to ${source.secretsPath()} when it starts\n`);
+        out(`ok ${fingerprint(cfg, servers)}`);
         return 0;
-      } catch (err) {
-        if (mode !== 'auto') { process.stderr.write(`error: ${err.message}\n`); return 1; }
-        process.stderr.write(`[tinywebui] could not fetch embedding model ${model} (${err.message}); starting with lexical search -- set retrieval.mode to 'lexical' to skip this\n`);
+    },
+    fingerprint(args, opts) {
+        const { cfg, servers } = loadAll(opts);
+        out(fingerprint(cfg, servers));
         return 0;
-      }
-    }
-    if (sub === 'verify') {
-      const r = await resolveRetrieval(cfg.retrieval, modelsDir);
-      if (!r.embedder) { out(r.fallback ? `retrieval.mode is auto: lexical, because ${r.fallback}` : 'retrieval.mode is lexical: no embedding model needed'); return 0; }
-      const e = r.embedder;
-      out({ mode: r.mode, model: e.spec.name, dir: e.spec.dir, sha256: e.sha256, pinned: Boolean(cfg.retrieval.modelSha256), dims: e.dim, key: e.key });
-      e.close?.();
-      return 0;
-    }
-    process.stderr.write('usage: tinywebui models pull <preset> [--dir path] | models ensure | models verify\n');
-    return 1;
-  },
-
-  config(args, opts) {
-    if (args[0] !== 'show') { process.stderr.write('usage: tinywebui config show [--role user|admin]\n'); return 1; }
-    return commands.effective(args.slice(1), opts);
-  },
-
-  /**
-   * Brings the database to the current schema, as a deliberate deployment
-   * step (pair it with autoMigrate: false). --check changes nothing and
-   * exits 1 when a migration is due.
-   */
-  migrate(args, opts) {
-    const { source, cfg } = loadAll(opts);
-    const file = source.dbPath(cfg);
-    if (file === ':memory:') { out('in-memory database: nothing to migrate'); return 0; }
-    if (args.includes('--check')) {
-      if (!existsSync(file)) { out(`no database yet at ${file}; it will be created at schema ${SCHEMA_VERSION}`); return 0; }
-      try {
-        new Store(file, { migrate: false }).close();
-        out(`up to date: schema ${SCHEMA_VERSION}`);
+    },
+    /**
+     * What is actually in effect, and how each setting can change. With
+     * --role, just what that role gets. Secrets show only as set/unset.
+     */
+    effective(args, opts) {
+        const { source, cfg, servers } = loadAll(opts);
+        const locked = source.lockedKeys();
+        const roleFor = (role) => ({
+            features: [...featuresFor(cfg, role)],
+            models: modelsFor(cfg, role)
+        });
+        const role = flag(args, 'role');
+        if (role) {
+            if (!ROLES.includes(role)) {
+                process.stderr.write(`error: unknown role "${role}" (known: ${ROLES.join(', ')})\n`);
+                return 1;
+            }
+            out({ role, authMode: cfg.authMode, ...roleFor(role) });
+            return 0;
+        }
+        const settings = {};
+        const values = cfg;
+        for (const key of Object.keys(cfg).sort()) {
+            if (key === 'access')
+                continue;
+            const value = SECRET_KEYS.has(key) ? (values[key] ? '<set>' : '<unset>') : values[key];
+            settings[key] = { value, change: WRITABLE.has(key) || FILE_ONLY.has(key) ? keyClass(key, locked) : 'file-only' };
+        }
+        const access = resolveAccess(cfg);
+        out({
+            fingerprint: fingerprint(cfg, servers),
+            files: { config: source.path(), secrets: source.secretsPath(), mcp: source.mcpPath() || '(in code)' },
+            authMode: cfg.authMode,
+            roles: ['multiuser', 'trusted-header'].includes(cfg.authMode)
+                ? Object.fromEntries(ROLES.map((r) => [r, roleFor(r)]))
+                : { you: roleFor(null) },
+            access: ['multiuser', 'trusted-header'].includes(cfg.authMode)
+                ? { bootstrapAdmins: access.bootstrapAdmins, newUsers: access.newUsers, users: access.users }
+                : undefined,
+            mcpServers: { locked: source.mcpLocked, servers: redactServers(servers) },
+            settings
+        });
         return 0;
-      } catch (err) {
-        if (!(err instanceof MigrationRequiredError)) throw err;
-        process.stderr.write(`${err.message}\n`);
+    },
+    /**
+     * users list            provisioned users, with any decision pinned in the files
+     * users set <id> [--role admin|user] [--status pending|approved|disabled] [--clear]
+     *                       records the decision in config.json (a running
+     *                       server picks it up through its file watch)
+     */
+    users(args, opts) {
+        const [sub, externalId] = args;
+        const { source, cfg } = loadAll(opts);
+        const access = resolveAccess(cfg);
+        if (sub === 'list' || !sub) {
+            const dbFile = source.dbPath(cfg);
+            const rows = dbFile !== ':memory:' && existsSync(dbFile) ? (() => {
+                const store = new Store(dbFile);
+                try {
+                    return store.users.list();
+                }
+                finally {
+                    store.close();
+                }
+            })() : [];
+            const seen = new Set();
+            const users = rows.map((u) => {
+                seen.add(u.external_id);
+                return {
+                    externalId: u.external_id, email: u.email, name: u.name, role: u.role, status: u.status,
+                    chats: u.chat_count, lastLoginAt: u.last_login_at ? new Date(u.last_login_at).toISOString() : null,
+                    pinned: access.bootstrapAdmins.includes(u.external_id) ? 'bootstrapAdmins' : access.users[u.external_id] || null
+                };
+            });
+            // Decisions for people who have not signed in yet still belong in the list.
+            for (const [id, pin] of Object.entries(access.users))
+                if (!seen.has(id))
+                    users.push({ externalId: id, pinned: pin, provisioned: false });
+            for (const id of access.bootstrapAdmins)
+                if (!seen.has(id))
+                    users.push({ externalId: id, pinned: 'bootstrapAdmins', provisioned: false });
+            out(users);
+            return 0;
+        }
+        if (sub === 'set') {
+            if (!externalId) {
+                process.stderr.write('usage: tinywebui users set <gateway user id> [--role r] [--status s] [--clear]\n');
+                return 1;
+            }
+            if (!source.path()) {
+                process.stderr.write('error: no config file to write to\n');
+                return 1;
+            }
+            if (source.codeAccessUsers()[externalId] || access.bootstrapAdmins.includes(externalId)) {
+                process.stderr.write(`error: "${externalId}" is declared in tinywebui.config.js; change it there\n`);
+                return 1;
+            }
+            const role = flag(args, 'role');
+            const status = flag(args, 'status');
+            const clear = args.includes('--clear');
+            if (!clear && role === undefined && status === undefined) {
+                process.stderr.write('error: nothing to set (use --role, --status or --clear)\n');
+                return 1;
+            }
+            if (role !== undefined && !ROLES.includes(role)) {
+                process.stderr.write(`error: role must be one of ${ROLES.join(', ')}\n`);
+                return 1;
+            }
+            if (status !== undefined && !STATUSES.includes(status)) {
+                process.stderr.write(`error: status must be one of ${STATUSES.join(', ')}\n`);
+                return 1;
+            }
+            const file = source.readFile();
+            const fileAccess = file.access && typeof file.access === 'object' ? file.access : {};
+            const users = { ...(fileAccess.users || {}) };
+            if (clear)
+                delete users[externalId];
+            else
+                users[externalId] = { ...(users[externalId] || {}), ...(role ? { role } : {}), ...(status ? { status } : {}) };
+            source.writeFile({ ...file, access: { ...fileAccess, users } });
+            out(clear ? `cleared ${externalId}` : `${externalId}: ${JSON.stringify(users[externalId])}`);
+            return 0;
+        }
+        process.stderr.write(`error: unknown users command "${sub}" (list, set)\n`);
         return 1;
-      }
-    }
-    const store = new Store(file);
-    try {
-      out(store.migratedFrom === null ? `created ${file} at schema ${SCHEMA_VERSION}`
-        : store.migratedFrom === SCHEMA_VERSION ? `up to date: schema ${SCHEMA_VERSION}`
-        : `migrated ${file} from schema ${store.migratedFrom} to ${SCHEMA_VERSION}`);
-    } finally { store.close(); }
-    return 0;
-  },
-
-  /**
-   * Checks everything a deployment depends on, one line per check, and
-   * exits 1 if any failed: node version, config, database, the model
-   * endpoint, and each enabled MCP server (connected, then closed).
-   */
-  async doctor(args, opts) {
-    const checks = [];
-    const check = async (name, fn) => {
-      try { const detail = await fn(); checks.push({ check: name, ok: true, ...(detail ? { detail } : {}) }); }
-      catch (err) { checks.push({ check: name, ok: false, detail: err.message }); }
-    };
-    let loaded;
-    await check('node', () => {
-      // node:sqlite is available without a flag from 22.13.
-      const [major, minor] = process.versions.node.split('.').map(Number);
-      if (major < 22 || (major === 22 && minor < 13)) throw new Error(`node ${process.versions.node}; TinyWebUI needs 22.13 or later`);
-      return process.versions.node;
-    });
-    await check('config', () => {
-      loaded = loadAll(opts);
-      const problems = [...configProblems(loaded.cfg), ...(loaded.mcpError ? [loaded.mcpError] : mcpProblems(loaded.servers))];
-      if (problems.length) throw new Error(problems.join('; '));
-      return `fingerprint ${fingerprint(loaded.cfg, loaded.servers)}`;
-    });
-    if (loaded) {
-      const { source, cfg, servers } = loaded;
-      await check('database', () => {
+    },
+    schema() {
+        out(configSchema());
+        return 0;
+    },
+    /**
+     * models pull <fast|balanced|quality|multilingual> [--dir path]
+     *   Fetches a preset embedding bundle from Hugging Face -- the same repos
+     *   and files TinySearch uses (multilingual is TinyWebUI's own) -- into the models folder, then prints the
+     *   model's sha256 to pin as retrieval.modelSha256.
+     * models ensure
+     *   Pulls the configured preset only if it is not there yet; `npm start`
+     *   runs it first. Nothing to do under mode 'lexical' or with a custom
+     *   modelDir. Under 'auto' a failed pull is a warning and exit 0 -- the
+     *   server then starts lexical -- so being offline never blocks a start.
+     * models verify
+     *   Loads the configured bundle exactly as startup would and embeds a probe.
+     *
+     * pull and ensure are the only code that downloads a model; the server never does.
+     */
+    async models(args, opts) {
+        const [sub, name] = args;
+        const { source, cfg } = loadAll(opts);
+        const modelsDir = defaultModelsDir(source.path());
+        if (sub === 'pull') {
+            const preset = PRESETS[name];
+            if (!preset) {
+                process.stderr.write(`usage: tinywebui models pull <${Object.keys(PRESETS).join('|')}> [--dir path]\n`);
+                return 1;
+            }
+            const dest = flag(args, 'dir') ? join(flag(args, 'dir')) : join(modelsDir, preset.localDir);
+            try {
+                out(await pullPreset(name, dest));
+                return 0;
+            }
+            catch (err) {
+                process.stderr.write(`error: ${err.message}\n`);
+                return 1;
+            }
+        }
+        if (sub === 'ensure') {
+            const { mode, model, modelDir } = cfg.retrieval;
+            if (mode === 'lexical' || modelDir || !PRESETS[model])
+                return 0;
+            const spec = resolveModel(cfg.retrieval, modelsDir);
+            if (findOnnxFile(spec) && existsSync(join(spec.dir, 'tokenizer.json')))
+                return 0;
+            try {
+                const pulled = await pullPreset(model, spec.dir);
+                process.stderr.write(`[tinywebui] embedding model ${model} ready (sha256 ${pulled.sha256})\n`);
+                return 0;
+            }
+            catch (err) {
+                if (mode !== 'auto') {
+                    process.stderr.write(`error: ${err.message}\n`);
+                    return 1;
+                }
+                process.stderr.write(`[tinywebui] could not fetch embedding model ${model} (${err.message}); starting with lexical search -- set retrieval.mode to 'lexical' to skip this\n`);
+                return 0;
+            }
+        }
+        if (sub === 'verify') {
+            const r = await resolveRetrieval(cfg.retrieval, modelsDir);
+            if (!r.embedder) {
+                out(r.fallback ? `retrieval.mode is auto: lexical, because ${r.fallback}` : 'retrieval.mode is lexical: no embedding model needed');
+                return 0;
+            }
+            const e = r.embedder;
+            out({ mode: r.mode, model: e.spec.name, dir: e.spec.dir, sha256: e.sha256, pinned: Boolean(cfg.retrieval.modelSha256), dims: e.dim, key: e.key });
+            e.close?.();
+            return 0;
+        }
+        process.stderr.write('usage: tinywebui models pull <preset> [--dir path] | models ensure | models verify\n');
+        return 1;
+    },
+    config(args, opts) {
+        if (args[0] !== 'show') {
+            process.stderr.write('usage: tinywebui config show [--role user|admin]\n');
+            return 1;
+        }
+        return commands.effective(args.slice(1), opts);
+    },
+    /**
+     * Brings the database to the current schema, as a deliberate deployment
+     * step (pair it with autoMigrate: false). --check changes nothing and
+     * exits 1 when a migration is due.
+     */
+    migrate(args, opts) {
+        const { source, cfg } = loadAll(opts);
         const file = source.dbPath(cfg);
-        if (file === ':memory:') return 'in memory';
-        if (!existsSync(file)) return `${file} will be created`;
-        let store;
-        try { store = new Store(file, { migrate: false }); } catch (err) {
-          // Behind is fine when startup is allowed to migrate; it is only a
-          // failure for deployments that migrate as a separate step.
-          if (err instanceof MigrationRequiredError && cfg.autoMigrate !== false && !/newer/.test(err.message)) {
-            return `${err.message.split(':')[0]}; will migrate on start (autoMigrate)`;
-          }
-          throw err;
+        if (file === ':memory:') {
+            out('in-memory database: nothing to migrate');
+            return 0;
+        }
+        if (args.includes('--check')) {
+            if (!existsSync(file)) {
+                out(`no database yet at ${file}; it will be created at schema ${SCHEMA_VERSION}`);
+                return 0;
+            }
+            try {
+                new Store(file, { migrate: false }).close();
+                out(`up to date: schema ${SCHEMA_VERSION}`);
+                return 0;
+            }
+            catch (err) {
+                if (!(err instanceof MigrationRequiredError))
+                    throw err;
+                process.stderr.write(`${err.message}\n`);
+                return 1;
+            }
+        }
+        const store = new Store(file);
+        try {
+            out(store.migratedFrom === null ? `created ${file} at schema ${SCHEMA_VERSION}`
+                : store.migratedFrom === SCHEMA_VERSION ? `up to date: schema ${SCHEMA_VERSION}`
+                    : `migrated ${file} from schema ${store.migratedFrom} to ${SCHEMA_VERSION}`);
+        }
+        finally {
+            store.close();
         }
-        try { return `${file} at schema ${store.schemaVersion()}`; } finally { store.close(); }
-      });
-      await check('model endpoint', async () => {
-        const r = await fetch(`${cfg.baseUrl}/models`, {
-          headers: cfg.apiKey ? { authorization: `Bearer ${cfg.apiKey}` } : {},
-          signal: AbortSignal.timeout(8000)
+        return 0;
+    },
+    /**
+     * Checks everything a deployment depends on, one line per check, and
+     * exits 1 if any failed: node version, config, database, the model
+     * endpoint, and each enabled MCP server (connected, then closed).
+     */
+    async doctor(args, opts) {
+        const checks = [];
+        const check = async (name, fn) => {
+            try {
+                const detail = await fn();
+                checks.push({ check: name, ok: true, ...(detail ? { detail } : {}) });
+            }
+            catch (err) {
+                checks.push({ check: name, ok: false, detail: err.message });
+            }
+        };
+        let loaded;
+        await check('node', () => {
+            // node:sqlite is available without a flag from 22.13.
+            const [major, minor] = process.versions.node.split('.').map(Number);
+            if (major < 22 || (major === 22 && minor < 13))
+                throw new Error(`node ${process.versions.node}; TinyWebUI needs 22.13 or later`);
+            return process.versions.node;
         });
-        // Some OpenAI-compatible servers have no /models; reachable is what counts.
-        if (r.status >= 500) throw new Error(`${r.status} from ${cfg.baseUrl}/models`);
-        if (r.status === 401 || r.status === 403) throw new Error(`${r.status} from ${cfg.baseUrl}: check apiKey`);
-        return `${cfg.baseUrl} answered ${r.status}`;
-      });
-      await check('retrieval', async () => {
-        const r = await resolveRetrieval(cfg.retrieval, defaultModelsDir(source.path()));
-        if (r.fallback) return `lexical (SQLite FTS5) -- auto, because ${r.fallback}`;
-        if (!r.embedder) return 'lexical (SQLite FTS5)';
-        const e = r.embedder;
-        e.close?.();
-        return `${r.mode} with ${e.spec.repoId}, ${e.dim} dims${cfg.retrieval.modelSha256 ? ', checksum pinned' : ', checksum NOT pinned (set retrieval.modelSha256)'}`;
-      });
-      const { McpHub } = await import('./mcp.js');
-      for (const [name, spec] of Object.entries(servers)) {
-        if (spec?.disabled) { checks.push({ check: `mcp ${name}`, ok: true, detail: 'disabled' }); continue; }
-        await check(`mcp ${name}`, async () => {
-          const hub = await new McpHub({ [name]: spec }).connect();
-          try {
-            if (hub.errors.length) throw new Error(hub.errors.join('; '));
-            return `${hub.tools.length} tool(s)`;
-          } finally { await hub.close(); }
+        await check('config', () => {
+            loaded = loadAll(opts);
+            const problems = [...configProblems(loaded.cfg), ...(loaded.mcpError ? [loaded.mcpError] : mcpProblems(loaded.servers))];
+            if (problems.length)
+                throw new Error(problems.join('; '));
+            return `fingerprint ${fingerprint(loaded.cfg, loaded.servers)}`;
         });
-      }
+        if (loaded) {
+            const { source, cfg, servers } = loaded;
+            await check('database', () => {
+                const file = source.dbPath(cfg);
+                if (file === ':memory:')
+                    return 'in memory';
+                if (!existsSync(file))
+                    return `${file} will be created`;
+                let store;
+                try {
+                    store = new Store(file, { migrate: false });
+                }
+                catch (err) {
+                    // Behind is fine when startup is allowed to migrate; it is only a
+                    // failure for deployments that migrate as a separate step.
+                    if (err instanceof MigrationRequiredError && cfg.autoMigrate !== false && !/newer/.test(err.message)) {
+                        return `${err.message.split(':')[0]}; will migrate on start (autoMigrate)`;
+                    }
+                    throw err;
+                }
+                try {
+                    return `${file} at schema ${store.schemaVersion()}`;
+                }
+                finally {
+                    store.close();
+                }
+            });
+            const endpoints = [
+                ...(cfg.baseUrl ? [{ name: 'model endpoint', baseUrl: cfg.baseUrl, apiKey: cfg.apiKey }] : []),
+                ...cfg.connectors.map((c) => ({ name: `connector ${c.id}`, baseUrl: String(c.baseUrl).replace(/\/+$/, ''), apiKey: c.apiKey || '' }))
+            ];
+            if (!endpoints.length)
+                await check('model endpoint', () => { throw new Error('none: add a connector and its models in Settings'); });
+            for (const ep of endpoints) {
+                await check(ep.name, async () => {
+                    const r = await fetch(`${ep.baseUrl}/models`, {
+                        headers: ep.apiKey ? { authorization: `Bearer ${ep.apiKey}` } : {},
+                        signal: AbortSignal.timeout(8000)
+                    });
+                    // Some OpenAI-compatible servers have no /models; reachable is what counts.
+                    if (r.status >= 500)
+                        throw new Error(`${r.status} from ${ep.baseUrl}/models`);
+                    if (r.status === 401 || r.status === 403)
+                        throw new Error(`${r.status} from ${ep.baseUrl}: check apiKey`);
+                    return `${ep.baseUrl} answered ${r.status}`;
+                });
+            }
+            await check('retrieval', async () => {
+                const r = await resolveRetrieval(cfg.retrieval, defaultModelsDir(source.path()));
+                if (r.fallback)
+                    return `lexical (SQLite FTS5) -- auto, because ${r.fallback}`;
+                if (!r.embedder)
+                    return 'lexical (SQLite FTS5)';
+                const e = r.embedder;
+                e.close?.();
+                return `${r.mode} with ${e.spec.repoId}, ${e.dim} dims${cfg.retrieval.modelSha256 ? ', checksum pinned' : ', checksum NOT pinned (set retrieval.modelSha256)'}`;
+            });
+            const { McpHub } = await import('./mcp.js');
+            for (const [name, spec] of Object.entries(servers)) {
+                if (spec?.disabled) {
+                    checks.push({ check: `mcp ${name}`, ok: true, detail: 'disabled' });
+                    continue;
+                }
+                await check(`mcp ${name}`, async () => {
+                    const hub = await new McpHub({ [name]: spec }).connect();
+                    try {
+                        if (hub.errors.length)
+                            throw new Error(hub.errors.join('; '));
+                        return `${hub.tools.length} tool(s)`;
+                    }
+                    finally {
+                        await hub.close();
+                    }
+                });
+            }
+        }
+        for (const c of checks)
+            out(JSON.stringify(c));
+        return checks.every((c) => c.ok) ? 0 : 1;
     }
-    for (const c of checks) out(JSON.stringify(c));
-    return checks.every((c) => c.ok) ? 0 : 1;
-  }
 };
-
 /** A JSON Schema for config.json (and the `config` block of tinywebui.config.js). */
 export function configSchema() {
-  const typeOf = (v) => (v === null ? ['number', 'null'] : Array.isArray(v) ? 'array' : typeof v);
-  const properties = {};
-  for (const [k, v] of Object.entries(DEFAULTS)) {
-    properties[k] = {
-      type: typeOf(v),
-      'x-change': FILE_ONLY.has(k) ? 'file-only' : WRITABLE.has(k) ? 'editable (UI writes it back here)' : 'file-only'
-    };
-  }
-  properties.authMode = { enum: ['none', 'single', 'trusted-header'], 'x-change': 'file-only' };
-  properties.toolApproval = { enum: ['writes', 'all', 'off'], 'x-change': 'editable (UI writes it back here)' };
-  properties.trustedProxyCidrs = { type: 'array', items: { type: 'string' }, 'x-change': 'file-only' };
-  const featureList = { oneOf: [{ const: '*' }, { type: 'array', items: { enum: FEATURES }, uniqueItems: true }] };
-  const role = {
-    type: 'object', additionalProperties: false,
-    properties: { features: featureList, models: { oneOf: [{ const: '*' }, { type: 'array', items: { type: 'string' } }] } }
-  };
-  properties.retrieval = {
-    type: 'object', additionalProperties: false, 'x-change': 'file-only (restart to apply)',
-    properties: {
-      mode: { enum: ['auto', 'lexical', 'dense', 'hybrid'], description: 'auto: hybrid when the model is installed, else lexical' },
-      model: { type: 'string', description: 'fast | balanced | quality | multilingual, or a name with modelDir' },
-      modelDir: { type: 'string' },
-      modelSha256: { type: 'string', pattern: '^([0-9a-fA-F]{64})?$' },
-      denseWeight: { type: 'number', exclusiveMinimum: 0, exclusiveMaximum: 1 },
-      rrfK: { type: 'integer', minimum: 0 },
-      queryPrefix: { type: 'string' },
-      documentPrefix: { type: 'string' }
-    }
-  };
-  const num = { type: ['number', 'null'] };
-  properties.connectors = {
-    type: 'array', 'x-change': 'editable (UI writes it back here)',
-    description: 'Named endpoints a catalog entry can use. apiKey is a secret: prefer an env var in committed config.',
-    items: {
-      type: 'object', additionalProperties: false, required: ['id', 'baseUrl'],
-      properties: { id: { type: 'string' }, label: { type: 'string' }, baseUrl: { type: 'string' }, apiKey: { type: 'string' } }
-    }
-  };
-  properties.models = {
-    type: 'array', 'x-change': 'editable (UI writes it back here)',
-    description: 'The model catalog. Empty: any model id. Listed: only these can be picked, by id, shown by label.',
-    items: {
-      type: 'object', additionalProperties: false, required: ['id'],
-      properties: {
-        id: { type: 'string', description: 'Stable handle: what model, roles and prefs name.' },
-        model: { type: 'string', description: "Provider model id; defaults to id." },
-        label: { type: 'string', description: 'What people see in the picker.' },
-        description: { type: 'string' },
-        enabled: { type: 'boolean', description: 'false: keep the settings, hide and refuse the model.' },
-        connector: { type: 'string', description: 'Id of an entry in connectors; absent: the default baseUrl/apiKey.' },
-        tags: { type: 'array', items: { enum: MODEL_TAGS }, uniqueItems: true, description: 'What the model can do.' },
-        systemPrompt: { type: 'string' },
-        temperature: num,
-        maxTokens: num,
-        maxToolRounds: { type: 'integer', minimum: 0 },
-        cacheMode: { enum: ['auto', 'implicit', 'explicit', 'rolling', 'off'] },
-        cacheTtl: { enum: ['5m', '1h'] },
-        reasoningReplay: { enum: REASONING_REPLAY },
-        extraBody: { type: 'object', description: 'Merged over the global extraBody.' }
-      }
+    const typeOf = (v) => (v === null ? ['number', 'null'] : Array.isArray(v) ? 'array' : typeof v);
+    const properties = {};
+    for (const [k, v] of Object.entries(DEFAULTS)) {
+        properties[k] = {
+            type: typeOf(v),
+            'x-change': FILE_ONLY.has(k) ? 'file-only' : WRITABLE.has(k) ? 'editable (UI writes it back here)' : 'file-only'
+        };
     }
-  };
-  properties.access = {
-    type: 'object', additionalProperties: false, 'x-change': 'file-only (admins write access.users back)',
-    properties: {
-      bootstrapAdmins: { type: 'array', items: { type: 'string' }, description: 'Gateway user ids that are always approved admins.' },
-      newUsers: { enum: ['approved', 'pending'] },
-      customize: { type: 'array', items: { enum: CUSTOMIZABLE }, uniqueItems: true, description: 'What users may personalise.' },
-      roles: { type: 'object', additionalProperties: false, properties: Object.fromEntries(ROLES.map((r) => [r, role])) },
-      users: {
-        type: 'object',
-        additionalProperties: {
-          type: 'object', additionalProperties: false,
-          properties: { role: { enum: ROLES }, status: { enum: STATUSES } }
+    properties.authMode = { enum: ['local', 'multiuser', 'trusted-header'], 'x-change': 'file-only' };
+    properties.toolApproval = { enum: ['writes', 'all', 'off'], 'x-change': 'editable (UI writes it back here)' };
+    properties.trustedProxyCidrs = { type: 'array', items: { type: 'string' }, 'x-change': 'file-only' };
+    const featureList = { oneOf: [{ const: '*' }, { type: 'array', items: { enum: FEATURES }, uniqueItems: true }] };
+    const role = {
+        type: 'object', additionalProperties: false,
+        properties: { features: featureList, models: { oneOf: [{ const: '*' }, { type: 'array', items: { type: 'string' } }] } }
+    };
+    properties.retrieval = {
+        type: 'object', additionalProperties: false, 'x-change': 'file-only (restart to apply)',
+        properties: {
+            mode: { enum: ['auto', 'lexical', 'dense', 'hybrid'], description: 'auto: hybrid when the model is installed, else lexical' },
+            model: { type: 'string', description: 'fast | balanced | quality | multilingual, or a name with modelDir' },
+            modelDir: { type: 'string' },
+            modelSha256: { type: 'string', pattern: '^([0-9a-fA-F]{64})?$' },
+            denseWeight: { type: 'number', exclusiveMinimum: 0, exclusiveMaximum: 1 },
+            rrfK: { type: 'integer', minimum: 0 },
+            queryPrefix: { type: 'string' },
+            documentPrefix: { type: 'string' }
         }
-      }
-    }
-  };
-  return {
-    $schema: 'https://json-schema.org/draft/2020-12/schema',
-    title: 'TinyWebUI configuration',
-    type: 'object',
-    properties
-  };
+    };
+    const num = { type: ['number', 'null'] };
+    const modelEntry = (nested) => ({
+        type: 'object', additionalProperties: false, required: ['id'],
+        properties: {
+            id: { type: 'string', description: 'Stable handle: what model, roles and prefs name.' },
+            model: { type: 'string', description: "Provider model id; defaults to id." },
+            label: { type: 'string', description: 'What people see in the picker.' },
+            description: { type: 'string' },
+            enabled: { type: 'boolean', description: 'false: keep the settings, hide and refuse the model.' },
+            ...(nested ? {} : { connector: { type: 'string', description: 'Older flat form: id of an entry in connectors. Prefer listing the model under that connector.' } }),
+            tags: { type: 'array', items: { enum: MODEL_TAGS }, uniqueItems: true, description: 'What the model can do.' },
+            systemPrompt: { type: 'string', description: 'Replaces the top-level systemPrompt for this model.' },
+            temperature: num,
+            maxTokens: num,
+            maxToolRounds: { type: 'integer', minimum: 0 },
+            cacheMode: { enum: ['auto', 'implicit', 'explicit', 'rolling', 'off'] },
+            cacheTtl: { enum: ['5m', '1h'] },
+            reasoningReplay: { enum: REASONING_REPLAY },
+            extraBody: { type: 'object', description: 'Merged over the global extraBody.' }
+        }
+    });
+    properties.connectors = {
+        type: 'array', 'x-change': 'editable (UI writes it back here)',
+        description: 'Named endpoints, each with the models it serves. apiKey is a secret: prefer an env var in committed config.',
+        items: {
+            type: 'object', additionalProperties: false, required: ['id', 'baseUrl'],
+            properties: {
+                id: { type: 'string' }, label: { type: 'string' }, baseUrl: { type: 'string' }, apiKey: { type: 'string' },
+                models: { type: 'array', items: modelEntry(true), description: 'Catalog entries served by this connector.' }
+            }
+        }
+    };
+    properties.models = {
+        type: 'array', 'x-change': 'editable (UI writes it back here)',
+        description: 'The model catalog on the default endpoint (models of a connector go under it). Empty everywhere: any model id. Listed: only these can be picked, by id, shown by label. Top-level settings are the defaults each entry may override.',
+        items: modelEntry(false)
+    };
+    properties.access = {
+        type: 'object', additionalProperties: false, 'x-change': 'file-only (admins write access.users back)',
+        properties: {
+            bootstrapAdmins: { type: 'array', items: { type: 'string' }, description: 'Gateway user ids that are always approved admins.' },
+            newUsers: { enum: ['approved', 'pending'] },
+            customize: { type: 'array', items: { enum: CUSTOMIZABLE }, uniqueItems: true, description: 'What users may personalise.' },
+            roles: { type: 'object', additionalProperties: false, properties: Object.fromEntries(ROLES.map((r) => [r, role])) },
+            users: {
+                type: 'object',
+                additionalProperties: {
+                    type: 'object', additionalProperties: false,
+                    properties: { role: { enum: ROLES }, status: { enum: STATUSES } }
+                }
+            }
+        }
+    };
+    return {
+        $schema: 'https://json-schema.org/draft/2020-12/schema',
+        title: 'TinyWebUI configuration',
+        type: 'object',
+        properties
+    };
 }
-
 export async function runCli(command, args, opts) {
-  const fn = commands[command];
-  if (!fn) return null;
-  return fn(args, opts);
+    const fn = commands[command];
+    if (!fn)
+        return null;
+    return fn(args, opts);
 }