pgkiln 0.66.0 → 0.68.0

package/src/builder/i18n.ts

@@ -19,20 +19,30 @@ import { de } from './i18n/de.ts';
 import { en } from './i18n/en.ts';
 import { es } from './i18n/es.ts';
 import { fr } from './i18n/fr.ts';
+import { it } from './i18n/it.ts';
+import { ja } from './i18n/ja.ts';
+import { ko } from './i18n/ko.ts';
 import { nl } from './i18n/nl.ts';
 import { pt } from './i18n/pt.ts';
+import { zhCN } from './i18n/zh-cn.ts';
+import { zhTW } from './i18n/zh-tw.ts';
 
 export type BuilderKey = keyof typeof en;
 export type BuilderMessages = Record<BuilderKey, string>;
 export type Params = Record<string, string | number | null | undefined>;
 
-/** The builder's languages, in the order of the language menu and the sign-in page's links (names in their own language). */
+/**
+ * The builder's languages, in the order of the language menu and the sign-in page's links (names in their own
+ * language): APEX's ten (sprint 134, SEC-5: Italiano, both Chinese scripts, Japanese, Korean) and Nederlands.
+ * The codes are BCP 47 tags, used as they are in <html lang>, the cookie and meta.developer.language.
+ */
 export const BUILDER_LANGUAGES = [
-  ['de', 'Deutsch'], ['en', 'English'], ['es', 'Español'], ['fr', 'Français'], ['nl', 'Nederlands'], ['pt', 'Português'],
+  ['de', 'Deutsch'], ['en', 'English'], ['es', 'Español'], ['fr', 'Français'], ['it', 'Italiano'], ['nl', 'Nederlands'], ['pt', 'Português'],
+  ['zh-CN', '中文 (简体)'], ['zh-TW', '中文 (繁體)'], ['ja', '日本語'], ['ko', '한국어'],
 ] as const;
 export type BuilderLang = (typeof BUILDER_LANGUAGES)[number][0];
 
-const MESSAGES: Record<BuilderLang, Readonly<Record<string, string>>> = { en, de, es, fr, nl, pt };
+const MESSAGES: Record<BuilderLang, Readonly<Record<string, string>>> = { en, de, es, fr, it, nl, pt, 'zh-CN': zhCN, 'zh-TW': zhTW, ja, ko };
 
 /** The cookie with the sign-in page's language on this device (the builder's path only). */
 export const LANG_COOKIE = 'pgkiln_blang';

package/src/builder/instance.ts

@@ -1,5 +1,7 @@
 import type { FastifyInstance, FastifyReply } from 'fastify';
 import { html } from '../html.ts';
+import { attention, confirmKeyBackup, forgetAttention, instanceChecks, type Level } from '../instance-checks.ts';
+import { relyingParty } from '../webauthn.ts';
 import { configurationValue, CONFIGURATION, INSTANCE_SETTINGS, instanceSetting, origin, refreshInstanceSettings, saveInstanceSettings } from '../instance.ts';
 import { clientIp, logActivity, type Session } from '../session.ts';
 import { isAdmin } from './locks.ts';
@@ -20,6 +22,20 @@ async function adminOnly(s: Session, reply: FastifyReply) {
   return false;
 }
 
+const LEVEL_TEXT: Record<Level, string> = { ok: 'OK', info: 'Note', warning: 'Warning', error: 'Error' };
+
+/** (sprint 133, SEC-6) Lockout checks: what could keep people out later (src/instance-checks.ts). */
+async function checksRegion(s: Session, rpId: string) {
+  const checks = await instanceChecks(rpId);
+  const n = attention(checks);
+  return region('Lockout checks', html`<p class="muted u-mt0">${n ? `${n} of these need attention.` : 'Nothing needs attention.'}
+      The same checks run on the server with <code>pgkiln admin check</code>; <code>--url</code> shows which passkeys a new address would break.</p>
+    <ul class="hits findings">${checks.map((c) => html`<li class="finding-${c.level}"><strong>${c.title}</strong><span></span>
+      <span class="tag tag-${c.level}">${LEVEL_TEXT[c.level]}</span><span class="hit-snippet">${c.detail}</span>
+      ${c.key === 'key_backup' && c.level === 'warning' ? html`<form method="post" action="${BASE}/instance/key-backup" class="u-mt0">${csrf(s)}
+        <button class="btn btn-sm">I have a copy of this key</button></form>` : ''}</li>`)}</ul>`, '', 'lockout-checks');
+}
+
 const ORIGIN_TEXT = { builder: 'set here', environment: 'environment variable', default: 'default' } as const;
 
 export async function instanceRoutes(app: FastifyInstance) {
@@ -39,6 +55,7 @@ export async function instanceRoutes(app: FastifyInstance) {
               <small class="help">${x.help} In effect: <b>${instanceSetting(x.key)}</b> (${ORIGIN_TEXT[from]}; <code>${x.env}</code>, default ${x.default}).</small></div>`;
           })}</div>
           <div class="buttons"><button class="btn btn-hot">Save</button></div></form>`)}
+        ${await checksRegion(s, relyingParty(req).rpId)}
         ${await releaseCheckRegion(s)}
         ${await policyRegion(s)}
         ${region('Configuration', html`<p class="muted u-mt0">Environment variables of this server (chapter 1 of the guide). Secrets show only whether they are set.</p>
@@ -51,6 +68,16 @@ export async function instanceRoutes(app: FastifyInstance) {
     return send(reply, s, shell(s, 'Instance settings', crumbs, main));
   });
 
+  app.post(`${BASE}/instance/key-backup`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    const ok = await confirmKeyBackup(s.username!);
+    forgetAttention();
+    if (ok) await logActivity({ username: s.username, event: 'secret_key', ip: clientIp(req), detail: 'backup confirmed' });
+    flash(s, ok ? 'Backup of the instance key confirmed.' : 'PGKILN_SECRET_KEY is not set.', ok ? 'ok' : 'error');
+    return back(reply, s, `${BASE}/instance#lockout-checks`);
+  });
+
   app.post(`${BASE}/instance`, async (req: Req, reply) => {
     const s = await developer(req, reply);
     if (!s || !(await adminOnly(s, reply))) return;

package/src/builder/workspace-home.ts

@@ -9,6 +9,7 @@ import { isAdmin } from './locks.ts';
 import { back, BASE, csrf, developer, flash, region, send, shell, type Req } from './ui.ts';
 import { currentWorkspace, inWorkspaceSql, workspaceAdmin } from './workspaces.ts';
 import { teamHomeBlock } from './team.ts';
+import { attentionCount } from '../instance-checks.ts';
 import { tr } from './i18n.ts';
 
 // (sprint 109, WS-1) The Workspace home (APEX: the workspace home page), the page the builder opens after
@@ -112,7 +113,12 @@ async function systemMessage(s: Session, admin: boolean) {
           ? html`<button class="btn btn-sm" name="check" value="now">${icon('refresh')} ${tx('wh.check_now')}</button><button class="btn btn-sm" name="check" value="off">${tx('wh.turn_off')}</button>`
           : html`<button class="btn btn-sm" name="check" value="on">${tx('wh.check_on')}</button>`}</div></form>`
     : '';
-  return region(tx('wh.system'), html`${body}${admins}`, '', 'system-message');
+  // (sprint 133, SEC-6) administrators hear about lockout risks here, not only on Instance settings
+  const lockout = admin ? await attentionCount() : 0;
+  const checks = lockout
+    ? html`<p class="alert alert-warning" role="status">${icon('alert')} ${tx('wh.lockout', { count: lockout })} <a href="${BASE}/instance#lockout-checks">${tx('wh.lockout_open')}</a></p>`
+    : '';
+  return region(tx('wh.system'), html`${checks}${body}${admins}`, '', 'system-message');
 }
 
 /** (Instance settings page) the release check: on, off, or as the environment says. */

package/src/cli/main.ts

@@ -471,24 +471,35 @@ const COMMANDS: Record<string, Command> = {
   },
   // (sprint 126, SEC-6) lockout recovery on the server, when nobody can sign in to the builder
   admin: {
-    usage: 'pgkiln admin reset-2fa <developer> | set-2fa-policy off|admins|all | builder-sign-in password [options]',
-    summary: 'lockout recovery: reset a developer\'s two-factor sign-in, set the two-factor policy, allow the password again',
+    usage: 'pgkiln admin reset-2fa <developer> | set-2fa-policy off|admins|all | builder-sign-in password | rotate-key | key-backed-up | check [options]',
+    summary: 'lockout recovery and checks: two-factor reset and policy, the builder password, the instance key',
     details:
       'pgkiln admin reset-2fa <developer> [--reason <text>]\n' +
       'pgkiln admin set-2fa-policy off|admins|all [--grace <days>]\n' +
-      'pgkiln admin builder-sign-in password\n\n' +
+      'pgkiln admin builder-sign-in password\n' +
+      'pgkiln admin rotate-key [--dry-run]\n' +
+      'pgkiln admin key-backed-up\n' +
+      'pgkiln admin check [--url <public url>]\n\n' +
       "reset-2fa removes the developer's authenticator apps and recovery codes and ends their builder\n" +
       'sessions; they sign in with the password and set up an app again. set-2fa-policy off always works\n' +
       "(and clears the workspaces' own two-factor rules); admins and all need at least two administrators\n" +
       'with two-factor sign-in (or every one, when there are fewer). builder-sign-in password turns off the\n' +
       "workspaces' single sign-on rule, for when the identity provider fails and no break-glass administrator\n" +
-      'can sign in. All are written to the activity log and mailed to the administrators.',
-    options: { reason: { type: 'string' }, grace: { type: 'string' } },
+      'can sign in. All are written to the activity log and mailed to the administrators.\n\n' +
+      'rotate-key re-encrypts every stored secret (authenticator apps, web credentials, mail and AI keys)\n' +
+      'from PGKILN_OLD_SECRET_KEY to PGKILN_SECRET_KEY in one transaction; stop the servers first and start\n' +
+      'them with the new key. key-backed-up records that the current key is kept outside the server.\n' +
+      'check prints the lockout checks of Instance settings (instance key, clock, passkey domain,\n' +
+      'break-glass administrators); --url shows which passkeys a move to that address would break.\n' +
+      'Exit code 1 when a check needs attention.',
+    options: { reason: { type: 'string' }, grace: { type: 'string' }, 'dry-run': { type: 'boolean' }, url: { type: 'string' } },
     optionHelp: [
       ['--reason <text>', 'reset-2fa: why (in the activity log and the mail)'],
       ['--grace <days>', 'set-2fa-policy: days developers have to set up an app (0 to 90, default 7)'],
+      ['--dry-run', 'rotate-key: count what would be re-encrypted, change nothing'],
+      ['--url <url>', 'check: the public address to check passkeys against (default PUBLIC_URL)'],
     ],
-    positionals: [2, 2],
+    positionals: [1, 2],
     async run(v, [action, arg]) {
       process.env.RUNTIME_DATABASE_URL ??= process.env.DATABASE_URL; // the CLI runs as the owner
       const mfa = await import('../mfa.ts');
@@ -497,6 +508,7 @@ const COMMANDS: Record<string, Command> = {
       const by = `pgkiln admin (${userInfo().username})`;
       try {
         if (action === 'reset-2fa') {
+          if (!arg) throw new UsageError('reset-2fa takes a developer');
           const reason = String(v.reason ?? 'command line').slice(0, 200);
           if (!(await mfa.resetFactors(arg))) throw new Error(`no developer ${arg}`);
           await logActivity({ username: arg, event: '2fa_reset', ip: 'cli', detail: `${arg}: ${reason} (${by})` });
@@ -530,7 +542,46 @@ const COMMANDS: Record<string, Command> = {
           out(`Developers may sign in with the pgkiln password again; ${cleared} workspace(s) no longer require single sign-on.${sent ? ` ${sent} administrator(s) told by mail.` : ''}\n`);
           return EXIT.ok;
         }
-        throw new UsageError(`unknown action ${action}: use reset-2fa, set-2fa-policy or builder-sign-in`);
+        // (sprint 133, SEC-6) the instance key and the lockout checks
+        if (action === 'rotate-key' || action === 'key-backed-up' || action === 'check') {
+          if (arg !== undefined) throw new UsageError(`${action} takes no argument`);
+          const checks = await import('../instance-checks.ts');
+          if (action === 'rotate-key') {
+            const oldKey = process.env.PGKILN_OLD_SECRET_KEY ?? '';
+            const newKey = process.env.PGKILN_SECRET_KEY ?? '';
+            if (!oldKey) throw new UsageError('set PGKILN_OLD_SECRET_KEY to the key the secrets are stored with, and PGKILN_SECRET_KEY to the new one');
+            const dry = !!v['dry-run'];
+            const counts = await checks.rotateKey(oldKey, newKey, { dryRun: dry });
+            const rotated = counts.reduce((n, c) => n + c.rotated, 0);
+            for (const c of counts) out(`  ${c.schema}.${c.table}.${c.column}: ${c.rotated} ${dry ? 'to re-encrypt' : 're-encrypted'}${c.already ? `, ${c.already} already with the new key` : ''}\n`);
+            if (dry) {
+              out(`Dry run: ${rotated} secret(s) would be re-encrypted; nothing changed.\n`);
+              return EXIT.ok;
+            }
+            await logActivity({ event: 'secret_key', ip: 'cli', detail: `rotated, ${rotated} secret(s) re-encrypted (${by})` });
+            const sent = await mfa.notifyMfa(await mfa.adminEmails(), 'pgkiln: the instance key was rotated',
+              `${by} re-encrypted ${rotated} stored secret(s) with a new PGKILN_SECRET_KEY on the server. Keep a copy of the new key and confirm it on Instance settings.`);
+            out(`${rotated} secret(s) re-encrypted with the new key. Start the servers with it, keep a copy, then run 'pgkiln admin key-backed-up'.${sent ? ` ${sent} administrator(s) told by mail.` : ''}\n`);
+            return EXIT.ok;
+          }
+          if (action === 'key-backed-up') {
+            if (!(await checks.confirmKeyBackup(by))) throw new Error('PGKILN_SECRET_KEY is not set (at least 32 characters)');
+            await logActivity({ event: 'secret_key', ip: 'cli', detail: `backup confirmed (${by})` });
+            out('Backup of the instance key confirmed.\n');
+            return EXIT.ok;
+          }
+          let rpId: string | null = null;
+          const url = (v.url as string | undefined) ?? process.env.PUBLIC_URL;
+          if (url) {
+            if (!URL.canParse(url) || !/^https?:$/.test(new URL(url).protocol)) throw new UsageError('--url takes an http(s) address');
+            rpId = (await import('../webauthn.ts')).rpIdOf(new URL(url).origin);
+          }
+          const list = await checks.instanceChecks(rpId);
+          for (const c of list) out(`${c.level.toUpperCase().padEnd(7)}  ${c.title}\n         ${c.detail}\n`);
+          if (!rpId) out('(passkey domain not checked: set PUBLIC_URL or pass --url)\n');
+          return checks.attention(list) ? EXIT.differences : EXIT.ok;
+        }
+        throw new UsageError(`unknown action ${action}: use reset-2fa, set-2fa-policy, builder-sign-in, rotate-key, key-backed-up or check`);
       } finally {
         await closePools();
       }

package/src/instance-checks.ts

@@ -0,0 +1,260 @@
+import { owner } from './db.ts';
+import { breakGlassAdmins } from './devsso.ts';
+import { mfaPolicy } from './mfa.ts';
+import { remoteClockOffset } from './release-check.ts';
+import { rpIdOf } from './webauthn.ts';
+import { decryptSecret, encryptSecret, keyFingerprint, secretKeyConfigured } from './secrets.ts';
+
+// (sprint 133, SEC-6) Lockout checks of the instance: what could keep developers or application users out
+// later, found before it happens. Shown on Instance settings (administrators), counted on the Workspace home
+// and printed by `pgkiln admin check`.
+//
+//  - Instance key: secrets stored without PGKILN_SECRET_KEY, secrets the current key can't decrypt (the key
+//    changed), and whether an administrator confirmed a backup of this key (meta.setting secret_key_backup,
+//    by fingerprint, so a new key asks again). Authenticator apps are encrypted with it; passkeys and the
+//    hashed recovery codes are not.
+//  - Clock: TOTP codes allow one 30-second step either side, so the server's clock is compared with the
+//    database's and, when the release check is on, with GitHub's.
+//  - Passkey domain: passkeys made for another RP ID than the one PUBLIC_URL (or the given address) gives,
+//    and how many people have no other factor left. `pgkiln admin check --url` asks before a move.
+//  - Break-glass: a workspace that requires single sign-on needs a break-glass administrator, and one with
+//    a second factor when the policy requires it.
+//
+// rotateKey() re-encrypts every stored secret from an old key to a new one in one transaction.
+
+export type Level = 'ok' | 'info' | 'warning' | 'error';
+
+export interface Check {
+  key: 'secret_key' | 'key_backup' | 'clock' | 'passkey_domain' | 'break_glass';
+  level: Level;
+  title: string;
+  detail: string;
+}
+
+/** Every column that holds a secret encrypted with the instance key ("v1:…"). */
+export interface SecretColumn {
+  schema: string;
+  table: string;
+  column: string;
+}
+
+export async function encryptedColumns(): Promise<SecretColumn[]> {
+  return (await owner.query<SecretColumn>(
+    `select table_schema as schema, table_name as table, column_name as column from information_schema.columns
+      where table_schema = 'meta' and data_type = 'text'
+        and (column_name like '%\\_enc' or (table_name = 'push_key' and column_name = 'private_key'))
+      order by 2, 3`)).rows;
+}
+
+const quoteIdent = (s: string) => `"${s.replace(/"/g, '""')}"`;
+const tableOf = (c: SecretColumn) => `${quoteIdent(c.schema)}.${quoteIdent(c.table)}`;
+
+/** Stored secrets per column, and how many the given key (default: the server's) can't decrypt. */
+export async function secretStatus(withKey?: string) {
+  const out: (SecretColumn & { stored: number; unreadable: number })[] = [];
+  const usable = (withKey ?? process.env.PGKILN_SECRET_KEY ?? '').length >= 32;
+  for (const c of await encryptedColumns()) {
+    const rows = (await owner.query<{ v: string }>(`select ${quoteIdent(c.column)} as v from ${tableOf(c)} where ${quoteIdent(c.column)} is not null`)).rows;
+    let unreadable = 0;
+    for (const r of rows) {
+      try {
+        if (!usable) throw new Error();
+        decryptSecret(r.v, withKey);
+      } catch {
+        unreadable++;
+      }
+    }
+    if (rows.length) out.push({ ...c, stored: rows.length, unreadable });
+  }
+  return out;
+}
+
+export interface KeyBackup {
+  fingerprint: string;
+  by: string;
+  at: string;
+}
+
+export async function keyBackup(): Promise<KeyBackup | null> {
+  const v = (await owner.one<{ value: string }>(`select value from meta.setting where name = 'secret_key_backup'`))?.value;
+  try {
+    const b = v ? JSON.parse(v) : null;
+    return b && typeof b.fingerprint === 'string' ? (b as KeyBackup) : null;
+  } catch {
+    return null;
+  }
+}
+
+/** An administrator says the current key is kept somewhere safe (false: no key configured). */
+export async function confirmKeyBackup(by: string) {
+  const fingerprint = keyFingerprint();
+  if (!fingerprint) return false;
+  const value = JSON.stringify({ fingerprint, by, at: new Date().toISOString() });
+  await owner.query(`insert into meta.setting (name, value) values ('secret_key_backup', $1) on conflict (name) do update set value = excluded.value`, [value]);
+  return true;
+}
+
+/** Milliseconds this server's clock is ahead of the database's (half the round trip taken off). */
+export async function databaseClockOffset() {
+  const t0 = Date.now();
+  const r = await owner.one<{ ms: string }>(`select (extract(epoch from clock_timestamp()) * 1000)::bigint as ms`);
+  const t1 = Date.now();
+  return Math.round((t0 + t1) / 2 - Number(r!.ms));
+}
+
+export const CLOCK_WARN_MS = 10_000;
+export const CLOCK_ERROR_MS = 30_000;
+
+export interface PasskeyDomain {
+  rpId: string;
+  who: 'developers' | 'app users';
+  passkeys: number;
+  people: number;
+  /** people with no authenticator app and no passkey for the current RP ID */
+  stranded: number;
+}
+
+/** Passkeys made for another RP ID than rpId. */
+export async function passkeysElsewhere(rpId: string): Promise<PasskeyDomain[]> {
+  const part = (table: string, key: string, who: string) => `
+    select '${who}'::text as who, coalesce(f.rp_id, '(unknown)') as "rpId", count(*)::int as passkeys, count(distinct f.${key})::int as people,
+           count(distinct f.${key}) filter (where not exists (select 1 from meta.${table} o where o.${key} = f.${key}
+             and (o.kind = 'totp' or (o.kind = 'passkey' and o.rp_id = $1))))::int as stranded
+      from meta.${table} f where f.kind = 'passkey' and f.rp_id is distinct from $1 group by f.rp_id`;
+  return (await owner.query<PasskeyDomain>(`${part('developer_factor', 'username', 'developers')} union all ${part('account_factor', 'account_id', 'app users')} order by 2, 1`, [rpId])).rows;
+}
+
+const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
+
+/** All checks; rpId: the RP ID passkeys must have (from PUBLIC_URL or the request; null skips that check). */
+export async function instanceChecks(rpId: string | null): Promise<Check[]> {
+  const checks: Check[] = [];
+
+  // the instance key
+  const status = await secretStatus();
+  const stored = status.reduce((n, s) => n + s.stored, 0);
+  const unreadable = status.filter((s) => s.unreadable);
+  if (!secretKeyConfigured())
+    checks.push(stored
+      ? { key: 'secret_key', level: 'error', title: 'Instance key missing', detail: `${plural(stored, 'secret')} are stored (${status.map((s) => `${s.table}.${s.column}`).join(', ')}), but PGKILN_SECRET_KEY is not set: authenticator apps, web credentials and mail passwords don't work. Set the key they were stored with.` }
+      : { key: 'secret_key', level: 'info', title: 'No instance key', detail: 'PGKILN_SECRET_KEY is not set, so nothing can be stored encrypted (authenticator apps, web credentials, mail and AI keys). Set one of at least 32 characters, e.g. the output of openssl rand -base64 32.' });
+  else if (unreadable.length)
+    checks.push({ key: 'secret_key', level: 'error', title: 'Secrets the instance key can\'t decrypt', detail: `${unreadable.map((s) => `${s.table}.${s.column}: ${s.unreadable} of ${s.stored}`).join('; ')}. PGKILN_SECRET_KEY changed without pgkiln admin rotate-key: set the old key again and rotate, or enter those secrets again (developers set up their authenticator app again after a reset).` });
+  else checks.push({ key: 'secret_key', level: 'ok', title: 'Instance key', detail: stored ? `PGKILN_SECRET_KEY decrypts all ${plural(stored, 'stored secret')}.` : 'PGKILN_SECRET_KEY is set; no secrets are stored yet.' });
+
+  if (secretKeyConfigured()) {
+    const backup = await keyBackup();
+    const fp = keyFingerprint()!;
+    checks.push(backup?.fingerprint === fp
+      ? { key: 'key_backup', level: 'ok', title: 'Instance key backed up', detail: `${backup.by} confirmed a copy of the key with fingerprint ${fp} on ${backup.at.slice(0, 10)}.` }
+      : { key: 'key_backup', level: 'warning', title: 'Instance key not confirmed as backed up', detail: `${backup ? 'The key changed since the last confirmation. ' : ''}Keep a copy of PGKILN_SECRET_KEY (fingerprint ${fp}) outside this server, e.g. in a password manager: without it the authenticator apps and stored credentials are lost. Passkeys and recovery codes keep working.` });
+  }
+
+  // the clock
+  const db = await databaseClockOffset();
+  const remote = remoteClockOffset();
+  const worst = Math.max(Math.abs(db), Math.abs(remote?.offsetMs ?? 0));
+  const s = (ms: number) => (Math.abs(ms) < 50 ? '0.0 s' : `${ms > 0 ? '+' : '−'}${(Math.abs(ms) / 1000).toFixed(1)} s`);
+  const seen = `database (${s(db)})${remote ? ` and GitHub's (${s(remote.offsetMs)}, at the release check)` : ''}`;
+  checks.push(worst >= CLOCK_ERROR_MS
+    ? { key: 'clock', level: 'error', title: 'Server clock is off', detail: `This server's clock differs from the ${seen}. Authenticator codes fail beyond about 30 seconds: synchronise the clock (NTP, e.g. systemd-timesyncd or chrony).` }
+    : worst >= CLOCK_WARN_MS
+      ? { key: 'clock', level: 'warning', title: 'Server clock drifts', detail: `This server's clock differs from the ${seen}. Authenticator codes fail beyond about 30 seconds: check the time synchronisation (NTP).` }
+      : { key: 'clock', level: 'ok', title: 'Clock', detail: `This server's clock agrees with the ${seen}.` });
+
+  // passkeys of another domain
+  if (rpId) {
+    const away = await passkeysElsewhere(rpId);
+    const strandedTotal = away.reduce((n, a) => n + a.stranded, 0);
+    checks.push(away.length
+      ? { key: 'passkey_domain', level: strandedTotal ? 'warning' : 'info', title: `Passkeys for another domain than ${rpId}`,
+          detail: `${away.map((a) => `${a.rpId}: ${plural(a.passkeys, 'passkey')} of ${plural(a.people, a.who === 'developers' ? 'developer' : 'app user')}${a.stranded ? `, ${a.stranded} with no other factor` : ''}`).join('; ')}. Passkeys work only on the domain they were made for${strandedTotal ? ': people with no other factor need a recovery code or a reset' : ''}. Set PUBLIC_URL (or PGKILN_RP_ID) back, or have them add passkeys again.` }
+      : { key: 'passkey_domain', level: 'ok', title: 'Passkey domain', detail: `Every passkey is for ${rpId}.` });
+  }
+
+  // break-glass administrators
+  const ssoOnly = (await owner.query<{ name: string }>(`select name from meta.workspace where sso_only order by name`)).rows.map((r) => r.name);
+  if (ssoOnly.length) {
+    const glass = await breakGlassAdmins();
+    const policy = (await mfaPolicy()).policy;
+    const withFactor = glass.length
+      ? (await owner.query<{ username: string }>(`select distinct username from meta.developer_factor where username = any($1)`, [glass])).rows.map((r) => r.username)
+      : [];
+    checks.push(!glass.length
+      ? { key: 'break_glass', level: 'error', title: 'No break-glass administrator', detail: `${ssoOnly.join(', ')} require${ssoOnly.length === 1 ? 's' : ''} single sign-on and no administrator may use the pgkiln password: when the identity provider fails, only pgkiln admin builder-sign-in password on the server gets anyone in. Mark an administrator as break-glass on Developers.` }
+      : policy !== 'off' && withFactor.length < glass.length
+        ? { key: 'break_glass', level: 'warning', title: 'Break-glass administrator without a second factor', detail: `${glass.filter((g) => !withFactor.includes(g)).join(', ')} must use two-factor sign-in but has no authenticator app or passkey: add one on Security before it is needed.` }
+        : { key: 'break_glass', level: 'ok', title: 'Break-glass administrators', detail: `${glass.join(', ')} may sign in with the pgkiln password when the identity provider fails (${ssoOnly.join(', ')} require${ssoOnly.length === 1 ? 's' : ''} single sign-on).` });
+  }
+  return checks;
+}
+
+/** How many checks need attention (warnings and errors). */
+export const attention = (checks: Check[]) => checks.filter((c) => c.level === 'warning' || c.level === 'error').length;
+
+let cached: { at: number; count: number } | null = null;
+
+/** (Workspace home) how many checks need attention, at most five minutes old; never throws. */
+export async function attentionCount(): Promise<number> {
+  if (cached && Date.now() - cached.at < 300_000) return cached.count;
+  try {
+    const pub = process.env.PUBLIC_URL;
+    const rpId = pub && URL.canParse(pub) ? rpIdOf(new URL(pub).origin) : null;
+    cached = { at: Date.now(), count: attention(await instanceChecks(rpId)) };
+  } catch {
+    return cached?.count ?? 0;
+  }
+  return cached.count;
+}
+
+/** Forget the cached count (after a change on Instance settings, and in tests). */
+export const forgetAttention = () => {
+  cached = null;
+};
+
+export class RotateError extends Error {}
+
+/**
+ * Re-encrypt every stored secret from oldKey to newKey, in one transaction. A value the new key already
+ * decrypts is left alone (a second run after a partial restart changes nothing); a value neither key
+ * decrypts stops the rotation and nothing is changed. dryRun: count only.
+ */
+export async function rotateKey(oldKey: string, newKey: string, { dryRun = false, columns }: { dryRun?: boolean; columns?: SecretColumn[] } = {}) {
+  if (oldKey.length < 32 || newKey.length < 32) throw new RotateError('both keys need at least 32 characters');
+  if (oldKey === newKey) throw new RotateError('the new key is the old key');
+  const cols = columns ?? (await encryptedColumns());
+  return owner.tx(async (c) => {
+    const counts: (SecretColumn & { rotated: number; already: number })[] = [];
+    const failed: string[] = [];
+    for (const col of cols) {
+      const rows = (await c.query<{ ctid: string; v: string }>(
+        `select ctid::text as ctid, ${quoteIdent(col.column)} as v from ${tableOf(col)} where ${quoteIdent(col.column)} is not null for update`)).rows;
+      let rotated = 0;
+      let already = 0;
+      for (const r of rows) {
+        let plain: string;
+        try {
+          decryptSecret(r.v, newKey);
+          already++;
+          continue;
+        } catch {
+          // not yet rotated
+        }
+        try {
+          plain = decryptSecret(r.v, oldKey);
+        } catch {
+          failed.push(`${col.schema}.${col.table}.${col.column}`);
+          continue;
+        }
+        if (!dryRun) await c.query(`update ${tableOf(col)} set ${quoteIdent(col.column)} = $1 where ctid = $2::tid`, [encryptSecret(plain, newKey), r.ctid]);
+        rotated++;
+      }
+      if (rows.length) counts.push({ ...col, rotated, already });
+    }
+    if (failed.length) {
+      const by = [...new Set(failed)].map((f) => `${f} (${failed.filter((x) => x === f).length})`).join(', ');
+      throw new RotateError(`secrets that neither key decrypts: ${by}; nothing was changed`);
+    }
+    return counts;
+  });
+}

package/src/instance.ts

@@ -120,7 +120,7 @@ export const CONFIGURATION: readonly { env: string; secret?: boolean; default: s
   { env: 'BACKGROUND_PROCESSES', default: 'on', help: 'This server runs background execution chains' },
   { env: 'PGKILN_REST_ALLOWED_HOSTS', default: '(none)', help: 'Hosts REST data sources may call' },
   { env: 'PGKILN_REST_PRIVATE_HOSTS', default: '(none)', help: 'Hosts that may resolve to private addresses' },
-  { env: 'PGKILN_SECRET_KEY', secret: true, default: '(none)', help: 'Encrypts web credential secrets and AI keys' },
+  { env: 'PGKILN_SECRET_KEY', secret: true, default: '(none)', help: 'Encrypts authenticator apps, web credential secrets, mail and AI keys (pgkiln admin rotate-key changes it)' },
   { env: 'API_URL', default: '(none)', help: 'PostgREST address (REST API pages)' },
   { env: 'API_JWT_SECRET', secret: true, default: '(none)', help: 'Signs REST API tokens' },
   { env: 'ANTHROPIC_API_KEY', secret: true, default: '(none)', help: 'Default key of Claude AI services' },

package/src/release-check.ts

@@ -29,6 +29,12 @@ export interface Release {
 
 let cache: { at: number; release: Release | null; error: string | null } | null = null;
 let inflight: Promise<void> | null = null;
+let remoteClock: { at: number; offsetMs: number } | null = null;
+
+/** How far this server's clock was ahead of GitHub's at the last release check (null: not asked yet). */
+export function remoteClockOffset() {
+  return remoteClock;
+}
 
 /** The address asked (PGKILN_RELEASE_URL for tests and mirrors; http(s) only). */
 function releaseUrl() {
@@ -91,6 +97,9 @@ async function fetchLatest(): Promise<Release | null> {
     redirect: 'error',
     signal: AbortSignal.timeout(TIMEOUT_MS),
   });
+  // (sprint 133, SEC-6) GitHub's clock, for the clock check on Instance settings (the header has whole seconds)
+  const date = Date.parse(res.headers.get('date') ?? '');
+  if (!Number.isNaN(date)) remoteClock = { at: Date.now(), offsetMs: Date.now() - date };
   if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
   const reader = res.body.getReader();
   const chunks: Uint8Array[] = [];
@@ -136,4 +145,5 @@ export function cachedRelease() {
 /** Forget the cache (tests). */
 export function resetReleaseCache() {
   cache = null;
+  remoteClock = null;
 }

package/src/secrets.ts

@@ -14,26 +14,36 @@ export class SecretKeyMissing extends Error {
   }
 }
 
-function key() {
-  const k = process.env.PGKILN_SECRET_KEY ?? '';
+function key(k = process.env.PGKILN_SECRET_KEY ?? '') {
   if (k.length < 32) throw new SecretKeyMissing();
   return createHash('sha256').update(`pgkiln-secret-v1\0${k}`).digest();
 }
 
 export const secretKeyConfigured = () => (process.env.PGKILN_SECRET_KEY ?? '').length >= 32;
 
-export function encryptSecret(plain: string): string {
+/**
+ * (sprint 133, SEC-6) A short fingerprint of a key, to tell which key a backup confirmation was for. A
+ * different derivation than the encryption key's, so it reveals nothing about that key.
+ */
+export function keyFingerprint(k = process.env.PGKILN_SECRET_KEY ?? ''): string | null {
+  if (k.length < 32) return null;
+  return createHash('sha256').update(`pgkiln-key-fingerprint\0${k}`).digest('hex').slice(0, 16).replace(/(.{4})(?!$)/g, '$1-');
+}
+
+/** Encrypt with the server's key, or (rotate-key) with the key given. */
+export function encryptSecret(plain: string, withKey?: string): string {
   const iv = randomBytes(12);
-  const c = createCipheriv('aes-256-gcm', key(), iv);
+  const c = createCipheriv('aes-256-gcm', key(withKey), iv);
   const ct = Buffer.concat([c.update(plain, 'utf8'), c.final()]);
   return VERSION + Buffer.concat([iv, c.getAuthTag(), ct]).toString('base64');
 }
 
-export function decryptSecret(stored: string): string {
+/** Decrypt with the server's key, or (rotate-key, checks) with the key given. */
+export function decryptSecret(stored: string, withKey?: string): string {
   if (!stored.startsWith(VERSION)) throw new Error('Unknown secret format.');
   const buf = Buffer.from(stored.slice(VERSION.length), 'base64');
   if (buf.length < 28) throw new Error('Damaged secret.');
-  const d = createDecipheriv('aes-256-gcm', key(), buf.subarray(0, 12));
+  const d = createDecipheriv('aes-256-gcm', key(withKey), buf.subarray(0, 12));
   d.setAuthTag(buf.subarray(12, 28));
   try {
     return Buffer.concat([d.update(buf.subarray(28)), d.final()]).toString('utf8');

package/src/server.ts

@@ -14,3 +14,13 @@ await startWorkflowRunner();
 await startProcessJobRunner();
 await startPushListener();
 await startMailListener();
+// (sprint 133, SEC-6) say at start-up what could lock people out: a changed instance key or address, the clock
+try {
+  const { instanceChecks } = await import('./instance-checks.ts');
+  const { rpIdOf } = await import('./webauthn.ts');
+  const pub = process.env.PUBLIC_URL;
+  for (const c of await instanceChecks(pub && URL.canParse(pub) ? rpIdOf(new URL(pub).origin) : null))
+    if (c.level === 'error' || (c.level === 'warning' && c.key !== 'key_backup')) console.warn(`pgkiln: ${c.title}: ${c.detail}`);
+} catch (e) {
+  console.warn('pgkiln: lockout checks failed:', (e as Error).message);
+}

package/src/webauthn.ts

@@ -218,8 +218,12 @@ export function relyingParty(req: { protocol: string; host: string }): { rpId: s
       // a bad PUBLIC_URL: the browser's address
     }
   }
+  return { rpId: rpIdOf(origin), origin };
+}
+
+/** The RP ID of an origin: its host, or PGKILN_RP_ID when that is the host or a parent domain of it. */
+export function rpIdOf(origin: string): string {
   const host = new URL(origin).hostname;
   const wanted = (process.env.PGKILN_RP_ID ?? '').trim().toLowerCase();
-  const rpId = wanted && (host === wanted || host.endsWith(`.${wanted}`)) ? wanted : host;
-  return { rpId, origin };
+  return wanted && (host === wanted || host.endsWith(`.${wanted}`)) ? wanted : host;
 }