slotlock 0.1.1 → 0.3.0

package/self-host.js

@@ -7,14 +7,17 @@ import { createHash, timingSafeEqual } from 'node:crypto';
 import { parseArgs } from 'node:util';
 import postgres from 'postgres';
 import { createSlotlockAgentServer, slotlockAgentOperationScope, } from './agent-server.js';
-import { createSlotlockStoreAgentBackend } from './agent-store-backend.js';
-import { SLOTLOCK_API_KEY_MAX_LIFETIME_DAYS, SLOTLOCK_API_KEY_SCOPES, createSlotlockApiKeyAuthenticator, createSlotlockApiKeyStore, slotlockBearerCredential, } from './api-keys.js';
+import { createSlotlockStoreAgentBackend, createSlotlockStoreApplicationBackend, } from './agent-store-backend.js';
+import { SLOTLOCK_API_KEY_DEFAULT_SCOPES, SLOTLOCK_API_KEY_MAX_LIFETIME_DAYS, SLOTLOCK_API_KEY_SCOPES, createSlotlockApiKeyAuthenticator, createSlotlockApiKeyStore, slotlockBearerCredential, } from './api-keys.js';
 import { createSlotlockDashboard, createSlotlockDashboardResources, createSlotlockDashboardState, } from './dashboard.js';
-import { SECRET_HINT, SlotlockConfigError, optional, readAvailability, readDashboardConfig, readSecret, readServerUrl, } from './config.js';
-import { SLOTLOCK_DASHBOARD_FUNCTIONS } from './ddl.js';
+import { SECRET_HINT, SlotlockConfigError, formatSlotlockRate, optional, readAvailability, readDashboardConfig, readLimits, readSecret, readServerUrl, readTrustedProxyHeader, slotlockKeySubscriptions, } from './config.js';
+import { SLOTLOCK_APPLICATION_SCOPE } from './application-api.js';
+import { SLOTLOCK_DASHBOARD_FUNCTIONS, SLOTLOCK_TENANT_TABLES } from './ddl.js';
+import { maintainSlotlockTenant, startSlotlockMaintenance, withSlotlockMaintenanceLock, } from './maintenance.js';
 import { createSlotlockNodeServer, } from './node-server.js';
 import { slotlockPackageVersion } from './package-version.js';
-import { createSlotlockStore } from './store.js';
+import { createSlotlockRequestLimits } from './rate-limit.js';
+import { assertSlotlockSchemaCurrent, createSlotlockStore, } from './store.js';
 import { parseAvailabilityRules } from './weekly-hours.js';
 export { SlotlockConfigError };
 /** Every variable the commands read. The README, `.env.example` and `server.json` document these. */
@@ -28,6 +31,8 @@ export const SLOTLOCK_ENVIRONMENT_VARIABLES = Object.freeze([
     'SLOTLOCK_TENANT',
     'SLOTLOCK_AVAILABILITY',
     'SLOTLOCK_REST_API',
+    'SLOTLOCK_LIMITS',
+    'SLOTLOCK_TRUSTED_PROXY_HEADER',
     'SLOTLOCK_GITHUB_CLIENT_ID',
     'SLOTLOCK_GITHUB_CLIENT_SECRET',
     'SLOTLOCK_SESSION_SECRET',
@@ -38,11 +43,20 @@ export const SLOTLOCK_ENVIRONMENT_VARIABLES = Object.freeze([
 const DEFAULT_HOST = '127.0.0.1';
 const DEFAULT_PORT = 8080;
 const DEFAULT_TENANT = 'default';
-/** The principal the server token resolves to: it acts in SLOTLOCK_TENANT with every scope. */
+/** The principal the server token resolves to: it acts in SLOTLOCK_TENANT with read and write. */
 const SELF_HOST_SUBJECT = 'self-host';
 /** Who `key create` records as a key's creator. */
 const CLI_KEY_CREATOR = 'cli';
 const DAY_MS = 86_400_000;
+/**
+ * The most resources `serve` lets one tenant create through the application API. A dashboard open to
+ * every GitHub user gives each a tenant, so the cap keeps one from filling the database.
+ */
+const SERVE_MAX_TENANT_RESOURCES = 10_000;
+/** How often `serve` maintains each tenant it serves (README, Operations: at least daily). */
+const SERVE_MAINTENANCE_INTERVAL_MS = 60 * 60 * 1_000;
+/** Passes `slotlock maintain` runs at most, each as big as `serve`'s, until nothing is left. */
+const MAINTAIN_MAX_PASSES = 100;
 const WRITE_OPERATIONS = [
     'slotlock_create_event',
     'slotlock_update_event',
@@ -63,8 +77,13 @@ const SERVED_OPERATIONS = new Set([
     'slotlock_delete_event',
 ]);
 const HEALTH_QUERY_TIMEOUT_MS = 2_000;
+/** How long one database check answers /healthz, so a flood of health checks is not one of queries. */
+const HEALTH_RESULT_MS = 1_000;
 const HEALTHCHECK_TIMEOUT_MS = 5_000;
+/** One pool for every tenant; SLOTLOCK_LIMITS `tenant_transactions` (config.ts) goes up to it. */
 const POOL_SIZE = 10;
+/** How long a request's transaction waits for any one lock before it answers 503 `server_busy`. */
+const SERVE_LOCK_TIMEOUT_MS = 5_000;
 const RESOURCE_PAGE_SIZE = 1_000;
 function readDatabaseUrl(env, name) {
     const value = optional(env, name);
@@ -110,7 +129,7 @@ function readTenant(env) {
     return value;
 }
 function readConfirmWrites(env) {
-    const value = optional(env, 'SLOTLOCK_CONFIRM_WRITES') ?? 'all';
+    const value = optional(env, 'SLOTLOCK_CONFIRM_WRITES') ?? 'none';
     if (value === 'all')
         return [...WRITE_OPERATIONS];
     if (value === 'none')
@@ -138,7 +157,7 @@ export function readSlotlockDatabaseConfig(env) {
     return config;
 }
 function readRestApi(env) {
-    const value = optional(env, 'SLOTLOCK_REST_API') ?? 'off';
+    const value = optional(env, 'SLOTLOCK_REST_API') ?? 'on';
     if (value !== 'on' && value !== 'off') {
         throw new SlotlockConfigError('SLOTLOCK_REST_API must be on or off');
     }
@@ -150,7 +169,7 @@ export function readSlotlockServeConfig(env) {
     const confirmWrites = readConfirmWrites(env);
     const confirmationSecret = readSecret(env, 'SLOTLOCK_CONFIRMATION_SECRET');
     if (confirmWrites.length > 0 && confirmationSecret === undefined) {
-        throw new SlotlockConfigError(`SLOTLOCK_CONFIRMATION_SECRET is required while SLOTLOCK_CONFIRM_WRITES guards writes (the default): it signs each pending confirmation; ${SECRET_HINT}, or set SLOTLOCK_CONFIRM_WRITES=none to let agents write without asking a person`);
+        throw new SlotlockConfigError(`SLOTLOCK_CONFIRMATION_SECRET is required while SLOTLOCK_CONFIRM_WRITES guards writes: it signs each pending confirmation; ${SECRET_HINT}, or set SLOTLOCK_CONFIRM_WRITES=none (the default) to let agents write without asking a person`);
     }
     if (confirmationSecret !== undefined &&
         authToken !== undefined &&
@@ -166,7 +185,11 @@ export function readSlotlockServeConfig(env) {
         confirmWrites,
         availability: readAvailability(env),
         restApi: readRestApi(env),
+        limits: readLimits(env),
     };
+    const trustedProxyHeader = readTrustedProxyHeader(env);
+    if (trustedProxyHeader !== undefined)
+        config.trustedProxyHeader = trustedProxyHeader;
     if (authToken !== undefined)
         config.authToken = authToken;
     const dashboard = readDashboardConfig(env);
@@ -208,8 +231,8 @@ export function createSlotlockTokenAuthenticator(token, principal) {
 }
 /**
  * What `serve` lets a principal call: a served operation its scopes cover. The server token's
- * principal carries every scope; a principal without scopes is refused everything here, though the
- * agent server alone would leave it to this function.
+ * principal carries read and write, never admin; a principal without scopes is refused everything
+ * here, though the agent server alone would leave it to this function.
  */
 export function authorizeSlotlockServeOperation(args) {
     return (SERVED_OPERATIONS.has(args.operation) &&
@@ -245,7 +268,10 @@ function secretsIn(env) {
     // Longest first, so a URL is replaced whole rather than around its password.
     return [...new Set(secrets)].sort((a, b) => b.length - a.length);
 }
-/** One JSON object per line: info on stdout, errors on stderr, secrets replaced by `[redacted]`. */
+/**
+ * One JSON object per line: info on stdout, warnings and errors on stderr, secrets replaced by
+ * `[redacted]`.
+ */
 function createLogger(io) {
     const secrets = secretsIn(io.env);
     const write = (stream, level, event, fields = {}) => {
@@ -256,6 +282,7 @@ function createLogger(io) {
     };
     return {
         info: (event, fields) => write(io.stdout, 'info', event, fields),
+        warn: (event, fields) => write(io.stderr, 'warn', event, fields),
         error: (event, fields) => write(io.stderr, 'error', event, fields),
     };
 }
@@ -272,6 +299,13 @@ function describeError(error) {
     }
     return detail;
 }
+/** An agent server event's fields under the log's snake_case keys (`durationMs` → `duration_ms`). */
+function snakeCaseKeys(fields) {
+    return Object.fromEntries(Object.entries(fields).map(([key, value]) => [
+        key.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`),
+        value,
+    ]));
+}
 function connect(url, max) {
     return postgres(url, { max, onnotice: () => { }, connect_timeout: 10 });
 }
@@ -340,6 +374,26 @@ function dashboardAllows(config, tenantRef) {
     const githubId = /^github:(\d+)$/.exec(tenantRef)?.[1];
     return githubId === undefined || users.includes(githubId);
 }
+function serveTenantLimits(caps) {
+    return {
+        resources: SERVE_MAX_TENANT_RESOURCES,
+        holds: caps.tenant_holds,
+        reservations: caps.tenant_reservations,
+        events: caps.tenant_events,
+        coverage: caps.tenant_coverage,
+    };
+}
+/**
+ * The store `serve` answers requests from. Every surface writes through it, so its tenant limits
+ * bound what any key can add, and its transaction limits what one tenant holds of the shared pool.
+ */
+export function createSlotlockServeStore(sql, caps) {
+    return createSlotlockStore(sql, {
+        tenantLimits: serveTenantLimits(caps),
+        maxTenantTransactions: caps.tenant_transactions,
+        tenantLockTimeoutMs: SERVE_LOCK_TIMEOUT_MS,
+    });
+}
 /** Serve MCP and A2A over HTTP from the Postgres store, as one token-authenticated tenant. */
 export async function startSlotlockServer(config, log) {
     const sql = connect(config.databaseUrl, POOL_SIZE);
@@ -358,6 +412,23 @@ export async function startSlotlockServer(config, log) {
         if (!schema?.present) {
             throw Object.assign(new Error('The slotlock schema is missing: run `slotlock migrate` or `slotlock serve --migrate`'), { code: 'schema_missing' });
         }
+        // Migrated by another release: an older schema needs this release's migrate first, and a newer
+        // one has tables and rules this release's code does not know.
+        await assertSlotlockSchemaCurrent(sql);
+        // A deployment run statement by statement lifts forced row-level security for its data steps
+        // and, stopped part way, leaves it lifted until the next one puts it back. The owning role then
+        // reads every tenant's rows: refuse to serve until then.
+        const lifted = await sql `
+      SELECT rel.relname AS name
+        FROM pg_catalog.pg_class rel
+        JOIN pg_catalog.pg_namespace ns ON ns.oid = rel.relnamespace
+       WHERE ns.nspname = 'slotlock'
+         AND rel.relname IN ${sql(SLOTLOCK_TENANT_TABLES)}
+         AND rel.relrowsecurity AND NOT rel.relforcerowsecurity
+       ORDER BY rel.relname`;
+        if (lifted.length > 0) {
+            throw Object.assign(new Error(`Row-level security is enabled but not forced on ${lifted.map(({ name }) => `slotlock.${name}`).join(', ')}: a deployment stopped part way. Run \`slotlock migrate\` (or \`slotlock serve --migrate\`) to put it back`), { code: 'row_security_lifted' });
+        }
         if (config.dashboard) {
             for (const signature of SLOTLOCK_DASHBOARD_FUNCTIONS) {
                 const [granted] = await sql `
@@ -368,16 +439,47 @@ export async function startSlotlockServer(config, log) {
                 }
             }
         }
-        const store = createSlotlockStore(sql);
+        const { caps } = config.limits;
+        const store = createSlotlockServeStore(sql, caps);
         const publicUrl = new URL(config.publicUrl);
-        const authenticateKey = createSlotlockApiKeyAuthenticator(createSlotlockApiKeyStore(sql));
+        const authenticateKey = createSlotlockApiKeyAuthenticator(createSlotlockApiKeyStore(sql), {
+            maxConcurrentLookups: caps.key_lookups,
+            maxWaitingLookups: caps.key_lookup_queue,
+        });
         const authenticateToken = config.authToken === undefined
             ? undefined
-            : createSlotlockTokenAuthenticator(config.authToken, {
-                subject: SELF_HOST_SUBJECT,
-                tenantRef: config.tenantRef,
-                scopes: SLOTLOCK_API_KEY_SCOPES,
-            });
+            : // Read and write, never admin: this is the token the docs put in agents' configuration,
+                // and the application API's writes ask no person. An admin key is made for the purpose.
+                createSlotlockTokenAuthenticator(config.authToken, {
+                    subject: SELF_HOST_SUBJECT,
+                    tenantRef: config.tenantRef,
+                    scopes: SLOTLOCK_API_KEY_DEFAULT_SCOPES,
+                });
+        // Set once the listener is up; the authenticator tells it which tenants this server serves.
+        let maintenance;
+        const basePath = publicUrl.pathname.replace(/\/$/, '');
+        const limits = createSlotlockRequestLimits({
+            limits: config.limits,
+            ...(config.trustedProxyHeader !== undefined
+                ? { trustedProxyHeader: config.trustedProxyHeader }
+                : {}),
+            basePath,
+            signInPaths: config.dashboard ? ['/dashboard/sign-in', '/dashboard/callback'] : [],
+            // The token is compared in constant time first; only then is a key looked up. A person the
+            // dashboard no longer allows loses their keys with their session.
+            authenticate: async (request) => {
+                const principal = (await authenticateToken?.(request)) ?? (await authenticateKey(request));
+                if (!principal || !dashboardAllows(config, principal.tenantRef))
+                    return null;
+                maintenance?.touch(principal.tenantRef);
+                return principal;
+            },
+            // Counts and pseudonyms only: never an address, a credential or a tenant.
+            onRefusals: (counts) => log.info('refusals', { ...counts }),
+            // Behind a proxy without the setting, every client shares the proxy's address budget.
+            onIgnoredClientField: (field) => log.warn('client_address_field_ignored', { field, setting: 'SLOTLOCK_TRUSTED_PROXY_HEADER' }),
+        });
+        let health;
         const agentServer = createSlotlockAgentServer({
             publicBaseUrl: config.publicUrl,
             // readPublicUrl admits plain HTTP only on a loopback host, which is all this permits anyway.
@@ -385,22 +487,51 @@ export async function startSlotlockServer(config, log) {
             backend: createSlotlockStoreAgentBackend(store, {
                 availabilityRules: async () => config.availability.map((rule) => ({ ...rule })),
             }),
-            // The token is compared in constant time first; only then is a key looked up. A person the
-            // dashboard no longer allows loses their keys with their session.
-            authenticate: async (request) => {
-                const principal = (await authenticateToken?.(request)) ?? (await authenticateKey(request));
-                return principal && dashboardAllows(config, principal.tenantRef) ? principal : null;
+            authenticate: limits.authenticate,
+            consumeRateLimit: limits.consumeRateLimit,
+            // A write that may expand a recurring series also waits for one of a few slots, so one
+            // tenant's expansions cannot hold the event loop from every other tenant's requests.
+            authorize: async ({ principal, operation, input }) => {
+                if (!authorizeSlotlockServeOperation({ principal, operation }))
+                    return false;
+                await limits.admit({ principal, operation, input });
+                return true;
+            },
+            subscriptions: {
+                maxTotal: caps.subscriptions,
+                maxPerTenant: caps.tenant_subscriptions,
+                maxPerPrincipal: slotlockKeySubscriptions(caps),
+            },
+            health: async () => {
+                if (!health || performance.now() - health.at > HEALTH_RESULT_MS) {
+                    health = { at: performance.now(), ready: databaseReachable(sql) };
+                }
+                return (await health.ready)
+                    ? { ready: true, checks: ['database'] }
+                    : { ready: false, checks: ['database_unreachable'] };
             },
-            authorize: async ({ principal, operation }) => authorizeSlotlockServeOperation({ principal, operation }),
-            health: async () => (await databaseReachable(sql))
-                ? { ready: true, checks: ['database'] }
-                : { ready: false, checks: ['database_unreachable'] },
             // Outcomes and counts only: the server never reports arguments or identities.
-            onEvent: ({ type, ...fields }) => log.info(type, fields),
+            onEvent: ({ type, ...fields }) => log.info(type, snakeCaseKeys(fields)),
             // What an agent sees only as internal_error: the operation and the error's message and code,
             // never the arguments or the request, with every configured secret redacted.
             onError: (error, { operation }) => log.error('agent_error', { operation, ...describeError(error) }),
-            ...(config.restApi ? { rest: true } : {}),
+            ...(config.restApi
+                ? {
+                    rest: true,
+                    // Resources, hours, holds, reservations, coverage and maintenance: admin keys only.
+                    application: {
+                        backend: createSlotlockStoreApplicationBackend(store, {
+                            maxTenantResources: SERVE_MAX_TENANT_RESOURCES,
+                        }),
+                        authorize: async ({ principal, operation, input }) => {
+                            if (principal.scopes?.includes(SLOTLOCK_APPLICATION_SCOPE) !== true)
+                                return false;
+                            await limits.admit({ principal, operation, input });
+                            return true;
+                        },
+                    },
+                }
+                : {}),
             ...(config.confirmationSecret !== undefined
                 ? {
                     confirmation: {
@@ -424,19 +555,30 @@ export async function startSlotlockServer(config, log) {
                 resources: createSlotlockDashboardResources(store),
                 state: createSlotlockDashboardState(sql),
                 availability: config.availability,
+                clientAddress: (request) => limits.clientAddress(request),
                 onError: (error) => log.error('dashboard_error', describeError(error)),
             });
-        const target = dashboard
+        const target = limits.serve(dashboard
             ? {
                 fetch: (request) => dashboard.handles(request) ? dashboard.fetch(request) : agentServer.fetch(request),
                 shutdown: () => agentServer.shutdown(),
             }
-            : agentServer;
+            : agentServer);
         const listener = createSlotlockNodeServer(target, {
             requestOrigin: publicUrl.origin,
             onError: (error, phase) => log.error('http_error', { phase, ...describeError(error) }),
         });
         const address = await listener.listen({ host: config.host, port: config.port });
+        maintenance = startSlotlockMaintenance({
+            // One tenant at a time, and its long passes may wait on a request's locks for longer.
+            store: createSlotlockStore(sql, { tenantLimits: serveTenantLimits(caps) }),
+            sql,
+            tenants: [config.tenantRef],
+            intervalMs: SERVE_MAINTENANCE_INTERVAL_MS,
+            // Counts only, never the tenant.
+            onPass: (result) => log.info('maintenance', { ...result }),
+            onError: (error) => log.error('maintenance_failed', describeError(error)),
+        });
         log.info('listening', {
             address: address.origin,
             mcp: `${config.publicUrl}/mcp`,
@@ -446,11 +588,22 @@ export async function startSlotlockServer(config, log) {
             dashboard: dashboard ? `${config.publicUrl}/dashboard` : null,
             confirm_writes: config.confirmWrites,
             availability_rules: config.availability.length,
+            rest: config.restApi ? `${config.publicUrl}/openapi.json` : null,
+            maintenance_interval_minutes: SERVE_MAINTENANCE_INTERVAL_MS / 60_000,
+            limits: {
+                ...Object.fromEntries(Object.entries(config.limits.rates).map(([name, rate]) => [name, formatSlotlockRate(rate)])),
+                ...caps,
+                // What a key may hold, which is below key_subscriptions when the tenant's cap is lower.
+                key_subscriptions: slotlockKeySubscriptions(caps),
+            },
+            client_address: config.trustedProxyHeader ?? 'socket',
         });
         return {
             address,
             async close() {
                 const result = await listener.close();
+                limits.close();
+                await maintenance?.close();
                 await sql.end({ timeout: 5 });
                 return result;
             },
@@ -509,8 +662,8 @@ function hoursLine(resource) {
         hours: resource.availabilityRules ?? null,
     })}\n`;
 }
-/** Every resource of the tenant, a page at a time. */
-async function tenantResources(store, tenantRef) {
+/** Every resource of the tenant in use (with `includeArchived`, every one), a page at a time. */
+async function tenantResources(store, tenantRef, includeArchived = false) {
     const all = [];
     let after;
     for (;;) {
@@ -518,6 +671,7 @@ async function tenantResources(store, tenantRef) {
             tenantRef,
             limit: RESOURCE_PAGE_SIZE,
             ...(after !== undefined ? { after } : {}),
+            ...(includeArchived ? { includeArchived } : {}),
         }));
         all.push(...page);
         after = page.at(-1)?.id;
@@ -564,6 +718,90 @@ async function runHours(config, out, target, rules) {
         await sql.end({ timeout: 5 });
     }
 }
+/** A resource as `resource archive` and `resource restore` print it. */
+function resourceStateLine(resource) {
+    return `${JSON.stringify({
+        id: resource.id,
+        external_ref: resource.externalRef,
+        timezone: resource.timezone,
+        visibility: resource.visibility ?? 'shared',
+        archived_at: resource.archivedAt?.toISOString() ?? null,
+    })}\n`;
+}
+/**
+ * `resource archive|restore|delete`: the tenant's resource named by id, else by reference, archived
+ * ones included. Returns false when no resource has that name, and `in_use` when a delete is
+ * refused because something on the resource has not ended.
+ */
+async function changeResource(config, out, name, action) {
+    const sql = connect(config.databaseUrl, 1);
+    try {
+        const store = createSlotlockStore(sql);
+        const tenantRef = config.tenantRef;
+        const resources = await tenantResources(store, tenantRef, true);
+        // An id first: another resource's reference may spell it.
+        const named = resources.find((resource) => resource.id === name) ??
+            resources.find((resource) => resource.externalRef === name);
+        if (!named)
+            return false;
+        const id = named.id;
+        if (action === 'delete') {
+            const result = await store.withTenant(tenantRef, (tenant) => tenant.deleteResource({ tenantRef, id }));
+            if (!result.ok)
+                return 'in_use';
+            if (!result.deleted)
+                return false;
+            out.write(`${JSON.stringify({ id, external_ref: named.externalRef, deleted: true })}\n`);
+            return true;
+        }
+        const changed = await store.withTenant(tenantRef, (tenant) => action === 'archive'
+            ? tenant.archiveResource({ tenantRef, id })
+            : tenant.restoreResource({ tenantRef, id }));
+        if (!changed)
+            return false;
+        out.write(resourceStateLine(changed));
+        return true;
+    }
+    finally {
+        await sql.end({ timeout: 5 });
+    }
+}
+/**
+ * `tenant status|suspend|unsuspend|erase` for SLOTLOCK_TENANT. Suspending and lifting a suspension
+ * are the deployment role's alone, so they connect as SLOTLOCK_MIGRATE_DATABASE_URL when it is set;
+ * the serving role reads a suspension and may erase.
+ */
+async function runTenantCommand(config, out, action, reason) {
+    const owner = action === 'suspend' || action === 'unsuspend';
+    const sql = connect(owner ? (config.migrateDatabaseUrl ?? config.databaseUrl) : config.databaseUrl, 1);
+    try {
+        const store = createSlotlockStore(sql);
+        const tenantRef = config.tenantRef;
+        let line;
+        if (action === 'status') {
+            line = { tenant: tenantRef, suspended: await store.isTenantSuspended({ tenantRef }) };
+        }
+        else if (action === 'suspend') {
+            const { suspended } = await store.suspendTenant({
+                tenantRef,
+                ...(reason !== undefined ? { reason } : {}),
+            });
+            line = { tenant: tenantRef, suspended: true, changed: suspended };
+        }
+        else if (action === 'unsuspend') {
+            const { unsuspended } = await store.unsuspendTenant({ tenantRef });
+            line = { tenant: tenantRef, suspended: false, changed: unsuspended };
+        }
+        else {
+            const erased = await store.eraseTenant({ tenantRef });
+            line = { tenant: tenantRef, total: erased.total, rows: erased.rows };
+        }
+        out.write(`${JSON.stringify(line)}\n`);
+    }
+    finally {
+        await sql.end({ timeout: 5 });
+    }
+}
 /** One key as `key create`, `key list` and `key revoke` print it: never the digest. */
 function apiKeyLine(apiKey, key) {
     return `${JSON.stringify({
@@ -579,17 +817,49 @@ function apiKeyLine(apiKey, key) {
         ...(key !== undefined ? { key } : {}),
     })}\n`;
 }
-/** `--scope`: read, write, or both comma-separated. */
+/** `--scope`: read, write and admin, comma-separated; read,write unless given. */
 function parseKeyScopes(value) {
     if (value === undefined)
-        return [...SLOTLOCK_API_KEY_SCOPES];
+        return [...SLOTLOCK_API_KEY_DEFAULT_SCOPES];
     const names = value.split(',').map((name) => name.trim());
     const scopes = SLOTLOCK_API_KEY_SCOPES.filter((scope) => names.includes(scope));
     if (names.length === 0 || names.some((name) => !scopes.includes(name))) {
-        throw new Error('--scope must be read, write or read,write');
+        throw new Error('--scope must be a comma-separated list of read, write and admin');
     }
     return scopes;
 }
+/**
+ * `maintain`: maintenance passes for the tenant until nothing is left, under the tenant's lock, so
+ * it never repeats what a server is doing. Null when another server holds the lock.
+ */
+async function maintainTenant(config) {
+    const sql = connect(config.databaseUrl, 2);
+    try {
+        const store = createSlotlockStore(sql);
+        const total = await withSlotlockMaintenanceLock(sql, config.tenantRef, async () => {
+            const sum = { extended: 0, conflicts: 0, refused: 0, pruned: 0, capped: true, passes: 0 };
+            while (sum.capped && sum.passes < MAINTAIN_MAX_PASSES) {
+                const pass = await maintainSlotlockTenant(store, config.tenantRef);
+                sum.passes += 1;
+                sum.extended += pass.extended;
+                sum.conflicts += pass.conflicts;
+                sum.refused += pass.refused;
+                sum.pruned += pass.pruned;
+                // A capped pass that got nowhere met only series it cannot extend. While its roll stopped
+                // before the series it had put back, the next pass meets new ones (and perhaps a healthy
+                // one); once it reached them, another pass would meet them again.
+                sum.capped = pass.capped;
+                if (pass.capped && pass.extended + pass.pruned === 0 && !pass.rollPending)
+                    break;
+            }
+            return sum;
+        });
+        return total ?? null;
+    }
+    finally {
+        await sql.end({ timeout: 5 });
+    }
+}
 /** `--expires-in-days`: a whole number of days up to the longest a key may live. */
 function parseKeyLifetime(value) {
     if (value === undefined)
@@ -632,23 +902,41 @@ export const SLOTLOCK_CLI_HELP = `Usage: slotlock <command> [options]
 Commands:
   migrate                  Apply the schema and row-level security, then grant the serving
                            role when SLOTLOCK_MIGRATE_DATABASE_URL names another role
-  serve [--migrate]        Serve MCP and A2A over HTTP (--migrate runs migrate first)
+  serve [--migrate]        Serve MCP, A2A and REST over HTTP (--migrate runs migrate first);
+                           maintains each tenant it serves every hour
+  maintain                 Run maintenance for the tenant now: extend recurring events to the
+                           rolling horizon and prune past the retention window (for cron)
   resource add <ref> [--timezone <IANA zone>]
                            Create a calendar resource (a car, room, person or machine)
   resource list            Print the tenant's resources, one JSON object per line
+  resource archive <ref>   Take a resource out of use: no new bookings, holds or events, and
+                           no longer listed; what is on it stays
+  resource restore <ref>   Put an archived resource back in use
+  resource delete <ref>    Delete a resource and its history; refused while a booking, hold
+                           or event on it has not ended
   hours show [--resource <ref>]
                            Print each resource's bookable hours (null: the server's)
   hours set <json> (--resource <ref> | --all)
                            Give one resource, or every one, weekly hours of its own; [] closes
   hours clear (--resource <ref> | --all)
                            Hand one resource, or every one, back to the server's hours
-  key create <name> [--scope read|write|read,write] [--expires-in-days <n>]
+  key create <name> [--scope read,write,admin] [--expires-in-days <n>]
                            Create an API key for the tenant (default scope read,write, no
-                           expiry) and print it once; only its SHA-256 is stored
+                           expiry; admin adds the application API) and print it once; only its
+                           SHA-256 is stored
   key list                 Print the tenant's API keys, one JSON object per line
   key rotate <id>          Give an active key a new secret and print it once; the old one stops
                            working at once, and the key keeps its id and what it booked
   key revoke <id>          Revoke one of the tenant's API keys at once
+  tenant status            Print whether the tenant is suspended
+  tenant suspend [--reason <text>]
+                           Suspend the tenant: it may read, and writes nothing; its keys get
+                           no new secret (as SLOTLOCK_MIGRATE_DATABASE_URL's role when set:
+                           only the schema owner suspends)
+  tenant unsuspend         Lift the tenant's suspension (as the schema owner, likewise)
+  tenant erase --confirm <tenant>
+                           Delete everything the tenant has, its API keys included, for good;
+                           --confirm repeats SLOTLOCK_TENANT
   healthcheck              Exit 0 when the local server reports ready
 
 Options:
@@ -659,15 +947,40 @@ Environment:
   DATABASE_URL                   postgres:// URL of the role Slotlock serves as (required); serve
                                  refuses a superuser or a role with BYPASSRLS
   SLOTLOCK_MIGRATE_DATABASE_URL  postgres:// URL of the schema owner, for migrate (optional)
-  SLOTLOCK_AUTH_TOKEN            Optional bearer token for SLOTLOCK_TENANT with every scope,
-                                 32+ random characters (serve); API keys work with or without it
-  SLOTLOCK_CONFIRMATION_SECRET   Signs pending write confirmations, 32+ random characters (serve)
-  SLOTLOCK_CONFIRM_WRITES        all (default), none, or a comma-separated list of write tools
+  SLOTLOCK_AUTH_TOKEN            Optional bearer token for SLOTLOCK_TENANT with read and write
+                                 access (never admin), 32+ random characters (serve); API keys
+                                 work with or without it
+  SLOTLOCK_CONFIRMATION_SECRET   Signs pending write confirmations, 32+ random characters (serve;
+                                 required when SLOTLOCK_CONFIRM_WRITES is not none)
+  SLOTLOCK_CONFIRM_WRITES        none (default), all, or a comma-separated list of write tools
+                                 that wait for a person (MCP 2026-07-28 form elicitation; refused
+                                 with confirmation_required on REST, A2A and older MCP)
   SLOTLOCK_PUBLIC_URL            URL clients use (default http://localhost:$PORT)
   SLOTLOCK_TENANT                Tenant of the token, resource and key commands (default "default")
   SLOTLOCK_AVAILABILITY          JSON array of weekly bookable-hours rules (default: none)
-  SLOTLOCK_REST_API              on: also serve the tools as a REST API under /v1, described at
-                                 /openapi.json, with the same keys and confirmation (default off)
+  SLOTLOCK_REST_API              on (default): also serve the tools as a REST API under /v1, and
+                                 the application API for admin keys, described at /openapi.json;
+                                 off: MCP and A2A only
+  SLOTLOCK_LIMITS                Request limits, comma-separated name=value over the defaults:
+                                 rates as <count>/<s|m|h>[:<burst>] or off for address (50/s:100,
+                                 every request per client, before any key lookup), key_read,
+                                 tenant_read (20/s:40), key_write, tenant_write (5/s:20),
+                                 global_read (500/s:1000), global_write (50/s:100), app_read
+                                 (20/s:40) and app_write (5/s:20) per tenant for the
+                                 application API, sign_in (20/m:20); counts for
+                                 tenant_series_writes and series_writes
+                                 (1: writes that may expand a recurring series at once),
+                                 maintenance_runs (1: application API maintenance runs at once),
+                                 subscriptions (256), tenant_subscriptions (16),
+                                 key_subscriptions (4, and fewer than tenant_subscriptions),
+                                 key_lookups (2), key_lookup_queue (256),
+                                 tenant_transactions (2: one tenant's database transactions at
+                                 once, of the pool's 10); what one tenant may store:
+                                 tenant_holds (10000), tenant_reservations, tenant_events and
+                                 tenant_coverage (100000)
+  SLOTLOCK_TRUSTED_PROXY_HEADER  The request field a proxy in front sets to the client address,
+                                 such as CF-Connecting-IP (default: the connection's address);
+                                 set it only when every request comes through that proxy
   SLOTLOCK_GITHUB_CLIENT_ID      Serve the dashboard at $SLOTLOCK_PUBLIC_URL/dashboard: the Client ID
                                  of a GitHub OAuth app with callback .../dashboard/callback
   SLOTLOCK_GITHUB_CLIENT_SECRET  That app's client secret (dashboard)
@@ -709,15 +1022,23 @@ export async function runSlotlockCli(argv, io) {
         io.stderr.write(SLOTLOCK_CLI_HELP);
         return 2;
     }
-    const subcommand = command === 'resource' || command === 'key' || command === 'hours'
+    const subcommand = command === 'resource' || command === 'key' || command === 'hours' || command === 'tenant'
         ? `${command} ${operands[0] ?? ''}`.trim()
         : command;
     const expectedOperands = {
         migrate: 0,
         serve: 0,
+        maintain: 0,
         healthcheck: 0,
         'resource add': 2,
         'resource list': 1,
+        'resource archive': 2,
+        'resource restore': 2,
+        'resource delete': 2,
+        'tenant status': 1,
+        'tenant suspend': 1,
+        'tenant unsuspend': 1,
+        'tenant erase': 1,
         'key create': 2,
         'key list': 1,
         'key rotate': 2,
@@ -741,6 +1062,15 @@ export async function runSlotlockCli(argv, io) {
             return usage(`--${option} applies only to key create`);
         }
     }
+    if (values.reason !== undefined && subcommand !== 'tenant suspend') {
+        return usage('--reason applies only to tenant suspend');
+    }
+    if (values.confirm !== undefined && subcommand !== 'tenant erase') {
+        return usage('--confirm applies only to tenant erase');
+    }
+    if (subcommand === 'tenant erase' && values.confirm === undefined) {
+        return usage('tenant erase needs --confirm <tenant>, the tenant it erases (SLOTLOCK_TENANT)');
+    }
     if (values.all && subcommand !== 'hours set' && subcommand !== 'hours clear') {
         return usage('--all applies only to hours set and hours clear');
     }
@@ -797,6 +1127,15 @@ export async function runSlotlockCli(argv, io) {
                 log.info('stopped', { forced });
                 return 0;
             }
+            case 'maintain': {
+                const result = await maintainTenant(readSlotlockDatabaseConfig(io.env));
+                if (!result) {
+                    io.stderr.write('slotlock: another server is maintaining this tenant now; nothing to do\n');
+                    return 0;
+                }
+                io.stdout.write(`${JSON.stringify(result)}\n`);
+                return 0;
+            }
             case 'resource add': {
                 const config = readSlotlockDatabaseConfig(io.env);
                 const resource = await addResource(config, operands[1], values.timezone ?? 'UTC');
@@ -806,6 +1145,45 @@ export async function runSlotlockCli(argv, io) {
             case 'resource list':
                 await listResources(readSlotlockDatabaseConfig(io.env), io.stdout);
                 return 0;
+            case 'resource archive':
+            case 'resource restore':
+            case 'resource delete': {
+                const changed = await changeResource(readSlotlockDatabaseConfig(io.env), io.stdout, operands[1], subcommand.slice('resource '.length));
+                if (changed === 'in_use') {
+                    io.stderr.write('slotlock: a booking, hold or calendar event on that resource has not ended; cancel or move it first, or archive the resource\n');
+                    return 1;
+                }
+                if (!changed) {
+                    io.stderr.write('slotlock: no resource with that reference or id in this tenant\n');
+                    return 1;
+                }
+                return 0;
+            }
+            case 'tenant status':
+            case 'tenant suspend':
+            case 'tenant unsuspend':
+            case 'tenant erase': {
+                const config = readSlotlockDatabaseConfig(io.env);
+                // The tenant named twice, so erasing the one SLOTLOCK_TENANT happens to name is deliberate.
+                if (subcommand === 'tenant erase' && values.confirm !== config.tenantRef) {
+                    return usage('--confirm must repeat SLOTLOCK_TENANT, the tenant tenant erase deletes');
+                }
+                try {
+                    await runTenantCommand(config, io.stdout, subcommand.slice('tenant '.length), values.reason);
+                }
+                catch (error) {
+                    const code = error?.code;
+                    if (code === 'tenant_suspension_not_permitted') {
+                        io.stderr.write('slotlock: only the schema owner suspends a tenant or lifts a suspension; set SLOTLOCK_MIGRATE_DATABASE_URL to its URL\n');
+                        return 1;
+                    }
+                    if (code === 'invalid_suspension_reason') {
+                        return usage('--reason must be 1-500 characters without control characters');
+                    }
+                    throw error;
+                }
+                return 0;
+            }
             case 'hours show':
             case 'hours set':
             case 'hours clear': {
@@ -889,6 +1267,8 @@ function parseCliArguments(argv) {
             'expires-in-days': { type: 'string' },
             resource: { type: 'string' },
             all: { type: 'boolean' },
+            reason: { type: 'string' },
+            confirm: { type: 'string' },
             help: { type: 'boolean', short: 'h' },
             version: { type: 'boolean', short: 'v' },
         },