pgkiln 0.78.0 → 0.80.0

package/src/remote-db.ts

@@ -0,0 +1,145 @@
+import pg from 'pg';
+
+// (sprint 150, AP-7 leftover; APEX: REST Enabled SQL, regions on another database) Remote databases: another
+// PostgreSQL database's tables as foreign tables (postgres_fdw) in a local schema of their own, which applications
+// query like their own tables (as their database role, in their transaction, with the grants given here). See
+// migration 145 for the table; Workspace utilities → Remote databases (src/builder/remote-databases.ts) for the page.
+//
+// Everything runs on the owner connection in the caller's transaction:
+//   create (or alter) server pgkiln_rdb_<name> foreign data wrapper postgres_fdw options (host, port, dbname)
+//   create (or alter) user mapping for public … options (user, password)   -- the password only lives here
+//   create schema <local> (marked as this remote database's by its comment), drop its foreign tables,
+//   import foreign schema <remote> [limit to (…)] from server … into <local>
+//   revoke from public; grant usage and select (with writable: insert, update, delete) to the chosen app roles
+// Names reach SQL quoted (escapeIdentifier) or as literals (escapeLiteral), after the checks below.
+
+export interface RemoteDb {
+  id?: number;
+  name: string;
+  description: string | null;
+  host: string;
+  port: number;
+  dbname: string;
+  user_name: string;
+  remote_schema: string;
+  local_schema: string;
+  tables: string[];
+  grant_roles: string[];
+  writable: boolean;
+}
+
+type Q = { query: (sql: string, params?: unknown[]) => Promise<pg.QueryResult<any>> };
+
+const id = pg.escapeIdentifier;
+const lit = pg.escapeLiteral;
+export const NAME = /^[A-Z][A-Z0-9_]{0,29}$/;
+const LOCAL = /^[a-z_][a-z0-9_]{0,62}$/;
+// a host name, an IPv4 or an IPv6 address (no spaces, quotes or option syntax)
+const HOST = /^(?:[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?|\[?[0-9A-Fa-f:.]{2,45}\]?)$/;
+// a remote name (database, user, schema, table): any printable text without control characters, at most 63
+const REMOTE = /^[^\x00-\x1f\x7f]{1,63}$/;
+const RESERVED = /^(pg_.*|information_schema|meta|public|pgkiln.*)$/;
+export const MAX_TABLES = 200;
+
+export const serverNameOf = (name: string) => `pgkiln_rdb_${name.toLowerCase()}`;
+const marker = (name: string) => `pgkiln remote database ${name}`;
+
+/** What is wrong with a remote database's settings (empty: nothing). `roles`: the roles that may be granted. */
+export function remoteDbProblems(x: RemoteDb, roles: string[]): string[] {
+  const p: string[] = [];
+  if (!NAME.test(x.name)) p.push('The name is an upper case static id (letters, digits, _; at most 30), e.g. CRM.');
+  if (!HOST.test(x.host)) p.push('The host is a host name or an IP address.');
+  if (!Number.isInteger(x.port) || x.port < 1 || x.port > 65535) p.push('The port is a number from 1 to 65535.');
+  if (!REMOTE.test(x.dbname)) p.push('The database is a name of 1 to 63 characters.');
+  if (!REMOTE.test(x.user_name)) p.push('The user is a name of 1 to 63 characters.');
+  if (!REMOTE.test(x.remote_schema)) p.push('The remote schema is a name of 1 to 63 characters.');
+  if (!LOCAL.test(x.local_schema) || RESERVED.test(x.local_schema)) p.push('The local schema is a new lower case name (not public, meta or pg_…), e.g. crm.');
+  if (x.tables.length > MAX_TABLES) p.push(`At most ${MAX_TABLES} tables.`);
+  for (const t of x.tables) if (!REMOTE.test(t)) p.push(`The table name "${t.slice(0, 70)}" is not a name of 1 to 63 characters.`);
+  for (const r of x.grant_roles) if (!roles.includes(r)) p.push(`${r} is not the database role of an application.`);
+  if (x.description && x.description.length > 1000) p.push('The description has at most 1000 characters.');
+  return p;
+}
+
+export interface FdwStatus { installed: boolean; available: boolean; canCreate: boolean }
+
+/** Whether postgres_fdw is installed in this database (and whether the connection could create it). */
+export async function fdwStatus(db: Q): Promise<FdwStatus> {
+  const r = (await db.query(
+    `select exists (select 1 from pg_catalog.pg_extension where extname = 'postgres_fdw') as installed,
+            exists (select 1 from pg_catalog.pg_available_extensions where name = 'postgres_fdw') as available,
+            (select rolsuper from pg_catalog.pg_roles where rolname = current_user) as superuser`)).rows[0];
+  return { installed: r.installed, available: r.available, canCreate: !!r.superuser };
+}
+
+/** The database roles of the applications (the roles a remote database's tables may be granted to). */
+export async function applicationRoles(db: Q): Promise<string[]> {
+  return (await db.query(
+    `select distinct a.db_role as r from meta.app a where a.db_role is not null
+        and exists (select 1 from pg_catalog.pg_roles x where x.rolname = a.db_role) order by 1`)).rows.map((x) => x.r);
+}
+
+/**
+ * Make (or change) the server, user mapping and local schema, and import the tables again: the caller's transaction
+ * (nothing stays when it fails or is rolled back). `password`: null keeps the mapping's password (it must exist
+ * then); `before`: the roles granted until now (their grants go first). Returns the number of foreign tables.
+ */
+export async function applyRemote(c: Q, x: RemoteDb, password: string | null, before: string[] = []): Promise<number> {
+  if (!(await fdwStatus(c)).installed) throw new Error('The postgres_fdw extension is not installed in this database (Workspace utilities → Remote databases).');
+  const server = serverNameOf(x.name);
+  const exists = (await c.query('select 1 from pg_catalog.pg_foreign_server where srvname = $1', [server])).rowCount;
+  const opts: [string, string][] = [['host', x.host], ['port', String(x.port)], ['dbname', x.dbname]];
+  if (exists) await c.query(`alter server ${id(server)} options (${opts.map(([k, v]) => `set ${k} ${lit(v)}`).join(', ')})`);
+  else await c.query(`create server ${id(server)} foreign data wrapper postgres_fdw options (${opts.map(([k, v]) => `${k} ${lit(v)}`).join(', ')})`);
+  const mapped = (await c.query(`select 1 from pg_catalog.pg_user_mappings where srvname = $1 and usename = 'public'`, [server])).rowCount;
+  if (!mapped) {
+    if (!password) throw new Error('A password is needed the first time (postgres_fdw connects with the remote user and its password).');
+    await c.query(`create user mapping for public server ${id(server)} options (user ${lit(x.user_name)}, password ${lit(password)})`);
+  } else await c.query(`alter user mapping for public server ${id(server)} options (set user ${lit(x.user_name)}${password ? `, set password ${lit(password)}` : ''})`);
+
+  // the local schema: new, or this remote database's own
+  const schema = (await c.query(
+    `select pg_catalog.obj_description(n.oid, 'pg_namespace') as comment from pg_catalog.pg_namespace n where n.nspname = $1`, [x.local_schema])).rows[0];
+  if (schema && schema.comment !== marker(x.name))
+    throw new Error(`Schema ${x.local_schema} exists and is not remote database ${x.name}'s: choose another local schema.`);
+  if (!schema) {
+    await c.query(`create schema ${id(x.local_schema)}`);
+    await c.query(`comment on schema ${id(x.local_schema)} is ${lit(marker(x.name))}`);
+  }
+  // the foreign tables again: drop the old ones (a view that uses one stops this, and says which)
+  const old = (await c.query(
+    `select c.relname as t from pg_catalog.pg_class c join pg_catalog.pg_namespace n on n.oid = c.relnamespace
+      where n.nspname = $1 and c.relkind = 'f'`, [x.local_schema])).rows;
+  for (const o of old) await c.query(`drop foreign table ${id(x.local_schema)}.${id(o.t)}`);
+  await c.query(`import foreign schema ${id(x.remote_schema)}${x.tables.length ? ` limit to (${x.tables.map(id).join(', ')})` : ''}
+                 from server ${id(server)} into ${id(x.local_schema)}`);
+  const count = Number((await c.query(
+    `select count(*) as n from pg_catalog.pg_class c join pg_catalog.pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind = 'f'`,
+    [x.local_schema])).rows[0].n);
+  // grants: only the chosen application roles, nobody else
+  await c.query(`revoke all on all tables in schema ${id(x.local_schema)} from public`);
+  await c.query(`revoke all on schema ${id(x.local_schema)} from public`);
+  for (const r of new Set([...before, ...x.grant_roles])) {
+    if (!(await c.query('select 1 from pg_catalog.pg_roles where rolname = $1', [r])).rowCount) continue;
+    await c.query(`revoke all on all tables in schema ${id(x.local_schema)} from ${id(r)}`);
+    await c.query(`revoke all on schema ${id(x.local_schema)} from ${id(r)}`);
+  }
+  for (const r of x.grant_roles) {
+    await c.query(`grant usage on schema ${id(x.local_schema)} to ${id(r)}`);
+    await c.query(`grant ${x.writable ? 'select, insert, update, delete' : 'select'} on all tables in schema ${id(x.local_schema)} to ${id(r)}`);
+  }
+  return count;
+}
+
+/** Drop the local schema (when it is this remote database's) and the server with its user mapping. */
+export async function dropRemote(c: Q, x: Pick<RemoteDb, 'name' | 'local_schema'>) {
+  const schema = (await c.query(
+    `select pg_catalog.obj_description(n.oid, 'pg_namespace') as comment from pg_catalog.pg_namespace n where n.nspname = $1`, [x.local_schema])).rows[0];
+  if (schema && schema.comment === marker(x.name)) await c.query(`drop schema ${id(x.local_schema)} cascade`);
+  await c.query(`drop server if exists ${id(serverNameOf(x.name))} cascade`);
+}
+
+/** postgres_fdw keeps connections per session: drop this connection's, so the next query uses new settings. */
+export async function disconnect(c: Q) {
+  await c.query(`select postgres_fdw_disconnect_all()`).catch(() => undefined);
+}

package/src/runtime/assistant.ts

@@ -382,7 +382,7 @@ export async function renderAssistant(ctx: PageContext, r: Region): Promise<Raw>
   }
   const welcome = c.welcome ? substitute(ctx.locale.tr(c.welcome), ctx, (v) => v) : '';
   const usable = mayUse(ctx, r);
-  return html`<div class="assistant" data-assistant="${r.id}">
+  return html`<div class="assistant" data-assistant="${r.id}" data-you="${t('assistant.you')}" data-who="${t('assistant.assistant')}" data-used="${t('assistant.used', { tools: '{tools}' })}">
     <ol class="assistant-log" aria-label="${t('assistant.conversation')}" aria-live="polite">
       ${welcome ? html`<li class="assistant-msg assistant-msg-assistant"><span class="assistant-who">${t('assistant.assistant')}</span><div class="assistant-text">${formatAnswer(welcome)}</div></li>` : ''}
       ${turns.map((x) => html`<li class="assistant-msg assistant-msg-${x.role === 'user' ? 'user' : 'assistant'}">
@@ -427,7 +427,10 @@ async function contextData(ctx: PageContext, c: AssistantConfig, question: strin
   return parts;
 }
 
-async function ask(ctx: PageContext, r: Region, message: string) {
+/** (sprint 148, WS-4) where a streamed answer goes while the model writes it */
+interface Live { text(delta: string): void; tool(name: string): void }
+
+async function ask(ctx: PageContext, r: Region, message: string, live?: Live) {
   const t = ctx.locale.t;
   const c = configOf(r);
   const agent = c.agent ? await agentFor(ctx.app.id, c.agent) : null;
@@ -482,6 +485,7 @@ async function ask(ctx: PageContext, r: Region, message: string) {
     maxRounds: agent ? (agent.agent.max_steps ?? DEFAULT_STEPS) : (c.max_rounds ?? DEFAULT_ROUNDS),
     runTool: runLogged,
     ...(agent ? { effort: agent.agent.effort ?? null, maxTokens: agent.agent.max_tokens ?? null } : {}),
+    ...(live ? { onText: live.text, onTool: live.tool } : {}),
   }, { appId: ctx.app.id, pageNo: ctx.page.page_no, user: ctx.user, source: 'assistant', debug: (l, x) => dbg(ctx, l, 'ai', x) });
   dbg(ctx, 9, 'ai', () => `answer: ${res.text}`);
   const turns: Turn[] = [...(conv?.turns ?? []), { role: 'user', text: message },
@@ -499,8 +503,11 @@ async function ask(ctx: PageContext, r: Region, message: string) {
   if (extras.actions.length) ctx.session.state[`__AIACT${r.id}`] = JSON.stringify(extras.actions);
 }
 
+/** (sprint 148) the browser asked for the answer as it is written (app.js sends this with JavaScript) */
+const wantsStream = (req: Req) => String(req.headers.accept ?? '').includes('text/event-stream');
+
 export async function assistantRoutes(app: FastifyInstance) {
-  const action = async (req: Req, reply: FastifyReply, run: (ctx: PageContext, r: Region) => Promise<void>) => {
+  const action = async (req: Req, reply: FastifyReply, run: (ctx: PageContext, r: Region, live?: Live) => Promise<void>, stream = false) => {
     const ctx = await loadContext(req, reply);
     if (!ctx) return;
     const body = req.body ?? {};
@@ -512,30 +519,60 @@ export async function assistantRoutes(app: FastifyInstance) {
     const r = ctx.page.regions.find((x) => x.id === Number(req.params.id) && x.type === 'ai_assistant');
     if (!r) return refuse(`ai assistant: region ${req.params.id} on page ${ctx.page.page_no}`);
     if (!mayUse(ctx, r)) return refuse(`ai assistant: region ${r.id}: not signed in`);
+    const q = typeof body.params === 'string' ? new URLSearchParams(body.params.slice(0, 4000)).toString() : '';
+    const url = `${ctx.base}/${ctx.page.page_no}${q ? `?${q}` : ''}#R${r.id}`;
+    // (sprint 148, WS-4) a streamed answer: server-sent events with the text as the model writes it (JSON strings,
+    // shown as text by app.js), each tool before it runs, then "done" with the page to load: it shows the saved
+    // answer, its links and actions, or the error, exactly as the redirect below does without JavaScript.
+    let live: Live | undefined;
+    let out: FastifyReply['raw'] | null = null;
+    if (stream) {
+      reply.hijack();
+      out = reply.raw;
+      out.writeHead(200, {
+        'content-type': 'text/event-stream; charset=utf-8', 'cache-control': 'no-store', 'x-accel-buffering': 'no',
+        'x-content-type-options': 'nosniff', 'referrer-policy': 'same-origin',
+      });
+      const raw = out;
+      const send = (event: string, data: unknown) => {
+        if (!raw.writableEnded && !raw.destroyed) raw.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+      };
+      live = { text: (d) => send('text', d), tool: (n) => send('tool', n) };
+    }
+    let forbidden = false;
     try {
-      await run(ctx, r);
+      await run(ctx, r, live);
     } catch (e) {
-      if (e instanceof Forbidden) return refuse(`ai assistant: region ${r.id} on page ${ctx.page.page_no}`);
-      const c = configOf(r);
-      const msg = e instanceof InputError ? e.message
-        : e instanceof AiError ? (c.error_message && e.kind !== 'limit' ? ctx.locale.tr(c.error_message) : e.message)
-        : await publicError(ctx, e, 'AI assistant');
-      if (e instanceof AiError) dbg(ctx, 1, 'ai', `AI assistant: ${e.kind}: ${e.message}`);
-      ctx.session.state.__FLASH_ERROR = msg;
-      if (typeof body.message === 'string') ctx.session.state[`__AIDRAFT${r.id}`] = body.message.slice(0, MAX_MESSAGE_CHARS);
+      if (e instanceof Forbidden) {
+        if (!out) return refuse(`ai assistant: region ${r.id} on page ${ctx.page.page_no}`);
+        logActivity({ appId: ctx.app.id, pageNo: ctx.page.page_no, username: ctx.user, event: 'forbidden', ip: ctx.ip, detail: `ai assistant: region ${r.id} on page ${ctx.page.page_no}` });
+        ctx.session.state.__FLASH_ERROR = ctx.locale.t('error.access_denied');
+        forbidden = true;
+      } else {
+        const c = configOf(r);
+        const msg = e instanceof InputError ? e.message
+          : e instanceof AiError ? (c.error_message && e.kind !== 'limit' ? ctx.locale.tr(c.error_message) : e.message)
+          : await publicError(ctx, e, 'AI assistant');
+        if (e instanceof AiError) dbg(ctx, 1, 'ai', `AI assistant: ${e.kind}: ${e.message}`);
+        ctx.session.state.__FLASH_ERROR = msg;
+        if (typeof body.message === 'string') ctx.session.state[`__AIDRAFT${r.id}`] = body.message.slice(0, MAX_MESSAGE_CHARS);
+      }
     }
     await saveState(ctx.session);
-    const q = typeof body.params === 'string' ? new URLSearchParams(body.params.slice(0, 4000)).toString() : '';
-    return reply.redirect(`${ctx.base}/${ctx.page.page_no}${q ? `?${q}` : ''}#R${r.id}`, 303);
+    if (out) {
+      if (!out.writableEnded && !out.destroyed) out.end(`event: done\ndata: ${JSON.stringify({ url, ...(forbidden ? { forbidden: true } : {}) })}\n\n`);
+      return;
+    }
+    return reply.redirect(url, 303);
   };
 
   app.post('/a/:alias/:page/assistant/:id/send', async (req: Req, reply) =>
-    action(req, reply, async (ctx, r) => {
+    action(req, reply, async (ctx, r, live) => {
       const message = typeof req.body?.message === 'string' ? req.body.message.replace(/\r\n?/g, '\n').trim() : '';
       if (!message) throw new InputError(ctx.locale.t('assistant.empty'));
       if (message.length > MAX_MESSAGE_CHARS) throw new InputError(ctx.locale.t('assistant.too_long', { max: String(MAX_MESSAGE_CHARS) }));
-      await ask(ctx, r, message);
-    }),
+      await ask(ctx, r, message, live);
+    }, wantsStream(req)),
   );
 
   app.post('/a/:alias/:page/assistant/:id/clear', async (req: Req, reply) =>