pgkiln 0.79.0 → 0.80.0

package/src/ai/chat.ts

@@ -52,6 +52,12 @@ export interface ChatRequest {
   effort?: 'low' | 'medium' | 'high' | null;
   /** (121) an agent's output limit per call (never above the service's maximum) */
   maxTokens?: number | null;
+  /**
+   * (sprint 148, WS-4) streamed answers: the text as the model writes it, in pieces (also the text of a round that
+   * then calls tools), and each tool before it runs. Without them a call waits for the whole answer, as before.
+   */
+  onText?: (delta: string) => void;
+  onTool?: (name: string) => void;
 }
 
 export interface ChatResult {
@@ -122,7 +128,7 @@ async function claudeChat(s: AiService, limits: Parameters<typeof metered>[1], r
     const msg = await metered(s, limits, info, async () => {
       let m: Anthropic.Beta.BetaMessage;
       try {
-        m = await client.beta.messages.stream({
+        const stream = client.beta.messages.stream({
           model: s.model,
           max_tokens: outputLimit(s, req),
           system: req.system,
@@ -130,7 +136,9 @@ async function claudeChat(s: AiService, limits: Parameters<typeof metered>[1], r
           ...(tools.length ? { tools } : {}),
           ...((req.effort ?? s.effort) ? { output_config: { effort: (req.effort ?? s.effort)! } } : {}),
           ...(s.refusal_fallback ? { betas: ['server-side-fallback-2026-07-01'], fallbacks: 'default' as const } : {}),
-        }).finalMessage();
+        });
+        if (req.onText) stream.on('text', (delta) => req.onText!(delta));
+        m = await stream.finalMessage();
       } catch (e) {
         throw claudeError(e);
       }
@@ -150,6 +158,7 @@ async function claudeChat(s: AiService, limits: Parameters<typeof metered>[1], r
     if (round >= req.maxRounds) throw new AiError('output', TOO_MANY_ROUNDS);
     const results: Anthropic.Beta.BetaToolResultBlockParam[] = [];
     for (const call of calls) {
+      req.onTool?.(call.name);
       const out = await req.runTool(call.name, (call.input ?? {}) as Record<string, unknown>);
       used.push({ name: call.name, ok: !out.isError });
       results.push({ type: 'tool_result', tool_use_id: call.id, content: clip(out.content), ...(out.isError ? { is_error: true } : {}) });
@@ -169,15 +178,32 @@ async function openAiChat(s: AiService, limits: Parameters<typeof metered>[1], r
     const res = await metered(s, limits, info, async () => {
       let r: OpenAI.Chat.ChatCompletion;
       try {
-        r = await client.chat.completions.create({
+        const params = {
           model: s.model,
           // the system prompt is sent with each request, not kept in the history
-          messages: [{ role: 'system', content: req.system }, ...messages],
+          messages: [{ role: 'system' as const, content: req.system }, ...messages],
           max_completion_tokens: outputLimit(s, req),
           ...(tools.length ? { tools } : {}),
           // (121) an agent's reasoning effort; models without reasoning refuse it, so none is sent without one
           ...(req.effort ? { reasoning_effort: req.effort } : {}),
-        });
+        };
+        if (req.onText) {
+          // (sprint 148) streamed: the content as it comes; the usage in the last chunk
+          // an OpenAI-compatible endpoint that refuses streaming (400) gets the plain request once
+          let streamed: OpenAI.Chat.ChatCompletion | null = null;
+          let sent = false;
+          try {
+            const stream = client.chat.completions.stream({ ...params, stream_options: { include_usage: true } });
+            stream.on('content', (delta) => {
+              sent = true;
+              req.onText!(delta);
+            });
+            streamed = await stream.finalChatCompletion();
+          } catch (e) {
+            if (sent || (e as { status?: number }).status !== 400) throw e;
+          }
+          r = streamed ?? (await client.chat.completions.create(params));
+        } else r = await client.chat.completions.create(params);
       } catch (e) {
         throw openAiError(e);
       }
@@ -198,6 +224,7 @@ async function openAiChat(s: AiService, limits: Parameters<typeof metered>[1], r
     for (const call of calls) {
       let input: Record<string, unknown>;
       let out: ToolOutcome;
+      req.onTool?.(call.function.name);
       try {
         input = JSON.parse(call.function.arguments || '{}');
         out = input && typeof input === 'object' && !Array.isArray(input) ? await req.runTool(call.function.name, input) : { content: 'The arguments are not a JSON object.', isError: true };

package/src/app.ts

@@ -99,6 +99,7 @@ import { mailQueueRoutes } from './builder/mail-queue.ts';
 import { networkAccessRoutes } from './builder/network-access.ts';
 import { workspaceMailRoutes } from './builder/workspace-mail.ts';
 import { workspaceResourceRoutes } from './builder/workspace-resources.ts';
+import { remoteDatabaseRoutes } from './builder/remote-databases.ts';
 import { runtimeScope } from './db.ts';
 // (sprint 139, AP-6) OpenTelemetry: a trace per request, /metrics, the browser relay
 import { startTelemetry } from './telemetry/export.ts';
@@ -288,6 +289,7 @@ export async function buildApp(opts: { logger?: boolean } = {}) {
   await app.register(networkAccessRoutes);
   await app.register(workspaceMailRoutes);
   await app.register(workspaceResourceRoutes);
+  await app.register(remoteDatabaseRoutes);
   // (sprint 138, AP-5) an application's Utilities
   await app.register(utilityRoutes);
   await app.register(dependencyRoutes);

package/src/builder/home.ts

@@ -288,6 +288,7 @@ export async function homeRoutes(app: FastifyInstance) {
         ${card(`${BASE}/backups`, icon('archive'), 'Manage backups', 'The backups of every application of this workspace; back them all up at once.')}
         ${card(`${BASE}/workspace-files`, icon('file'), 'Static workspace files', 'CSS, JavaScript, images and fonts every application of this workspace can load (#WORKSPACE_FILES#).')}
         ${card(`${BASE}/remote-servers`, icon('server'), 'Remote servers', 'Named base URLs for REST data sources, catalogs and AI services: change one value to move them all.')}
+        ${(await isAdmin(s.username)) ? card(`${BASE}/remote-databases`, icon('database'), 'Remote databases', 'Another PostgreSQL database\'s tables as foreign tables (postgres_fdw) the applications query like their own (administrators).') : ''}
         ${card(`${BASE}/workspace-credentials`, icon('key'), 'Web credentials', 'Keys and OAuth2 clients the applications of this workspace use by name (secrets encrypted, write-only).')}
         ${card(`${BASE}/rest-catalogs`, icon('globe'), 'REST Source Catalogs', 'Web services described once for the workspace (e.g. from an OpenAPI document); applications make REST data sources from them.')}
         ${s.workspace?.admin ? card(`${BASE}/workspace-export`, icon('download'), 'Export and import', 'This workspace\'s components as one file: settings, developers, groups, links, catalogs, scripts, blueprints (workspace administrators).') : ''}

package/src/builder/remote-databases.ts

@@ -0,0 +1,215 @@
+import type { FastifyInstance } from 'fastify';
+import { owner } from '../db.ts';
+import { html, raw } from '../html.ts';
+import { icon } from '../icons.ts';
+import { applicationRoles, applyRemote, disconnect, dropRemote, fdwStatus, MAX_TABLES, remoteDbProblems, serverNameOf, type RemoteDb } from '../remote-db.ts';
+import { clientIp, logActivity, type Session } from '../session.ts';
+import { administrator, back, BASE, csrf, flash, input, region, send, shell, type Req } from './ui.ts';
+
+// (sprint 150, AP-7 leftover; APEX: REST Enabled SQL services) Workspace utilities → Remote databases
+// (administrators): another PostgreSQL database's tables as foreign tables in a local schema (src/remote-db.ts).
+//
+//   GET  /builder/remote-databases                 postgres_fdw's state, the remote databases, the form (?edit=<id>, ?new=1)
+//   POST /builder/remote-databases                 save and import (mode=test: import in a transaction that is rolled back)
+//   POST /builder/remote-databases/:rid/refresh    import the tables again (new or changed remote tables)
+//   POST /builder/remote-databases/:rid/delete     drop the local schema and the server
+//   POST /builder/remote-databases/extension       create extension postgres_fdw (a superuser connection)
+
+const PAGE = `${BASE}/remote-databases`;
+const TITLE = 'Remote databases';
+const crumbs: [string, string?][] = [['App Builder', BASE], ['Workspace utilities', `${BASE}/utilities`], [TITLE]];
+const rid = (v: unknown) => (/^\d{1,9}$/.test(String(v ?? '')) ? Number(v) : null);
+const when = (d: Date | string | null) => (d ? new Date(d).toISOString().slice(0, 16).replace('T', ' ') : '');
+const list = (v: unknown) => (Array.isArray(v) ? v.map(String) : typeof v === 'string' && v ? [v] : []);
+
+interface Row extends RemoteDb { id: number; imported_at: Date | null; imported: number | null; error: string | null; updated_by: string }
+
+const rows = async () => (await owner.query<Row>('select * from meta.remote_database order by name')).rows;
+
+function fromBody(b: Record<string, unknown>): RemoteDb {
+  const text = (k: string) => String(b[k] ?? '').trim();
+  return {
+    name: text('name').toUpperCase(), description: text('description') || null, host: text('host'), port: Number(text('port') || '5432'),
+    dbname: text('dbname'), user_name: text('user_name'), remote_schema: text('remote_schema') || 'public', local_schema: text('local_schema').toLowerCase(),
+    tables: text('tables').split(/[,\n]/).map((t) => t.trim()).filter(Boolean), grant_roles: list(b.grant_roles), writable: b.writable === 'true',
+  };
+}
+
+function form(s: Session, x: Partial<Row>, roles: string[]) {
+  const editing = !!x.id;
+  return region(editing ? `Edit ${x.name}` : 'Add a remote database', html`<form method="post" action="${PAGE}" id="rdb-form">${csrf(s)}
+    ${editing ? html`<input type="hidden" name="id" value="${x.id}">` : ''}
+    <div class="form-grid">
+      ${input('name', 'Name', x.name ?? '', { help: 'A static id, e.g. CRM. The server is pgkiln_rdb_<name>.', required: true })}
+      ${input('description', 'Description', x.description ?? '')}
+      ${input('host', 'Host', x.host ?? '', { help: 'As the database server reaches it (it makes the connection, not pgkiln).', required: true })}
+      ${input('port', 'Port', String(x.port ?? 5432), { type: 'number' })}
+      ${input('dbname', 'Database', x.dbname ?? '', { required: true })}
+      ${input('user_name', 'User', x.user_name ?? '', { required: true, help: 'A user of the remote database: what it may read or change there is what the applications may.' })}
+      ${input('password', 'Password', '', { type: 'password', auto: 'new-password', help: editing ? 'Kept by PostgreSQL in the user mapping and never shown; type a new one to replace it.' : 'Kept by PostgreSQL in the user mapping (pgkiln never stores it).' })}
+      ${input('remote_schema', 'Remote schema', x.remote_schema ?? 'public')}
+      ${input('local_schema', 'Local schema', x.local_schema ?? '', { required: true, help: 'A new schema here for the foreign tables, e.g. crm: regions query crm.<table>.' })}
+      ${input('tables', 'Tables', (x.tables ?? []).join(', '), { help: `Remote tables and views to import, separated by commas (at most ${MAX_TABLES}); empty: all of the remote schema's.` })}
+    </div>
+    <fieldset class="prop-group u-mt1"><legend>Grant to the applications' database roles</legend>
+      ${roles.length
+        ? html`<div class="check-list">${roles.map((r) => html`<label><input type="checkbox" name="grant_roles" value="${r}"${(x.grant_roles ?? []).includes(r) ? raw(' checked') : ''}> ${r}</label>`)}</div>`
+        : html`<p class="muted">No application has a database role yet (Application settings → Database role).</p>`}
+      <label class="u-mt1"><input type="checkbox" name="writable" value="true"${x.writable ? raw(' checked') : ''}> Also INSERT, UPDATE and DELETE (else SELECT only)</label>
+    </fieldset>
+    <div class="buttons u-mt1">
+      <button class="btn btn-hot" name="mode" value="save">${editing ? 'Save and import again' : 'Add and import'}</button>
+      <button class="btn" name="mode" value="test">Test the connection</button>
+      <a class="btn" href="${PAGE}">Cancel</a>
+    </div></form>`);
+}
+
+export async function remoteDatabaseRoutes(app: FastifyInstance) {
+  app.get(PAGE, async (req: Req, reply) => {
+    const s = await administrator(req, reply, TITLE);
+    if (!s) return;
+    const q = (req.query ?? {}) as Record<string, string>;
+    const st = await fdwStatus(owner);
+    const all = await rows();
+    const roles = await applicationRoles(owner);
+    const edit = rid(q.edit) ? all.find((r) => r.id === rid(q.edit)) : undefined;
+    const main = html`<h1 class="u-mb1">${TITLE}</h1>
+      <p class="muted u-mt0">Another PostgreSQL database's tables as foreign tables (postgres_fdw) in a local schema: regions, reports, forms and lists
+        of values use them like the application's own tables, as its database role, with the grants given here.</p>
+      ${st.installed
+        ? html`<p class="muted">${icon('check')} postgres_fdw is installed.</p>`
+        : html`<div class="alert alert-info" role="status"><p><strong>postgres_fdw is not installed in this database.</strong></p>
+            ${st.available && st.canCreate
+              ? html`<form method="post" action="${PAGE}/extension">${csrf(s)}<button class="btn">Create extension postgres_fdw</button></form>`
+              : html`<p>${st.available ? 'A superuser runs' : 'The server needs PostgreSQL\'s contrib package, then a superuser runs'} <code>create extension postgres_fdw;</code>
+                  and <code>grant usage on foreign data wrapper postgres_fdw to</code> pgkiln's owner role.</p>`}</div>`}
+      ${region('Remote databases', html`<div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Name</th><th>Connection</th><th>Local schema</th>
+          <th class="num">Tables</th><th>Granted to</th><th>Imported</th><th><span class="sr-only">Actions</span></th></tr></thead><tbody>
+        ${all.length ? all.map((r) => html`<tr>
+            <td data-label="Name"><strong>${r.name}</strong>${r.description ? html`<br><small class="muted">${r.description}</small>` : ''}</td>
+            <td data-label="Connection"><code>${r.user_name}@${r.host}:${r.port}/${r.dbname}</code> (${r.remote_schema})</td>
+            <td data-label="Local schema"><code>${r.local_schema}</code></td>
+            <td class="num" data-label="Tables">${r.imported ?? '—'}</td>
+            <td data-label="Granted to">${r.grant_roles.join(', ') || '—'}${r.writable ? ' (read and write)' : ''}</td>
+            <td data-label="Imported">${when(r.imported_at)}${r.error ? html`<br><span class="wf-problem">${r.error}</span>` : ''}</td>
+            <td data-label="Actions"><a class="btn btn-sm" href="${PAGE}?edit=${r.id}#rdb-form">Edit</a>
+              <form method="post" action="${`${PAGE}/${r.id}/refresh`}" class="inline">${csrf(s)}<button class="btn btn-sm">Import again</button></form>
+              <form method="post" action="${`${PAGE}/${r.id}/delete`}" class="inline">${csrf(s)}<button class="btn btn-sm btn-danger" data-confirm="${`Delete ${r.name}? Its schema ${r.local_schema} and its foreign tables are dropped.`}">Delete</button></form></td>
+          </tr>`) : html`<tr><td colspan="7" class="empty">No remote databases yet.</td></tr>`}
+        </tbody></table></div>`)}
+      ${st.installed ? form(s, edit ?? {}, roles) : ''}`;
+    return send(reply, s, shell(s, TITLE, crumbs, main));
+  });
+
+  app.post(PAGE, async (req: Req, reply) => {
+    const s = await administrator(req, reply, TITLE);
+    if (!s) return;
+    const b = (req.body ?? {}) as Record<string, unknown>;
+    const x = fromBody(b);
+    const editId = rid(b.id);
+    const password = typeof b.password === 'string' && b.password ? b.password : null;
+    const retry = editId ? `${PAGE}?edit=${editId}#rdb-form` : `${PAGE}#rdb-form`;
+    const problems = remoteDbProblems(x, await applicationRoles(owner));
+    if (password && password.length > 1000) problems.push('The password has at most 1000 characters.');
+    if (problems.length) {
+      flash(s, problems.join(' '), 'error');
+      return back(reply, s, retry);
+    }
+    const test = b.mode === 'test';
+    try {
+      const count = await owner.tx(async (c) => {
+        const old = editId ? (await c.query<Row>('select * from meta.remote_database where id = $1 for update', [editId])).rows[0] : undefined;
+        if (editId && !old) throw new Error('No such remote database.');
+        // a renamed one, or a new local schema: the old server and schema go first
+        if (old && (old.name !== x.name || old.local_schema !== x.local_schema)) {
+          if (!password) throw new Error('Renaming it, or moving it to another local schema, needs the password again.');
+          await dropRemote(c, old);
+        }
+        const n = await applyRemote(c, x, password, old && old.name === x.name && old.local_schema === x.local_schema ? old.grant_roles : []);
+        if (test) throw Object.assign(new Error('rolled back'), { tested: n });
+        if (old)
+          await c.query(
+            `update meta.remote_database set name = $2, description = $3, host = $4, port = $5, dbname = $6, user_name = $7, remote_schema = $8, local_schema = $9,
+                    tables = $10, grant_roles = $11, writable = $12, server_name = $13, imported_at = now(), imported = $14, error = null, updated_by = $15, updated_at = now()
+              where id = $1`,
+            [editId, x.name, x.description, x.host, x.port, x.dbname, x.user_name, x.remote_schema, x.local_schema, x.tables, x.grant_roles, x.writable, serverNameOf(x.name), n, s.username]);
+        else
+          await c.query(
+            `insert into meta.remote_database (name, description, host, port, dbname, user_name, remote_schema, local_schema, tables, grant_roles, writable, server_name,
+                                               imported_at, imported, created_by, updated_by)
+             values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $14)`,
+            [x.name, x.description, x.host, x.port, x.dbname, x.user_name, x.remote_schema, x.local_schema, x.tables, x.grant_roles, x.writable, serverNameOf(x.name), n, s.username]);
+        return n;
+      });
+      await disconnect(owner);
+      await logActivity({ username: s.username, event: 'workspace', ip: clientIp(req),
+        detail: `remote database ${x.name} ${editId ? 'changed' : 'added'}: ${x.user_name}@${x.host}:${x.port}/${x.dbname} into ${x.local_schema} (${count} tables)`.slice(0, 1000) });
+      flash(s, `${x.name}: ${count} table(s) imported into ${x.local_schema}.`);
+      return back(reply, s, PAGE);
+    } catch (e) {
+      const tested = (e as { tested?: number }).tested;
+      if (tested !== undefined) {
+        flash(s, `The connection works: ${tested} table(s) would be imported into ${x.local_schema}. Nothing was saved.`);
+      } else {
+        const msg = (e as Error).message.includes('remote_database_name_key') ? `There is a remote database ${x.name} already.`
+          : (e as Error).message.includes('remote_database_local_schema_key') ? `Another remote database uses schema ${x.local_schema}.` : (e as Error).message;
+        flash(s, `${test ? 'The test failed' : 'Nothing was changed'}: ${msg}`, 'error');
+      }
+      return back(reply, s, retry);
+    }
+  });
+
+  app.post(`${PAGE}/:rid/refresh`, async (req: Req, reply) => {
+    const s = await administrator(req, reply, TITLE);
+    if (!s) return;
+    const r = (await rows()).find((x) => x.id === rid(req.params.rid));
+    if (!r) {
+      flash(s, 'No such remote database.', 'error');
+      return back(reply, s, PAGE);
+    }
+    try {
+      const n = await owner.tx(async (c) => applyRemote(c, r, null, r.grant_roles));
+      await owner.query('update meta.remote_database set imported_at = now(), imported = $2, error = null, updated_by = $3, updated_at = now() where id = $1', [r.id, n, s.username]);
+      await logActivity({ username: s.username, event: 'workspace', ip: clientIp(req), detail: `remote database ${r.name} imported again (${n} tables)` });
+      flash(s, `${r.name}: ${n} table(s) imported into ${r.local_schema}.`);
+    } catch (e) {
+      await owner.query('update meta.remote_database set error = $2 where id = $1', [r.id, (e as Error).message.slice(0, 1000)]);
+      flash(s, `${r.name}: nothing was changed: ${(e as Error).message}`, 'error');
+    }
+    return back(reply, s, PAGE);
+  });
+
+  app.post(`${PAGE}/:rid/delete`, async (req: Req, reply) => {
+    const s = await administrator(req, reply, TITLE);
+    if (!s) return;
+    const r = (await rows()).find((x) => x.id === rid(req.params.rid));
+    if (!r) {
+      flash(s, 'No such remote database.', 'error');
+      return back(reply, s, PAGE);
+    }
+    try {
+      await owner.tx(async (c) => {
+        await dropRemote(c, r);
+        await c.query('delete from meta.remote_database where id = $1', [r.id]);
+      });
+      await logActivity({ username: s.username, event: 'workspace', ip: clientIp(req), detail: `remote database ${r.name} deleted (schema ${r.local_schema})` });
+      flash(s, `${r.name} deleted with its schema ${r.local_schema}.`);
+    } catch (e) {
+      flash(s, `${r.name} was not deleted: ${(e as Error).message}`, 'error');
+    }
+    return back(reply, s, PAGE);
+  });
+
+  app.post(`${PAGE}/extension`, async (req: Req, reply) => {
+    const s = await administrator(req, reply, TITLE);
+    if (!s) return;
+    try {
+      await owner.query('create extension if not exists postgres_fdw');
+      await logActivity({ username: s.username, event: 'sql_command', ip: clientIp(req), detail: 'create extension if not exists postgres_fdw' });
+      flash(s, 'The postgres_fdw extension is installed.');
+    } catch (e) {
+      flash(s, `create extension postgres_fdw failed: ${(e as Error).message}`, 'error');
+    }
+    return back(reply, s, PAGE);
+  });
+}

package/src/builder/workflow-migrate.ts

@@ -35,7 +35,8 @@ interface Running {
   version: string;
   state: string;
   current_step: string | null;
-  branches: boolean;
+  /** (144) the steps its open parallel branches are at */
+  branch_steps: string[];
 }
 
 const ID = /^\d{1,9}$/;
@@ -101,7 +102,8 @@ export async function workflowMigrateRoutes(app: FastifyInstance) {
     const running = (
       await owner.query<Running>(
         `select w.id::text, w.title, coalesce(w.version, '1') as version, w.state, w.current_step,
-                exists (select 1 from meta.workflow_branch b where b.workflow_id = w.id and b.state in ('active', 'waiting', 'faulted')) as branches
+                array(select b.current_step from meta.workflow_branch b where b.workflow_id = w.id and b.state in ('active', 'waiting', 'faulted')
+                       and b.current_step is not null order by b.id) as branch_steps
            from meta.workflow w where w.definition_id = $1 and w.state in ('active', 'waiting', 'faulted') order by w.id`,
         [d.id],
       )
@@ -117,16 +119,17 @@ export async function workflowMigrateRoutes(app: FastifyInstance) {
     const targetSteps = (target?.steps ?? []).filter((x) => x && typeof x.name === 'string' && x.type !== 'draft');
     const asked = mappingOf(q);
     // each step a chosen instance is at: the mapping asked for, or the step of the same name
-    const current = [...new Set(chosen.map((w) => w.current_step).filter((x): x is string => !!x && STEP.test(x)))].sort();
+    // (144) and each step its open parallel branches are at
+    const current = [...new Set(chosen.flatMap((w) => [w.current_step, ...w.branch_steps]).filter((x): x is string => !!x && STEP.test(x)))].sort();
     const mapping = Object.fromEntries(current.map((st) => [st, st in asked ? asked[st] : targetSteps.some((x) => x.name === st) ? st : '']));
 
-    let preview: { workflow_id: string; from_version: string | null; from_step: string | null; to_step: string | null; problem: string | null }[] | null = null;
+    let preview: { workflow_id: string; branch: string | null; from_version: string | null; from_step: string | null; to_step: string | null; problem: string | null }[] | null = null;
     let previewError: string | null = null;
     if (q.preview && to) {
       if (!chosen.length) previewError = 'Choose the instances to migrate.';
       else {
         try {
-          preview = (await owner.query('select workflow_id::text, from_version, from_step, to_step, problem from meta.migrate_workflows($1, $2, $3::bigint[], $4, false, $5)',
+          preview = (await owner.query('select workflow_id::text, branch, from_version, from_step, to_step, problem from meta.migrate_workflows($1, $2, $3::bigint[], $4, false, $5)',
             [d.id, to, chosen.map((w) => w.id), mapping, s.username])).rows;
         } catch (e) {
           previewError = (e as Error).message;
@@ -134,6 +137,7 @@ export async function workflowMigrateRoutes(app: FastifyInstance) {
       }
     }
     const ready = !!preview && preview.every((r) => !r.problem);
+    const instances = preview ? new Set(preview.map((r) => r.workflow_id)).size : 0;
     const self = `${url(d)}/migrate`;
     const hidden = html`<input type="hidden" name="to" value="${to ?? ''}">${chosen.map((w) => html`<input type="hidden" name="wf" value="${w.id}">`)}${Object.entries(mapping).map(([k, v]) => html`<input type="hidden" name="${`m.${k}`}" value="${v}">`)}`;
     const body = !running.length
@@ -144,7 +148,8 @@ export async function workflowMigrateRoutes(app: FastifyInstance) {
         : html`${region('Migrate running instances', html`<form method="get" action="${self}">
             <input type="hidden" name="preview" value="1">
             <p class="muted u-mt0">Move running instances to a newer version: each goes on at the step of the new version its current step is mapped to,
-              keeping its variables and the task it waits for. Every instance must have a mapping, or nothing moves. Instances inside parallel branches move once their branches have joined.</p>
+              keeping its variables and the task it waits for. An instance inside parallel branches moves as a whole: its main path at its join and each open
+              branch at its own step, each mapped (a join stays a join). Every position must have a mapping, or nothing moves.</p>
             <div class="form-grid">
               <div class="field"><label class="label" for="wf-to">To version</label>
                 <select id="wf-to" name="to">${targets.map((v) => html`<option value="${v}"${v === to ? raw(' selected') : ''}>${v}${v === d.version ? ' (active)' : ' (inactive)'}</option>`)}</select>
@@ -155,7 +160,7 @@ export async function workflowMigrateRoutes(app: FastifyInstance) {
                 ? html`<div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Migrate</th><th class="num">Id</th><th>Title</th><th>Version</th><th>State</th><th>At</th></tr></thead><tbody>
                     ${older.map((w) => html`<tr><td data-label="Migrate"><input type="checkbox" name="wf" value="${w.id}" id="${`wf-${w.id}`}"${picked.includes(w.id) ? raw(' checked') : ''} aria-label="${`Migrate instance ${w.id}`}"></td>
                       <td class="num" data-label="Id"><label for="${`wf-${w.id}`}">${w.id}</label></td><td data-label="Title">${w.title}</td><td data-label="Version">${w.version}</td>
-                      <td data-label="State">${w.state}</td><td data-label="At">${w.branches ? 'parallel branches' : (w.current_step ?? '—')}</td></tr>`)}
+                      <td data-label="State">${w.state}</td><td data-label="At">${w.current_step ?? '—'}${w.branch_steps.length ? html`<br><small class="muted">branches at ${w.branch_steps.join(', ')}</small>` : ''}</td></tr>`)}
                   </tbody></table></div>`
                 : html`<p class="muted">No running instance is older than version ${to}.</p>`}
             </fieldset>
@@ -169,14 +174,14 @@ export async function workflowMigrateRoutes(app: FastifyInstance) {
           </form>`)}
           ${previewError ? html`<div class="alert alert-error u-mt1" role="alert">${previewError}</div>` : ''}
           ${preview
-            ? region('Preview', html`<div class="table-wrap"><table class="report report-reflow"><thead><tr><th class="num">Id</th><th>From</th><th>To</th><th>Result</th></tr></thead><tbody>
-                  ${preview.map((r) => html`<tr><td class="num" data-label="Id">${r.workflow_id}</td><td data-label="From">${r.from_version ? `${r.from_version}: ${r.from_step ?? '—'}` : '—'}</td>
+            ? region('Preview', html`<div class="table-wrap"><table class="report report-reflow"><thead><tr><th class="num">Id</th><th>Path</th><th>From</th><th>To</th><th>Result</th></tr></thead><tbody>
+                  ${preview.map((r) => html`<tr><td class="num" data-label="Id">${r.workflow_id}</td><td data-label="Path">${r.branch ? `branch ${r.branch}` : 'main'}</td><td data-label="From">${r.from_version ? `${r.from_version}: ${r.from_step ?? '—'}` : '—'}</td>
                     <td data-label="To">${r.to_step ? `${to}: ${r.to_step}` : '—'}</td><td data-label="Result">${r.problem ? html`<span class="wf-problem">${r.problem}</span>` : html`<span class="tag tag-info">ready</span>`}</td></tr>`)}
                 </tbody></table></div>
                 ${ready
                   ? html`<form method="post" action="${self}" class="u-mt1">${csrf(s)}${hidden}
-                      <button class="btn btn-hot" data-confirm="${`Migrate ${preview.length} instance(s) to version ${to}?`}">Migrate ${preview.length} instance(s)</button></form>`
-                  : html`<p class="wf-problem">Map a step for every instance (or leave the others out) and preview again: nothing is migrated while one can't move.</p>`}`)
+                      <button class="btn btn-hot" data-confirm="${`Migrate ${instances} instance(s) to version ${to}?`}">Migrate ${instances} instance(s)</button></form>`
+                  : html`<p class="wf-problem">Map a step for every instance and branch (or leave the instances out) and preview again: nothing is migrated while one can't move.</p>`}`)
             : ''}
           ${target ? region(`Version ${to}`, html`<div class="wf-wrap">${workflowDiagram(target.steps, { id: 'target' })}</div>`) : ''}`;
     const main = html`${appHeader(a, 'shared', s)}<div class="ide-body">${body}</div>`;
@@ -198,8 +203,9 @@ export async function workflowMigrateRoutes(app: FastifyInstance) {
       if (!to) throw new Error('Choose the version to migrate to.');
       const rows = await owner.tx(async (c) =>
         (await c.query('select workflow_id::text, from_step, to_step from meta.migrate_workflows($1, $2, $3::bigint[], $4, true, $5)', [d.id, to, ids, mapping, s.username])).rows);
-      logActivity({ appId: d.app_id, username: s.username, event: 'workflow', ip: clientIp(req), detail: `${d.name}: ${rows.length} instance(s) migrated to version ${to}` });
-      flash(s, `${rows.length} instance(s) of ${d.name} migrated to version ${to}.`);
+      const moved = new Set(rows.map((r) => r.workflow_id)).size;
+      logActivity({ appId: d.app_id, username: s.username, event: 'workflow', ip: clientIp(req), detail: `${d.name}: ${moved} instance(s) migrated to version ${to}` });
+      flash(s, `${moved} instance(s) of ${d.name} migrated to version ${to}.`);
       return back(reply, s, `${BASE}/apps/${d.app_id}/shared?c=workflow_definition-${d.id}`);
     } catch (e) {
       flash(s, (e as Error).message, 'error');

package/src/remote-db.ts

@@ -0,0 +1,145 @@
+import pg from 'pg';
+
+// (sprint 150, AP-7 leftover; APEX: REST Enabled SQL, regions on another database) Remote databases: another
+// PostgreSQL database's tables as foreign tables (postgres_fdw) in a local schema of their own, which applications
+// query like their own tables (as their database role, in their transaction, with the grants given here). See
+// migration 145 for the table; Workspace utilities → Remote databases (src/builder/remote-databases.ts) for the page.
+//
+// Everything runs on the owner connection in the caller's transaction:
+//   create (or alter) server pgkiln_rdb_<name> foreign data wrapper postgres_fdw options (host, port, dbname)
+//   create (or alter) user mapping for public … options (user, password)   -- the password only lives here
+//   create schema <local> (marked as this remote database's by its comment), drop its foreign tables,
+//   import foreign schema <remote> [limit to (…)] from server … into <local>
+//   revoke from public; grant usage and select (with writable: insert, update, delete) to the chosen app roles
+// Names reach SQL quoted (escapeIdentifier) or as literals (escapeLiteral), after the checks below.
+
+export interface RemoteDb {
+  id?: number;
+  name: string;
+  description: string | null;
+  host: string;
+  port: number;
+  dbname: string;
+  user_name: string;
+  remote_schema: string;
+  local_schema: string;
+  tables: string[];
+  grant_roles: string[];
+  writable: boolean;
+}
+
+type Q = { query: (sql: string, params?: unknown[]) => Promise<pg.QueryResult<any>> };
+
+const id = pg.escapeIdentifier;
+const lit = pg.escapeLiteral;
+export const NAME = /^[A-Z][A-Z0-9_]{0,29}$/;
+const LOCAL = /^[a-z_][a-z0-9_]{0,62}$/;
+// a host name, an IPv4 or an IPv6 address (no spaces, quotes or option syntax)
+const HOST = /^(?:[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?|\[?[0-9A-Fa-f:.]{2,45}\]?)$/;
+// a remote name (database, user, schema, table): any printable text without control characters, at most 63
+const REMOTE = /^[^\x00-\x1f\x7f]{1,63}$/;
+const RESERVED = /^(pg_.*|information_schema|meta|public|pgkiln.*)$/;
+export const MAX_TABLES = 200;
+
+export const serverNameOf = (name: string) => `pgkiln_rdb_${name.toLowerCase()}`;
+const marker = (name: string) => `pgkiln remote database ${name}`;
+
+/** What is wrong with a remote database's settings (empty: nothing). `roles`: the roles that may be granted. */
+export function remoteDbProblems(x: RemoteDb, roles: string[]): string[] {
+  const p: string[] = [];
+  if (!NAME.test(x.name)) p.push('The name is an upper case static id (letters, digits, _; at most 30), e.g. CRM.');
+  if (!HOST.test(x.host)) p.push('The host is a host name or an IP address.');
+  if (!Number.isInteger(x.port) || x.port < 1 || x.port > 65535) p.push('The port is a number from 1 to 65535.');
+  if (!REMOTE.test(x.dbname)) p.push('The database is a name of 1 to 63 characters.');
+  if (!REMOTE.test(x.user_name)) p.push('The user is a name of 1 to 63 characters.');
+  if (!REMOTE.test(x.remote_schema)) p.push('The remote schema is a name of 1 to 63 characters.');
+  if (!LOCAL.test(x.local_schema) || RESERVED.test(x.local_schema)) p.push('The local schema is a new lower case name (not public, meta or pg_…), e.g. crm.');
+  if (x.tables.length > MAX_TABLES) p.push(`At most ${MAX_TABLES} tables.`);
+  for (const t of x.tables) if (!REMOTE.test(t)) p.push(`The table name "${t.slice(0, 70)}" is not a name of 1 to 63 characters.`);
+  for (const r of x.grant_roles) if (!roles.includes(r)) p.push(`${r} is not the database role of an application.`);
+  if (x.description && x.description.length > 1000) p.push('The description has at most 1000 characters.');
+  return p;
+}
+
+export interface FdwStatus { installed: boolean; available: boolean; canCreate: boolean }
+
+/** Whether postgres_fdw is installed in this database (and whether the connection could create it). */
+export async function fdwStatus(db: Q): Promise<FdwStatus> {
+  const r = (await db.query(
+    `select exists (select 1 from pg_catalog.pg_extension where extname = 'postgres_fdw') as installed,
+            exists (select 1 from pg_catalog.pg_available_extensions where name = 'postgres_fdw') as available,
+            (select rolsuper from pg_catalog.pg_roles where rolname = current_user) as superuser`)).rows[0];
+  return { installed: r.installed, available: r.available, canCreate: !!r.superuser };
+}
+
+/** The database roles of the applications (the roles a remote database's tables may be granted to). */
+export async function applicationRoles(db: Q): Promise<string[]> {
+  return (await db.query(
+    `select distinct a.db_role as r from meta.app a where a.db_role is not null
+        and exists (select 1 from pg_catalog.pg_roles x where x.rolname = a.db_role) order by 1`)).rows.map((x) => x.r);
+}
+
+/**
+ * Make (or change) the server, user mapping and local schema, and import the tables again: the caller's transaction
+ * (nothing stays when it fails or is rolled back). `password`: null keeps the mapping's password (it must exist
+ * then); `before`: the roles granted until now (their grants go first). Returns the number of foreign tables.
+ */
+export async function applyRemote(c: Q, x: RemoteDb, password: string | null, before: string[] = []): Promise<number> {
+  if (!(await fdwStatus(c)).installed) throw new Error('The postgres_fdw extension is not installed in this database (Workspace utilities → Remote databases).');
+  const server = serverNameOf(x.name);
+  const exists = (await c.query('select 1 from pg_catalog.pg_foreign_server where srvname = $1', [server])).rowCount;
+  const opts: [string, string][] = [['host', x.host], ['port', String(x.port)], ['dbname', x.dbname]];
+  if (exists) await c.query(`alter server ${id(server)} options (${opts.map(([k, v]) => `set ${k} ${lit(v)}`).join(', ')})`);
+  else await c.query(`create server ${id(server)} foreign data wrapper postgres_fdw options (${opts.map(([k, v]) => `${k} ${lit(v)}`).join(', ')})`);
+  const mapped = (await c.query(`select 1 from pg_catalog.pg_user_mappings where srvname = $1 and usename = 'public'`, [server])).rowCount;
+  if (!mapped) {
+    if (!password) throw new Error('A password is needed the first time (postgres_fdw connects with the remote user and its password).');
+    await c.query(`create user mapping for public server ${id(server)} options (user ${lit(x.user_name)}, password ${lit(password)})`);
+  } else await c.query(`alter user mapping for public server ${id(server)} options (set user ${lit(x.user_name)}${password ? `, set password ${lit(password)}` : ''})`);
+
+  // the local schema: new, or this remote database's own
+  const schema = (await c.query(
+    `select pg_catalog.obj_description(n.oid, 'pg_namespace') as comment from pg_catalog.pg_namespace n where n.nspname = $1`, [x.local_schema])).rows[0];
+  if (schema && schema.comment !== marker(x.name))
+    throw new Error(`Schema ${x.local_schema} exists and is not remote database ${x.name}'s: choose another local schema.`);
+  if (!schema) {
+    await c.query(`create schema ${id(x.local_schema)}`);
+    await c.query(`comment on schema ${id(x.local_schema)} is ${lit(marker(x.name))}`);
+  }
+  // the foreign tables again: drop the old ones (a view that uses one stops this, and says which)
+  const old = (await c.query(
+    `select c.relname as t from pg_catalog.pg_class c join pg_catalog.pg_namespace n on n.oid = c.relnamespace
+      where n.nspname = $1 and c.relkind = 'f'`, [x.local_schema])).rows;
+  for (const o of old) await c.query(`drop foreign table ${id(x.local_schema)}.${id(o.t)}`);
+  await c.query(`import foreign schema ${id(x.remote_schema)}${x.tables.length ? ` limit to (${x.tables.map(id).join(', ')})` : ''}
+                 from server ${id(server)} into ${id(x.local_schema)}`);
+  const count = Number((await c.query(
+    `select count(*) as n from pg_catalog.pg_class c join pg_catalog.pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind = 'f'`,
+    [x.local_schema])).rows[0].n);
+  // grants: only the chosen application roles, nobody else
+  await c.query(`revoke all on all tables in schema ${id(x.local_schema)} from public`);
+  await c.query(`revoke all on schema ${id(x.local_schema)} from public`);
+  for (const r of new Set([...before, ...x.grant_roles])) {
+    if (!(await c.query('select 1 from pg_catalog.pg_roles where rolname = $1', [r])).rowCount) continue;
+    await c.query(`revoke all on all tables in schema ${id(x.local_schema)} from ${id(r)}`);
+    await c.query(`revoke all on schema ${id(x.local_schema)} from ${id(r)}`);
+  }
+  for (const r of x.grant_roles) {
+    await c.query(`grant usage on schema ${id(x.local_schema)} to ${id(r)}`);
+    await c.query(`grant ${x.writable ? 'select, insert, update, delete' : 'select'} on all tables in schema ${id(x.local_schema)} to ${id(r)}`);
+  }
+  return count;
+}
+
+/** Drop the local schema (when it is this remote database's) and the server with its user mapping. */
+export async function dropRemote(c: Q, x: Pick<RemoteDb, 'name' | 'local_schema'>) {
+  const schema = (await c.query(
+    `select pg_catalog.obj_description(n.oid, 'pg_namespace') as comment from pg_catalog.pg_namespace n where n.nspname = $1`, [x.local_schema])).rows[0];
+  if (schema && schema.comment === marker(x.name)) await c.query(`drop schema ${id(x.local_schema)} cascade`);
+  await c.query(`drop server if exists ${id(serverNameOf(x.name))} cascade`);
+}
+
+/** postgres_fdw keeps connections per session: drop this connection's, so the next query uses new settings. */
+export async function disconnect(c: Q) {
+  await c.query(`select postgres_fdw_disconnect_all()`).catch(() => undefined);
+}