pgkiln 0.70.0 → 0.72.0

package/src/builder/telemetry.ts

@@ -0,0 +1,206 @@
+import type { FastifyInstance, FastifyReply } from 'fastify';
+import { owner } from '../db.ts';
+import { html, raw, type Raw } from '../html.ts';
+import { newToken } from '../security.ts';
+import { clientIp, logActivity, type Session } from '../session.ts';
+import {
+  instanceTelemetry, refreshTelemetry, saveInstanceTelemetry, saveWorkspaceTelemetry, setMetricsToken, workspaceTelemetry,
+  type InstanceForm, type Origin, type WorkspaceForm,
+} from '../telemetry/config.ts';
+import { destinationStatus, sendTest } from '../telemetry/export.ts';
+import { isAdmin } from './locks.ts';
+import { back, BASE, csrf, developer, flash, input, region, select, send, shell, type Req } from './ui.ts';
+import { workspaceAdmin } from './workspaces.ts';
+
+// (sprint 139, AP-6; APEX: Workspace Utilities → OpenTelemetry) Workspace
+// utilities → OpenTelemetry (administrators): the instance's collector, what
+// it receives, GET /metrics, and each workspace's choice; a workspace's
+// administrators switch its applications off or add a collector of their own
+// on /builder/workspaces/:id/telemetry. Headers (an API key) are write-only:
+// stored encrypted, only their names are shown.
+
+const crumbs: [string, string?][] = [['App Builder', BASE], ['Workspace utilities', `${BASE}/utilities`], ['OpenTelemetry']];
+const ORIGIN: Record<Origin, string> = { builder: 'set here', environment: 'environment variable', default: 'default' };
+
+const check = (name: string, label: string, on: boolean, help?: string) =>
+  html`<div class="field"><label class="check"><input type="checkbox" name="${name}" value="true"${on ? raw(' checked') : ''}> ${label}</label>${help ? html`<small class="help">${help}</small>` : ''}</div>`;
+
+const headersField = (names: string[], help: string) => html`<div class="field"><label class="label" for="f_headers">Headers</label>
+  <textarea id="f_headers" name="headers" rows="2" maxlength="4000" spellcheck="false" autocomplete="off" placeholder="${names.length ? `${names.join(', ')} set: type new headers to replace them` : 'e.g. x-honeycomb-team=…, Authorization=Bearer …'}"></textarea>
+  <small class="help">${help}${names.length ? html` Set now: <b>${names.join(', ')}</b> (values hidden).` : ''}</small>
+  ${names.length ? html`<label class="check"><input type="checkbox" name="remove_headers" value="true"> Remove the headers</label>` : ''}</div>`;
+
+const when = (ms: number | null) => (ms ? new Date(ms).toISOString().replace('T', ' ').slice(0, 19) + ' UTC' : '—');
+
+function statusCell(key: string): Raw {
+  const st = destinationStatus(key);
+  return html`${st.sent} sent${st.dropped ? html`, <span class="otel-problem">${st.dropped} dropped</span>` : ''} · last success ${when(st.lastOkAt)}
+    ${st.lastError ? html`<br><span class="otel-problem">Last problem (${when(st.lastErrorAt)}): ${st.lastError}</span>` : ''}`;
+}
+
+async function adminOnly(s: Session, reply: FastifyReply) {
+  if (await isAdmin(s.username)) return true;
+  await send(reply.code(403), s, shell(s, 'OpenTelemetry', crumbs, html`<h1 class="u-mb1">OpenTelemetry</h1><div class="alert alert-error" role="alert">Only administrators set up OpenTelemetry for the installation.</div>`));
+  return false;
+}
+
+export async function telemetryBuilderRoutes(app: FastifyInstance) {
+  app.get(`${BASE}/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    await refreshTelemetry(true);
+    const t = instanceTelemetry();
+    const token = typeof s.state.__METRICS_TOKEN === 'string' ? s.state.__METRICS_TOKEN : null;
+    if (token) {
+      delete s.state.__METRICS_TOKEN;
+      const { saveState } = await import('../session.ts');
+      await saveState(s);
+    }
+    const workspaces = (await owner.query<{ id: number; name: string; telemetry: string; telemetry_endpoint: string | null; apps: number }>(
+      `select w.id, w.name, w.telemetry, w.telemetry_endpoint, (select count(*) from meta.app a where meta.app_workspace(a.id) = w.id)::int as apps
+         from meta.workspace w order by w.name`)).rows;
+    const from = (k: string) => html`<span class="muted">(${ORIGIN[t.origin[k] ?? 'default']})</span>`;
+    const pub = (process.env.PUBLIC_URL ?? '').replace(/\/+$/, '');
+    const main = html`<h1 class="u-mb1">OpenTelemetry</h1>
+      <p class="muted u-mt0">Send traces, logs and metrics of the applications to your own monitoring (Grafana, Jaeger, Datadog, Honeycomb, OCI APM,
+        or anything that speaks OTLP) through an OpenTelemetry Collector. A trace per page request, with a span for each SQL statement, REST data source call,
+        AI request and tool call; mail and workflow steps as traces of their own. pgkiln sends OTLP over HTTP as JSON. The standard <code>OTEL_*</code>
+        environment variables work too; a value set here wins. Chapter 1 of the guide (Monitoring with OpenTelemetry) explains every choice.</p>
+      ${t.protocolWarning ? html`<div class="alert alert-warning" role="status">${t.protocolWarning}</div>` : ''}
+      <div class="columns wide-left">
+        ${region('Collector', html`<form method="post" action="${BASE}/telemetry">${csrf(s)}
+          ${check('enabled', 'Send to this collector', t.enabled, 'Off: nothing leaves the server (workspaces with a collector of their own still send to it).')}
+          <div class="form-grid">
+            ${input('endpoint', 'Collector URL', t.origin.otel_endpoint === 'builder' ? t.endpoint : '', { placeholder: t.endpoint || 'http://otel-collector:4318', help: html`The OTLP/HTTP base address; /v1/traces, /v1/logs and /v1/metrics are added. In effect: <b>${t.endpoint || 'none'}</b> ${from('otel_endpoint')} (<code>OTEL_EXPORTER_OTLP_ENDPOINT</code>).` as unknown as string })}
+            ${input('service_name', 'Service name', t.origin.otel_service_name === 'builder' ? t.serviceName : '', { placeholder: t.serviceName, help: `service.name of everything sent (OTEL_SERVICE_NAME). In effect: ${t.serviceName}.` })}
+            ${input('sample', 'Sample ratio', t.origin.otel_sample === 'builder' ? String(t.sample) : '', { placeholder: String(t.sample), help: 'Share of traces sent, 0 to 1 (OTEL_TRACES_SAMPLER_ARG). A trace with an error is always sent. A caller\'s traceparent decides for its trace.' })}
+            ${select('user', 'User', t.user, [['hashed', 'Hashed (the same user gives the same value, the name is not sent)'], ['name', 'User name'], ['none', 'Not sent']], 'enduser.id on the page span and on logs.')}
+          </div>
+          ${headersField(t.headerNames, 'Sent with every request to the collector, name=value, one per line or comma separated (OTEL_EXPORTER_OTLP_HEADERS). Stored encrypted.')}
+          <fieldset class="field"><legend class="label">Send</legend><div class="radio-group">
+            ${check('traces', 'Traces', t.traces)}${check('logs', 'Logs: errors, refused access and failed sign-ins', t.logs)}${check('metrics', `Metrics, every ${Math.round(t.intervalMs / 1000)} seconds`, t.metrics)}
+          </div></fieldset>
+          ${check('values', 'Include values', t.values, 'SQL statements with their literals, bind values, query strings and AI tool arguments. Off: literals become ?, values are left out and error messages have quoted values removed.')}
+          ${check('builder', 'Trace builder requests too', t.builder, 'Applications only by default.')}
+          ${check('browser', 'Browser spans', t.browser, 'Pages report their load time and Ajax calls to pgkiln, which adds them to the page\'s trace. The browser never sees the collector or its headers.')}
+          ${check('prometheus', 'Prometheus endpoint /metrics', t.prometheus, 'For administrators signed in to the builder, or a scraper with the metrics token below (PGKILN_METRICS, PGKILN_METRICS_TOKEN).')}
+          <div class="buttons"><button class="btn btn-hot">Save</button></div></form>
+          <form method="post" action="${BASE}/telemetry/test" class="u-mt1">${csrf(s)}<button class="btn">Send a test span</button></form>`)}
+        ${region('Status', html`<p class="muted u-mt0">Since this server started. Each server of the installation sends its own.</p>
+          <div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Collector</th><th>Sending</th></tr></thead><tbody>
+            <tr><td data-label="Collector">This instance's${t.enabled ? '' : html` <span class="muted">(off)</span>`}</td><td data-label="Sending">${statusCell('instance')}</td></tr>
+            ${workspaces.filter((w) => w.telemetry === 'own').map((w) => html`<tr><td data-label="Collector">Workspace ${w.name}</td><td data-label="Sending">${statusCell(`ws:${w.id}`)}</td></tr>`)}
+          </tbody></table></div>`)}
+        ${region('Prometheus', html`<p class="muted u-mt0">${t.prometheus ? html`<code>GET ${pub || ''}/metrics</code> answers with the counters, durations and queue lengths in the Prometheus text format.` : 'Off: switch it on above.'}
+            A scraper sends <code>Authorization: Bearer &lt;token&gt;</code>. ${t.prometheusTokenHash ? (t.origin.metrics_token_hash === 'builder' || !process.env.PGKILN_METRICS_TOKEN ? 'A token is set.' : 'The token is PGKILN_METRICS_TOKEN.') : 'No token is set.'}</p>
+          ${token ? html`<div class="alert alert-success" role="status">The new token (shown once): <code class="cell-break">${token}</code></div>` : ''}
+          <form method="post" action="${BASE}/telemetry/token" class="buttons">${csrf(s)}<button class="btn" name="action" value="new">${t.prometheusTokenHash ? 'Replace the token' : 'Make a token'}</button>
+            ${t.prometheusTokenHash ? html`<button class="btn btn-danger" name="action" value="remove" data-confirm="Remove the metrics token? Scrapers that use it get 401.">Remove the token</button>` : ''}</form>`)}
+        ${region('Workspaces', html`<p class="muted u-mt0">A workspace's administrators may switch its applications off, or send them to a collector of their own as well.</p>
+          <div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Workspace</th><th>Applications</th><th>Sends</th></tr></thead><tbody>
+          ${workspaces.length ? workspaces.map((w) => html`<tr><td data-label="Workspace"><a href="${BASE}/workspaces/${w.id}/telemetry">${w.name}</a></td><td data-label="Applications" class="num">${w.apps}</td>
+            <td data-label="Sends" class="cell-break">${w.telemetry === 'off' ? 'nothing' : w.telemetry === 'own' ? html`to the instance's collector and ${w.telemetry_endpoint}` : 'to the instance\'s collector'}</td></tr>`)
+            : html`<tr><td colspan="3" class="empty">No workspaces.</td></tr>`}</tbody></table></div>`)}
+      </div>`;
+    return send(reply, s, shell(s, 'OpenTelemetry', crumbs, main));
+  });
+
+  app.post(`${BASE}/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    const problem = await saveInstanceTelemetry((req.body ?? {}) as InstanceForm);
+    if (!problem) {
+      const t = instanceTelemetry();
+      await logActivity({ username: s.username, event: 'instance_settings', ip: clientIp(req),
+        detail: `OpenTelemetry: ${t.enabled ? `on, ${t.endpoint}` : 'off'}, sample ${t.sample}, user ${t.user}, values ${t.values}, browser ${t.browser}, /metrics ${t.prometheus}` });
+    }
+    flash(s, problem ?? 'OpenTelemetry settings saved.', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/telemetry`);
+  });
+
+  app.post(`${BASE}/telemetry/test`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    const problem = await sendTest(null);
+    flash(s, problem ? `The collector did not take the test span: ${problem}` : 'The collector took the test span ("pgkiln test span").', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/telemetry`);
+  });
+
+  app.post(`${BASE}/telemetry/token`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    const action = (req.body as { action?: string } | undefined)?.action;
+    if (action === 'new') {
+      const token = `pgkm_${newToken()}`;
+      await setMetricsToken(token);
+      s.state.__METRICS_TOKEN = token;
+      await logActivity({ username: s.username, event: 'instance_settings', ip: clientIp(req), detail: 'metrics token made' });
+    } else if (action === 'remove') {
+      await setMetricsToken(null);
+      await logActivity({ username: s.username, event: 'instance_settings', ip: clientIp(req), detail: 'metrics token removed' });
+      flash(s, 'The metrics token is removed.');
+    }
+    return back(reply, s, `${BASE}/telemetry`);
+  });
+
+  // ---------------------------------------------------------------- a workspace's own
+
+  const wsAllowed = async (s: Session, reply: FastifyReply, id: string) => {
+    if (await workspaceAdmin(s.username, id)) return true;
+    await send(reply.code(403), s, shell(s, 'OpenTelemetry', [['Administration'], ['OpenTelemetry']],
+      html`<h1 class="u-mb1">OpenTelemetry</h1><div class="alert alert-error" role="alert">Only the workspace's administrators change its OpenTelemetry setting.</div>`, 'users'));
+    return false;
+  };
+
+  app.get(`${BASE}/workspaces/:id(^\\d{1,9}$)/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await wsAllowed(s, reply, req.params.id))) return;
+    const w = await owner.one<{ id: number; name: string; telemetry: string; telemetry_endpoint: string | null; telemetry_values: boolean; has_headers: boolean }>(
+      'select id, name, telemetry, telemetry_endpoint, telemetry_values, telemetry_headers_enc is not null as has_headers from meta.workspace where id = $1', [req.params.id]);
+    if (!w) return reply.code(404).send('Not found');
+    await refreshTelemetry(true);
+    const t = instanceTelemetry();
+    const names = Object.keys(workspaceTelemetry(w.id)?.headers ?? {});
+    const radio = (v: string, label: string, help: string) => html`<label class="check"><input type="radio" name="mode" value="${v}"${w.telemetry === v ? raw(' checked') : ''}> ${label} <span class="muted small">${help}</span></label>`;
+    const main = html`<div class="title-row"><h1>OpenTelemetry · ${w.name}</h1></div>
+      <p class="muted u-mt0">Traces and logs of this workspace's applications. ${t.enabled ? 'The instance sends them to its collector.' : 'The instance sends nothing at the moment.'}
+        A collector of the workspace's own is reached through the installation's allow-list and this workspace's <a href="${BASE}/network?ws=${w.id}">network access</a> list.</p>
+      <div class="columns wide-left">
+        ${region('Setting', html`<form method="post" action="${BASE}/workspaces/${w.id}/telemetry">${csrf(s)}
+          <fieldset class="field"><legend class="label">The applications send</legend><div class="radio-group">
+            ${radio('instance', 'As the instance does', '(default)')}
+            ${radio('off', 'Nothing', 'not even to the instance\'s collector')}
+            ${radio('own', 'Also to our own collector', '')}
+          </div></fieldset>
+          <div class="form-grid">${input('endpoint', 'Own collector URL', w.telemetry_endpoint ?? '', { placeholder: 'https://otlp.example.com', help: 'The OTLP/HTTP base address; /v1/traces and /v1/logs are added.' })}</div>
+          ${headersField(names.length ? names : w.has_headers ? ['(unreadable)'] : [], 'Sent with every request to the own collector (an API key). Stored encrypted.')}
+          ${check('values', 'Include values for the own collector', w.telemetry_values, 'SQL statements with their literals, bind values and query strings.')}
+          <div class="buttons"><button class="btn btn-hot">Save</button></div></form>
+          ${w.telemetry === 'own' ? html`<form method="post" action="${BASE}/workspaces/${w.id}/telemetry/test" class="u-mt1">${csrf(s)}<button class="btn">Send a test span</button></form>` : ''}`)}
+        ${w.telemetry === 'own' ? region('Status', html`<p class="u-mt0">${statusCell(`ws:${w.id}`)}</p>`) : ''}
+      </div>`;
+    return send(reply, s, shell(s, `OpenTelemetry · ${w.name}`, [['Administration'], [w.name, `${BASE}/workspaces/${w.id}`], ['OpenTelemetry']], main, 'users'));
+  });
+
+  app.post(`${BASE}/workspaces/:id(^\\d{1,9}$)/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await wsAllowed(s, reply, req.params.id))) return;
+    const id = Number(req.params.id);
+    if (!(await owner.one('select 1 from meta.workspace where id = $1', [id]))) return reply.code(404).send('Not found');
+    const f = (req.body ?? {}) as WorkspaceForm;
+    const problem = await saveWorkspaceTelemetry(id, f);
+    if (!problem)
+      await logActivity({ username: s.username, event: 'workspace', ip: clientIp(req), detail: `workspace ${id} OpenTelemetry: ${f.mode}${f.mode === 'own' ? ` ${f.endpoint}` : ''}` });
+    flash(s, problem ?? 'Saved.', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/workspaces/${id}/telemetry`);
+  });
+
+  app.post(`${BASE}/workspaces/:id(^\\d{1,9}$)/telemetry/test`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await wsAllowed(s, reply, req.params.id))) return;
+    const id = Number(req.params.id);
+    const problem = workspaceTelemetry(id)?.mode === 'own' ? await sendTest(id) : 'The workspace has no collector of its own.';
+    flash(s, problem ? `The collector did not take the test span: ${problem}` : 'The collector took the test span ("pgkiln test span").', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/workspaces/${id}/telemetry`);
+  });
+}

package/src/builder/ui-defaults.ts

@@ -0,0 +1,161 @@
+import type { FastifyInstance } from 'fastify';
+import { owner } from '../db.ts';
+import { html, raw } from '../html.ts';
+import { icon } from '../icons.ts';
+import { back, BASE, csrf, developer, flash, region, send, shell, type Body, type Req } from './ui.ts';
+import { usableSchemas, workshopDb } from './workspaces.ts';
+
+// (sprint 138, AP-5) UI defaults (APEX: Attribute Dictionary / Table UI
+// Defaults): per table column a label, format mask, help text, alignment and
+// list of values, in meta.ui_default (migration 138). The page wizards and the
+// Create Application wizard take the label from meta.wizard_column_label (a UI
+// default wins); a form item a wizard makes also gets the mask, help and list
+// of values (the meta.item trigger of migration 138). "Seed from comments" fills
+// the columns without a UI default from their COMMENT ON: a short comment (60
+// characters at most) as the label, a longer one as the help text.
+//
+// The tables are those the workspace's SQL Workshop connection can see
+// (workshopDb: an isolated workspace's own schemas); a table or column name
+// from a request is only used after it was found in that catalog.
+
+export const ALIGNMENTS = ['', 'start', 'center', 'end'] as const;
+/** Format masks: numbers (999G990D00, FML…) and dates (DD.MM.YYYY HH24:MI); the database checks the same. */
+export const MASK = /^[A-Za-z0-9GDLSCFM.,:/ -]{1,60}$/;
+const MAX_COLUMNS = 400;
+
+interface Column {
+  name: string;
+  type: string;
+  comment: string | null;
+  wizard: string;
+  d: { label: string | null; format_mask: string | null; help: string | null; alignment: string | null; lov: string | null; updated_by: string | null; updated_at: string | null } | null;
+}
+
+/** A table the session's catalog can see: [schema, table], or null. */
+async function findTable(s: Parameters<typeof workshopDb>[0], schema: unknown, table: unknown) {
+  if (typeof schema !== 'string' || typeof table !== 'string' || !schema || !table) return null;
+  const db = await workshopDb(s);
+  if (!(await usableSchemas(db)).includes(schema)) return null;
+  const r = await db.one<{ oid: number }>(
+    `select c.oid::int as oid from pg_class c join pg_namespace n on n.oid = c.relnamespace
+      where n.nspname = $1 and c.relname = $2 and c.relkind in ('r', 'p', 'v', 'm', 'f')`, [schema, table]);
+  return r ? { schema, table, oid: r.oid } : null;
+}
+
+async function columnsOf(oid: number, schema: string, table: string): Promise<Column[]> {
+  const cols = (await owner.query(
+    `select a.attname as name, format_type(a.atttypid, a.atttypmod) as type, col_description(a.attrelid, a.attnum) as comment,
+            meta.wizard_column_label(a.attrelid::regclass, a.attname) as wizard
+       from pg_attribute a where a.attrelid = $1 and a.attnum > 0 and not a.attisdropped order by a.attnum limit ${MAX_COLUMNS}`, [oid])).rows;
+  const defaults = new Map((await owner.query(
+    `select column_name, label, format_mask, help, alignment, lov, updated_by, updated_at::text from meta.ui_default where schema_name = $1 and table_name = $2`,
+    [schema, table])).rows.map((r) => [r.column_name, r]));
+  return cols.map((c) => ({ ...c, d: defaults.get(c.name) ?? null }));
+}
+
+const trim = (v: unknown, max: number) => (typeof v === 'string' && v.trim() ? v.trim().slice(0, max) : null);
+
+export async function uiDefaultRoutes(app: FastifyInstance) {
+  app.get(`${BASE}/ui-defaults`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s) return;
+    const db = await workshopDb(s);
+    const schemas = await usableSchemas(db);
+    const q = req.query ?? {};
+    const schema = typeof q.schema === 'string' && schemas.includes(q.schema) ? q.schema : '';
+    const tables = schema
+      ? (await db.query<{ name: string; n: number }>(
+          `select c.relname as name, (select count(*) from meta.ui_default d where d.schema_name = $1 and d.table_name = c.relname)::int as n
+             from pg_class c join pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind in ('r', 'p', 'v', 'm', 'f') and not c.relispartition order by 1`,
+          [schema])).rows
+      : [];
+    const t = schema ? await findTable(s, schema, q.table) : null;
+    const counts = (await owner.query<{ schema_name: string; n: number }>('select schema_name, count(*)::int as n from meta.ui_default group by 1')).rows;
+    const pick = html`<form method="get" class="util-filters">
+        <div class="field"><label class="label" for="f_schema">Schema</label><select id="f_schema" name="schema"><option value="">Choose a schema</option>
+          ${schemas.map((n) => html`<option value="${n}"${n === schema ? raw(' selected') : ''}>${n}${counts.find((c) => c.schema_name === n) ? ` (${counts.find((c) => c.schema_name === n)!.n})` : ''}</option>`)}</select></div>
+        ${schema ? html`<div class="field"><label class="label" for="f_table">Table or view</label><select id="f_table" name="table"><option value="">Choose a table</option>
+          ${tables.map((x) => html`<option value="${x.name}"${x.name === t?.table ? raw(' selected') : ''}>${x.name}${x.n ? ` (${x.n})` : ''}</option>`)}</select></div>` : ''}
+        <div class="buttons"><button class="btn btn-sm">${icon('table')} Show</button></div></form>`;
+    let body = html`${pick}`;
+    if (t) {
+      const cols = await columnsOf(t.oid, t.schema, t.table);
+      body = html`${pick}
+        <form method="post" action="${BASE}/ui-defaults">${csrf(s)}<input type="hidden" name="schema" value="${t.schema}"><input type="hidden" name="table" value="${t.table}">
+          <div class="table-wrap"><table class="table ui-defaults report-reflow"><thead><tr><th scope="col">Column</th><th scope="col">Label</th><th scope="col">Format mask</th><th scope="col">Alignment</th><th scope="col">Help</th><th scope="col">List of values</th></tr></thead><tbody>
+          ${cols.map((c, i) => html`<tr>
+            <td data-label="Column"><code>${c.name}</code><br><span class="muted small">${c.type}</span><input type="hidden" name="c.${i}.name" value="${c.name}"></td>
+            <td data-label="Label"><input id="f_l${i}" aria-label="Label of ${c.name}" name="c.${i}.label" value="${c.d?.label ?? ''}" maxlength="200" placeholder="${c.wizard}"></td>
+            <td data-label="Format mask"><input id="f_m${i}" aria-label="Format mask of ${c.name}" name="c.${i}.format_mask" value="${c.d?.format_mask ?? ''}" maxlength="60" placeholder="${/numeric|integer|bigint|smallint|real|double/.test(c.type) ? '999G990D00' : /date|time/.test(c.type) ? 'DD.MM.YYYY' : ''}"></td>
+            <td data-label="Alignment"><select id="f_a${i}" aria-label="Alignment of ${c.name}" name="c.${i}.alignment">${ALIGNMENTS.map((x) => html`<option value="${x}"${(c.d?.alignment ?? '') === x ? raw(' selected') : ''}>${x || '–'}</option>`)}</select></td>
+            <td data-label="Help"><textarea id="f_h${i}" aria-label="Help of ${c.name}" name="c.${i}.help" rows="1" maxlength="4000">${c.d?.help ?? ''}</textarea></td>
+            <td data-label="List of values"><input id="f_v${i}" aria-label="List of values of ${c.name}" name="c.${i}.lov" value="${c.d?.lov ?? ''}" maxlength="4000" placeholder="LOV:NAME"></td>
+          </tr>`)}</tbody></table></div>
+          <div class="buttons"><button class="btn btn-hot" name="mode" value="save">${icon('save')} Save</button>
+            <button class="btn" name="mode" value="seed">${icon('edit')} Seed from comments</button></div>
+          <p class="muted small">An empty row has no UI default. The placeholder of a label shows what the wizards use without one. Lists of values: <code>LOV:NAME</code> (a shared list of values of the application), <code>STATIC:Yes;Y,No;N</code> or a query (<code>select name d, id r from sales.status</code>).</p>
+        </form>`;
+    }
+    const main = html`<h1 class="u-mb1">UI defaults</h1>
+      ${region('UI defaults per column', html`<p class="muted u-mt0">Label, format mask, help text, alignment and list of values per table column (APEX: Attribute Dictionary). The page wizards and the Create Application wizard use the label for every generated report heading and form item, and a generated form item also gets the format mask, help and list of values. They apply to pages generated after the change, in every application.</p>${body}`)}`;
+    return send(reply, s, shell(s, 'UI defaults', [['App Builder', BASE], ['Workspace utilities', `${BASE}/utilities`], ['UI defaults']], main));
+  });
+
+  app.post(`${BASE}/ui-defaults`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s) return;
+    const b: Body = req.body ?? {};
+    const t = await findTable(s, b.schema, b.table);
+    if (!t) return reply.code(404).send('Not found');
+    const to = `${BASE}/ui-defaults?${new URLSearchParams({ schema: t.schema, table: t.table })}`;
+    const cols = await columnsOf(t.oid, t.schema, t.table);
+    const known = new Map(cols.map((c) => [c.name, c]));
+    try {
+      if (b.mode === 'seed') {
+        // columns without a UI default: a short comment is the label, a longer one the help text
+        let n = 0;
+        for (const c of cols) {
+          if (c.d || !c.comment?.trim()) continue;
+          const text = c.comment.trim();
+          const short = text.length <= 60 && !text.includes('\n');
+          await owner.query(
+            `insert into meta.ui_default (schema_name, table_name, column_name, label, help, updated_by) values ($1, $2, $3, $4, $5, $6) on conflict do nothing`,
+            [t.schema, t.table, c.name, short ? text : null, short ? null : text.slice(0, 4000), s.username]);
+          n++;
+        }
+        flash(s, n ? `UI defaults of ${n} column${n === 1 ? '' : 's'} made from their comments.` : 'No column without a UI default has a comment.');
+        return back(reply, s, to);
+      }
+      const rows: { name: string; label: string | null; mask: string | null; help: string | null; align: string | null; lov: string | null }[] = [];
+      for (let i = 0; i < MAX_COLUMNS; i++) {
+        const name = b[`c.${i}.name`];
+        if (typeof name !== 'string' || !known.has(name)) continue;
+        const mask = trim(b[`c.${i}.format_mask`], 60);
+        if (mask && !MASK.test(mask)) throw new Error(`Format mask of ${name}: letters, digits and . , : / - only (e.g. 999G990D00 or DD.MM.YYYY).`);
+        const align = trim(b[`c.${i}.alignment`], 10);
+        if (align && !(ALIGNMENTS as readonly string[]).includes(align)) throw new Error(`Alignment of ${name}: start, center or end.`);
+        rows.push({ name, label: trim(b[`c.${i}.label`], 200), mask, help: trim(b[`c.${i}.help`], 4000), align, lov: trim(b[`c.${i}.lov`], 4000) });
+      }
+      await owner.tx(async (c) => {
+        for (const r of rows) {
+          const before = known.get(r.name)!.d;
+          if (!r.label && !r.mask && !r.help && !r.align && !r.lov) {
+            if (before) await c.query('delete from meta.ui_default where schema_name = $1 and table_name = $2 and column_name = $3', [t.schema, t.table, r.name]);
+            continue;
+          }
+          if (before && before.label === r.label && before.format_mask === r.mask && before.help === r.help && before.alignment === r.align && before.lov === r.lov) continue;
+          await c.query(
+            `insert into meta.ui_default (schema_name, table_name, column_name, label, format_mask, help, alignment, lov, updated_by)
+             values ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+             on conflict (schema_name, table_name, column_name) do update set label = excluded.label, format_mask = excluded.format_mask, help = excluded.help,
+               alignment = excluded.alignment, lov = excluded.lov, updated_by = excluded.updated_by, updated_at = now()`,
+            [t.schema, t.table, r.name, r.label, r.mask, r.help, r.align, r.lov, s.username]);
+        }
+      });
+      flash(s, `UI defaults of ${t.schema}.${t.table} saved.`);
+    } catch (e) {
+      flash(s, (e as Error).message, 'error');
+    }
+    return back(reply, s, to);
+  });
+}

package/src/builder/ui.ts

@@ -243,7 +243,7 @@ export const region = (title: string, body: Raw | Raw[], extra: Raw | '' = '', i
   html`<section class="region region-standard"${id ? raw(` id="${id.replace(/[^a-z0-9-]/g, '')}"`) : ''}><header class="region-header"><h2>${title}</h2>${extra}</header><div class="region-body">${body}</div></section>`;
 
 /** Title and tabs of an application's builder pages (Pages, Shared Components, …), in the session's builder language. */
-export const appHeader = (a: any, active: 'pages' | 'shared' | 'settings' | 'activity' | 'api' | 'search' | 'advisor', s?: Pick<Session, 'state'>) => {
+export const appHeader = (a: any, active: 'pages' | 'shared' | 'settings' | 'activity' | 'api' | 'search' | 'advisor' | 'utilities', s?: Pick<Session, 'state'>) => {
   const tx = tr(s);
   const tab = (key: typeof active, href: string, ic: string, label: string) =>
     html`<a class="ide-tab" href="${href}"${active === key ? raw(' aria-current="page"') : ''}>${bicon(ic)}<span>${label}</span></a>`;
@@ -266,6 +266,7 @@ export const appHeader = (a: any, active: 'pages' | 'shared' | 'settings' | 'act
         ${tab('settings', `${BASE}/apps/${a.id}/settings`, 'settings', tx('app.tab.settings'))}
         ${tab('search', `${BASE}/apps/${a.id}/search`, 'search', tx('common.search'))}
         ${tab('advisor', `${BASE}/apps/${a.id}/advisor`, 'check', tx('app.tab.advisor'))}
+        ${tab('utilities', `${BASE}/apps/${a.id}/utilities`, 'wrench', tx('app.tab.utilities'))}
       </nav>
     </div>`;
 };

package/src/builder/upgrade.ts

@@ -0,0 +1,154 @@
+import type { FastifyInstance } from 'fastify';
+import { stableJson } from '../appfiles.ts';
+import { owner } from '../db.ts';
+import { recordHistory } from '../history.ts';
+import { html, raw } from '../html.ts';
+import { icon } from '../icons.ts';
+import { appHeader, back, BASE, csrf, designerUrl, developer, flash, region, send, shell, type Req } from './ui.ts';
+import { diffHtml } from './utilities.ts';
+
+// (sprint 138, AP-5) App Builder → Utilities → Upgrade application (APEX:
+// Upgrade Application): components made before a pgkiln version that brought a
+// better default, each with its reason, a preview of the change and an Upgrade
+// button (or all of a kind at once). The candidates are worked out again on
+// every request: a POST names a candidate's key, and only a candidate that is
+// still found is changed. Each upgrade is recorded in the change history.
+//
+// Rules:
+//  - nav_appearance: an application without a navigation appearance gets
+//    Pillar Light (sprint 117), the look of new applications.
+//  - date_picker: a text item of a form whose column is a date or a timestamp
+//    becomes a date / date-time item with pgkiln's date picker (sprint 84).
+//  - popup_lov: a select list whose list of values reads a table of more than
+//    POPUP_ROWS rows (the planner's estimate) becomes a popup list of values,
+//    which searches and pages on the server instead of sending every row.
+
+export const POPUP_ROWS = 1000;
+
+export interface Candidate {
+  key: string;
+  rule: Rule;
+  label: string;
+  page: number | null;
+  url: string;
+  reason: string;
+  before: Record<string, unknown>;
+  after: Record<string, unknown>;
+  /** the change: a statement and its parameters */
+  sql: [string, unknown[]];
+}
+
+export type Rule = 'nav_appearance' | 'date_picker' | 'popup_lov';
+export const RULES: Record<Rule, { title: string; since: string }> = {
+  nav_appearance: { title: 'Navigation appearance', since: 'Navigation appearances (Pillar Light, Pillar Dark) came with sprint 117.' },
+  date_picker: { title: 'Date picker for date columns', since: 'Date and date-time items have had pgkiln\'s date picker since sprint 84.' },
+  popup_lov: { title: 'Paged list of values for large tables', since: 'Popup lists of values search and page on the server.' },
+};
+
+/** The relations a list of values query reads (FROM / JOIN names). */
+export const lovTables = (sql: string) =>
+  [...sql.matchAll(/\b(?:from|join)\s+((?:"[^"]+"|[A-Za-z_][\w$]*)(?:\s*\.\s*(?:"[^"]+"|[A-Za-z_][\w$]*))?)/gi)].map((m) => m[1].replace(/\s+/g, ''));
+
+/** Every upgrade candidate of an application. */
+export async function upgradeCandidates(appId: number): Promise<Candidate[]> {
+  const out: Candidate[] = [];
+  const app = await owner.one('select id, theme from meta.app where id = $1', [appId]);
+  if (!app) return out;
+  const theme = app.theme && typeof app.theme === 'object' ? app.theme : {};
+  if (typeof theme.appearance !== 'string' || !theme.appearance) {
+    out.push({
+      key: 'nav_appearance:app', rule: 'nav_appearance', label: 'Application settings › Theme', page: null, url: `${BASE}/apps/${appId}/theme`,
+      reason: 'The application has no navigation appearance: Pillar Light gives the header strip and the filled current page of new applications.',
+      before: { appearance: theme.appearance ?? null }, after: { appearance: 'pillar_light' },
+      sql: [`update meta.app set theme = coalesce(theme, '{}'::jsonb) || '{"appearance": "pillar_light"}' where id = $1 and coalesce(theme ->> 'appearance', '') = ''`, [appId]],
+    });
+  }
+  const dates = (await owner.query(
+    `select i.id, i.name, i.type, p.page_no, i.source_column, r.table_name, format_type(a.atttypid, a.atttypmod) as coltype,
+            a.atttypid = 'date'::regtype as is_date
+       from meta.item i join meta.page p on p.id = i.page_id join meta.region r on r.id = i.region_id
+       join pg_attribute a on a.attrelid = to_regclass(r.table_name) and a.attname = i.source_column and not a.attisdropped
+      where p.app_id = $1 and i.type = 'text' and r.type = 'form' and r.table_name is not null
+        and a.atttypid in ('date'::regtype, 'timestamp'::regtype, 'timestamptz'::regtype)
+      order by p.page_no, i.seq, i.id`, [appId])).rows;
+  for (const r of dates) {
+    const to = r.is_date ? 'date' : 'datetime';
+    out.push({
+      key: `date_picker:${r.id}`, rule: 'date_picker', label: `Page ${r.page_no} › item ${r.name}`, page: r.page_no, url: designerUrl(appId, r.page_no, `c=item-${r.id}`),
+      reason: `${r.table_name}.${r.source_column} is a ${r.coltype} column shown as a text field: a ${to} item has the date picker and reads the date in the application's format.`,
+      before: { type: 'text' }, after: { type: to },
+      sql: [`update meta.item set type = $2 where id = $1 and type = 'text'`, [r.id, to]],
+    });
+  }
+  const selects = (await owner.query(
+    `select i.id, i.name, p.page_no, i.lov from meta.item i join meta.page p on p.id = i.page_id
+      where p.app_id = $1 and i.type = 'select' and i.lov is not null and i.lov !~* '^\\s*(STATIC|LOV):'
+      order by p.page_no, i.seq, i.id`, [appId])).rows;
+  for (const r of selects) {
+    const names = lovTables(r.lov);
+    if (!names.length) continue;
+    const big = (await owner.query<{ name: string; n: number }>(
+      `select x as name, c.reltuples::bigint::int as n from unnest($1::text[]) x join pg_class c on c.oid = to_regclass(x) where c.reltuples > $2 order by c.reltuples desc limit 1`,
+      [names, POPUP_ROWS]).catch(() => ({ rows: [] }))).rows[0];
+    if (!big) continue;
+    out.push({
+      key: `popup_lov:${r.id}`, rule: 'popup_lov', label: `Page ${r.page_no} › item ${r.name}`, page: r.page_no, url: designerUrl(appId, r.page_no, `c=item-${r.id}`),
+      reason: `Its list of values reads ${big.name} (about ${big.n} rows): a select list sends every row with the page; a popup list of values searches and pages on the server.`,
+      before: { type: 'select' }, after: { type: 'popup_lov' },
+      sql: [`update meta.item set type = 'popup_lov' where id = $1 and type = 'select'`, [r.id]],
+    });
+  }
+  return out;
+}
+
+export async function upgradeRoutes(app: FastifyInstance) {
+  const appOf = async (id: string) => (/^\d{1,9}$/.test(id) ? owner.one('select * from meta.app where id = $1', [id]) : undefined);
+
+  app.get(`${BASE}/apps/:id/utilities/upgrade`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s) return;
+    const a = await appOf(req.params.id);
+    if (!a) return reply.code(404).send('Not found');
+    const all = await upgradeCandidates(a.id);
+    const preview = typeof req.query?.preview === 'string' ? all.find((c) => c.key === req.query.preview) : undefined;
+    const u = `${BASE}/apps/${a.id}/utilities/upgrade`;
+    const sections = (Object.keys(RULES) as Rule[]).map((rule) => {
+      const list = all.filter((c) => c.rule === rule);
+      return html`<section class="util-upgrade" aria-labelledby="up-${rule}"><h3 id="up-${rule}">${RULES[rule].title} <span class="tag">${list.length}</span></h3>
+        <p class="muted small">${RULES[rule].since}</p>
+        ${list.length ? html`<div class="table-wrap"><table class="table report-reflow"><thead><tr><th scope="col">Component</th><th scope="col">Why</th><th scope="col">Actions</th></tr></thead><tbody>
+          ${list.map((c) => html`<tr${preview?.key === c.key ? raw(' aria-current="true"') : ''}><td data-label="Component"><a href="${c.url}">${c.label}</a></td><td data-label="Why">${c.reason}</td>
+            <td data-label="Actions"><div class="buttons"><a class="btn btn-sm" href="${u}?preview=${encodeURIComponent(c.key)}#preview">Preview</a>
+              <form method="post" action="${u}">${csrf(s)}<input type="hidden" name="key" value="${c.key}"><button class="btn btn-sm btn-hot">${icon('refresh')} Upgrade</button></form></div></td></tr>`)}</tbody></table></div>
+          ${list.length > 1 ? html`<form method="post" action="${u}" class="u-mt075">${csrf(s)}<input type="hidden" name="rule" value="${rule}"><button class="btn btn-sm">${icon('refresh')} Upgrade all ${list.length}</button></form>` : ''}`
+          : html`<p class="muted">Nothing to upgrade.</p>`}</section>`;
+    });
+    const main = html`${appHeader(a, 'utilities', s)}
+      ${region('Upgrade application', html`<p class="muted u-mt0">Components made before a pgkiln version that brought a better default. Preview a change, then upgrade one component or all of a kind; every upgrade is in the change history (and the backups) if you want to go back.</p>${sections}`)}
+      ${preview ? region(`Preview: ${preview.label}`, html`<p class="muted u-mt0">${preview.reason}</p>${diffHtml('settings', stableJson(preview.before), stableJson(preview.after), ['now', 'upgraded'])}
+        <form method="post" action="${u}">${csrf(s)}<input type="hidden" name="key" value="${preview.key}"><div class="buttons"><button class="btn btn-hot">${icon('refresh')} Upgrade</button></div></form>`, '', 'preview') : ''}`;
+    return send(reply, s, shell(s, `Upgrade application · ${a.name}`, [['App Builder', BASE], [a.name, `${BASE}/apps/${a.id}`], ['Utilities', `${BASE}/apps/${a.id}/utilities`], ['Upgrade application']], main));
+  });
+
+  app.post(`${BASE}/apps/:id/utilities/upgrade`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s) return;
+    const a = await appOf(req.params.id);
+    if (!a) return reply.code(404).send('Not found');
+    const all = await upgradeCandidates(a.id);
+    const key = typeof req.body?.key === 'string' ? req.body.key : '';
+    const rule = typeof req.body?.rule === 'string' ? req.body.rule : '';
+    const chosen = key ? all.filter((c) => c.key === key) : all.filter((c) => c.rule === rule);
+    const to = `${BASE}/apps/${a.id}/utilities/upgrade`;
+    if (!chosen.length) {
+      flash(s, 'Nothing to upgrade: the component was changed or upgraded in the meantime.', 'error');
+      return back(reply, s, to);
+    }
+    await owner.tx(async (c) => {
+      for (const x of chosen) await c.query(x.sql[0], x.sql[1]);
+    });
+    await recordHistory(a.id, s.username, 'upgrade').catch((err) => req.log.warn({ err }, 'change history not recorded'));
+    flash(s, chosen.length === 1 ? `${chosen[0].label} upgraded.` : `${chosen.length} components upgraded.`);
+    return back(reply, s, to);
+  });
+}