pgkiln 0.52.0 → 0.53.0

package/src/builder/home.ts

@@ -20,7 +20,7 @@ const SORTS: Record<Sort, string> = {
   id: 'a.id',
   alias: 'a.alias',
   pages: 'pages desc, lower(a.name)',
-  updated: 'a.updated_at desc nulls last, lower(a.name)',
+  updated: 'greatest(a.updated_at, c.updated_at) desc nulls last, lower(a.name)',
 };
 const RECENT_MAX = 8;
 
@@ -45,6 +45,10 @@ export function ago(at: Date | string | null | undefined, now = Date.now()) {
   return 'just now';
 }
 
+/** (116) the App Builder list's working copies count and lock state */
+const copiesText = (n: number) => `${n} working cop${n === 1 ? 'y' : 'ies'}`;
+const lockedText = (by: string) => `Locked by ${by}`;
+
 const appIcon = (a: { id: number; name: string }, cls = '') =>
   html`<span class="app-icon app-color-${a.id % APP_COLORS}${cls ? ` ${cls}` : ''}" aria-hidden="true">${a.name.slice(0, 1).toUpperCase()}</span>`;
 
@@ -108,10 +112,14 @@ export async function homeRoutes(app: FastifyInstance) {
     const group = groups.find((g) => String(g.id) === s.state.__BGROUP) ?? null;
     const [apps, total, recent] = await Promise.all([
       owner.query(
-        `select a.id, a.name, a.alias, a.authentication, a.db_role, a.updated_at, g.name as group_name,
+        `select a.id, a.name, a.alias, a.authentication, a.db_role, greatest(a.updated_at, c.updated_at) as updated_at, g.name as group_name,
+                c.updated_by,
+                (select count(*) from meta.working_copy w where w.main_app_id = a.id)::int as copies,
+                (select locked_by from meta.builder_lock k where k.app_id = a.id and k.page_no = 0) as locked_by,
                 (select count(*) from meta.page p where p.app_id = a.id)::int as pages,
                 (select count(*) from meta.activity_log l where l.app_id = a.id and l.event = 'page_view' and l.at > now() - interval '1 day')::int as views
            from meta.app a
+           left join meta.app_change c on c.app_id = a.id
            left join meta.app_group_member gm on gm.app_id = a.id
            left join meta.app_group g on g.id = gm.group_id
           where ($1 = '' or a.name ilike '%' || $1 || '%' or a.alias ilike '%' || $1 || '%' or a.id::text = $1) and ${inWorkspaceSql('a', 2)}
@@ -137,22 +145,25 @@ export async function homeRoutes(app: FastifyInstance) {
       : view === 'grid'
         ? html`<div class="ab-cards">${rows.map((a) => html`<article class="ab-card" data-filter-row>
             <a class="ab-card-main" href="${BASE}/apps/${a.id}">${appIcon(a)}<span><strong>${a.name}</strong><small>${a.id} · /a/${a.alias}</small></span></a>
-            <div class="ab-card-meta">${a.group_name ? html`<span class="tag">${a.group_name}</span>` : ''}<span>${a.pages} pages</span><span>${a.views} views today</span><span>${ago(a.updated_at)}</span></div>
+            <div class="ab-card-meta">${a.group_name ? html`<span class="tag">${a.group_name}</span>` : ''}<span>${a.pages} pages</span><span>${a.views} views today</span><span>${ago(a.updated_at)}${a.updated_by ? html` by <span class="ab-by">${a.updated_by}</span>` : ''}</span></div>
+            ${a.copies || a.locked_by ? html`<div class="ab-card-badges">${a.copies ? html`<a class="tag" href="${BASE}/apps/${a.id}/working-copies">${icon('layers')} ${copiesText(a.copies)}</a>` : ''}${a.locked_by ? html`<span class="tag ab-locked" title="${lockedText(a.locked_by)}">${icon('lock')} <span class="ab-by">${lockedText(a.locked_by)}</span></span>` : ''}</div>` : ''}
             <div class="ab-card-actions">
               <a class="tb-btn" href="${BASE}/apps/${a.id}" title="Edit ${a.name}">${icon('edit')}<span class="sr-only">Edit ${a.name}</span></a>
               <a class="tb-btn" href="/a/${a.alias}" target="_blank" rel="noopener" title="Run ${a.name}">${icon('play')}<span class="sr-only">Run ${a.name}</span></a>
             </div></article>`)}</div>`
         : html`<div class="table-wrap ab-table"><table class="report ab-apps">
             <caption class="sr-only">Applications</caption>
-            <thead><tr>${th('id', 'Application', 'num')}${th('name', 'Name')}${th('alias', 'Alias')}${th('pages', 'Pages', 'num')}<th scope="col" class="num">Views (24h)</th><th scope="col">Sign-in</th>${th('updated', 'Updated')}<th scope="col" class="ab-actions-col">Actions</th></tr></thead>
+            <thead><tr>${th('id', 'Application', 'num')}${th('name', 'Name')}${th('alias', 'Alias')}${th('pages', 'Pages', 'num')}<th scope="col" class="num">Views (24h)</th><th scope="col">Sign-in</th>${th('updated', 'Updated')}<th scope="col">Updated by</th><th scope="col" class="num">Working copies</th><th scope="col" class="ab-actions-col">Actions</th></tr></thead>
             <tbody>${rows.map((a) => html`<tr data-filter-row>
               <td class="num">${a.id}</td>
-              <td class="ab-name"><a href="${BASE}/apps/${a.id}">${appIcon(a)}<span>${a.name}</span></a></td>
+              <td class="ab-name"><a href="${BASE}/apps/${a.id}">${appIcon(a)}<span>${a.name}</span></a>${a.locked_by ? html`<span class="ab-lock" role="img" aria-label="${lockedText(a.locked_by)}" title="${lockedText(a.locked_by)}">${icon('lock')}</span>` : ''}</td>
               <td>${a.alias}${a.group_name ? html` <span class="tag">${a.group_name}</span>` : ''}</td>
               <td class="num">${a.pages}</td>
               <td class="num">${a.views}</td>
               <td>${a.authentication === 'none' ? 'Public' : a.authentication === 'header' ? 'HTTP header' : a.authentication === 'database' ? 'Database accounts' : a.authentication === 'custom' ? 'Custom' : 'App users'}</td>
               <td title="${a.updated_at ? new Date(a.updated_at).toISOString().slice(0, 16).replace('T', ' ') : ''}">${ago(a.updated_at)}</td>
+              <td class="ab-by">${a.updated_by ?? ''}</td>
+              <td class="num">${a.copies ? html`<a href="${BASE}/apps/${a.id}/working-copies" aria-label="${copiesText(a.copies)} of ${a.name}">${a.copies}</a>` : '0'}</td>
               <td class="ab-actions">
                 <a class="tb-btn" href="${BASE}/apps/${a.id}" title="Edit ${a.name}">${icon('edit')}<span class="sr-only">Edit ${a.name}</span></a>
                 <a class="tb-btn" href="/a/${a.alias}" target="_blank" rel="noopener" title="Run ${a.name}">${icon('play')}<span class="sr-only">Run ${a.name}</span></a>

package/src/builder/mail-queue.ts

@@ -0,0 +1,212 @@
+import type { FastifyInstance, FastifyReply } from 'fastify';
+import { owner } from '../db.ts';
+import { html } from '../html.ts';
+import { icon } from '../icons.ts';
+import { clientIp, logActivity, type Session } from '../session.ts';
+import { isAdmin } from './locks.ts';
+import { back, BASE, csrf, developer, flash, region, select, send, shell, type Req } from './ui.ts';
+import { workspaceAdmin } from './workspaces.ts';
+
+// (sprint 118, WS-9; APEX 26.2 Administration → Mail queue) Delivery health of
+// the mail queue (meta.mail_queue): queued, sent, failed and retrying mail per
+// hour and per workspace / application, the slowest provider calls, and the
+// failures with the provider's answer, with Retry and Purge.
+//
+// Who sees what: administrators see every workspace (or one); a workspace
+// administrator sees the mail of their current workspace's applications only.
+// Every action checks the mail's workspace again on the server. Bodies and
+// attachments are never shown here.
+
+export const PERIODS: [string, string][] = [['24', 'Last 24 hours'], ['48', 'Last 48 hours'], ['168', 'Last 7 days']];
+
+/** The mail a developer may see: every workspace (null) or one; false when none. */
+export async function queueScope(s: Session): Promise<{ admin: boolean; workspace: number | null } | false> {
+  if (await isAdmin(s.username)) return { admin: true, workspace: null };
+  if (s.workspace && (await workspaceAdmin(s.username, s.workspace.id))) return { admin: false, workspace: s.workspace.id };
+  return false;
+}
+
+async function refuse(s: Session, reply: FastifyReply) {
+  const main = html`<h1 class="u-mb1">Mail queue</h1><div class="alert alert-error" role="alert">Only administrators and workspace administrators see the mail queue.</div>`;
+  await send(reply.code(403), s, shell(s, 'Mail queue', [['Administration'], ['Mail queue']], main, 'users'));
+}
+
+const when = (d: Date | string | null) => (d ? new Date(d).toISOString().slice(0, 16).replace('T', ' ') : '');
+const id = (v: unknown) => (/^\d{1,9}$/.test(String(v ?? '')) ? Number(v) : null);
+/** the scope condition on meta.mail_queue q: $1 workspace (null: all), $2 application (null: all) */
+const IN_SCOPE = '($1::int is null or meta.app_workspace(q.app_id) = $1) and ($2::int is null or q.app_id = $2)';
+
+export async function mailQueueRoutes(app: FastifyInstance) {
+  app.get(`${BASE}/mail-queue`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s) return;
+    const scope = await queueScope(s);
+    if (!scope) return refuse(s, reply);
+    const hours = PERIODS.some(([v]) => v === req.query.period) ? Number(req.query.period) : 24;
+    const ws = scope.admin ? id(req.query.ws) : scope.workspace;
+    const appId = id(req.query.app);
+    const p = [ws, appId, hours];
+    const since = `now() - make_interval(hours => $3)`;
+    const [workspaces, apps, totals, perHour, usage, slowest, failures] = await Promise.all([
+      scope.admin ? owner.query<{ id: number; name: string }>('select id, name from meta.workspace order by id = 1 desc, lower(name)').then((r) => r.rows) : [],
+      owner.query<{ id: number; alias: string }>('select id, alias from meta.app a where $1::int is null or meta.app_workspace(a.id) = $1 order by alias', [ws]).then((r) => r.rows),
+      owner.one(
+        `select count(*) filter (where status = 'queued' and attempts = 0)::int as queued,
+                count(*) filter (where status = 'queued' and attempts > 0)::int as retrying,
+                count(*) filter (where status = 'sending')::int as sending,
+                count(*) filter (where status = 'sent' and sent_at >= ${since})::int as sent,
+                count(*) filter (where status = 'error' and coalesce(last_attempt_at, created_at) >= ${since})::int as failed,
+                round(avg(duration_ms) filter (where last_attempt_at >= ${since}))::int as avg_ms
+           from meta.mail_queue q where ${IN_SCOPE}`, p),
+      owner.query(
+        `select h as hour, count(*) filter (where k = 'queued')::int as queued, count(*) filter (where k = 'sent')::int as sent,
+                count(*) filter (where k = 'failed')::int as failed, count(*) filter (where k = 'retrying')::int as retrying
+           from (select date_trunc('hour', q.created_at) as h, 'queued' as k from meta.mail_queue q where ${IN_SCOPE} and q.created_at >= ${since}
+                 union all select date_trunc('hour', q.sent_at), 'sent' from meta.mail_queue q where ${IN_SCOPE} and q.status = 'sent' and q.sent_at >= ${since}
+                 union all select date_trunc('hour', coalesce(q.last_attempt_at, q.created_at)), 'failed' from meta.mail_queue q
+                            where ${IN_SCOPE} and q.status = 'error' and coalesce(q.last_attempt_at, q.created_at) >= ${since}
+                 union all select date_trunc('hour', q.last_attempt_at), 'retrying' from meta.mail_queue q
+                            where ${IN_SCOPE} and q.status = 'queued' and q.attempts > 0 and q.last_attempt_at >= ${since}) e
+          group by h order by h desc`, p).then((r) => r.rows),
+      owner.query(
+        `select w.name as workspace, a.alias, a.name, q.app_id,
+                count(*) filter (where q.status = 'sent' and q.sent_at >= ${since})::int as sent,
+                count(*) filter (where q.status = 'error' and coalesce(q.last_attempt_at, q.created_at) >= ${since})::int as failed,
+                count(*) filter (where q.status = 'queued' and q.attempts > 0)::int as retrying,
+                count(*) filter (where q.status in ('queued', 'sending') and q.attempts = 0)::int as queued,
+                round(avg(q.duration_ms) filter (where q.last_attempt_at >= ${since}))::int as avg_ms,
+                max(q.duration_ms) filter (where q.last_attempt_at >= ${since}) as max_ms
+           from meta.mail_queue q join meta.app a on a.id = q.app_id join meta.workspace w on w.id = meta.app_workspace(q.app_id)
+          where ${IN_SCOPE} and (q.created_at >= ${since} or q.last_attempt_at >= ${since} or q.status in ('queued', 'sending'))
+          group by w.name, a.alias, a.name, q.app_id order by lower(w.name), a.alias`, p).then((r) => r.rows),
+      owner.query(
+        `select q.id, q.last_attempt_at, a.alias, q.service, q.provider, q.status, q.duration_ms, q.subject
+           from meta.mail_queue q join meta.app a on a.id = q.app_id
+          where ${IN_SCOPE} and q.last_attempt_at >= ${since} and q.duration_ms is not null
+          order by q.duration_ms desc, q.id desc limit 10`, p).then((r) => r.rows),
+      // failures are listed until they are retried or purged (kept 30 days), whatever the period
+      owner.query(
+        `select q.id, q.created_at, coalesce(q.last_attempt_at, q.created_at) as failed_at, a.alias, w.name as workspace, q.to_addresses, q.cc_addresses,
+                q.bcc_addresses, q.subject, q.service, q.provider, q.attempts, q.message
+           from meta.mail_queue q join meta.app a on a.id = q.app_id join meta.workspace w on w.id = meta.app_workspace(q.app_id)
+          where ($1::int is null or meta.app_workspace(q.app_id) = $1) and ($2::int is null or q.app_id = $2) and q.status = 'error'
+          order by q.id desc limit 100`, [ws, appId]).then((r) => r.rows),
+    ]);
+    const wsName = ws ? (scope.admin ? workspaces.find((w) => w.id === ws)?.name : s.workspace?.name) : null;
+    const filters = html`<form method="get" class="filter-row mail-queue-filters">
+        ${select('period', 'Period', String(hours), PERIODS)}
+        ${scope.admin ? select('ws', 'Workspace', ws ? String(ws) : '', [['', 'All workspaces'], ...workspaces.map((w): [string, string] => [String(w.id), w.name])]) : ''}
+        ${select('app', 'Application', appId ? String(appId) : '', [['', 'All applications'], ...apps.map((a): [string, string] => [String(a.id), a.alias])])}
+        <div class="buttons"><button class="btn">Show</button></div></form>`;
+    const total = totals.sent + totals.failed;
+    const health = total ? Math.round((totals.sent / total) * 1000) / 10 : null;
+    const stat = (n: unknown, label: string, cls = '') => html`<div class="stat${cls}"><b>${n ?? '–'}</b><span>${label}</span></div>`;
+    const keep = new URLSearchParams(Object.entries({ period: String(hours), ...(scope.admin && ws ? { ws: String(ws) } : {}), ...(appId ? { app: String(appId) } : {}) })).toString();
+    const action = (path: string, label: unknown, cls: string, confirm?: string) =>
+      html`<form method="post" action="${BASE}/mail-queue${path}?${keep}" class="inline-form">${csrf(s)}<button class="btn btn-sm${cls}"${confirm ? html` data-confirm="${confirm}"` : ''}>${label}</button></form>`;
+    const main = html`
+      <div class="title-row"><h1>Mail queue</h1></div>
+      <p class="muted u-mt0">Delivery of the mail ${wsName ? html`of workspace <b>${wsName}</b>` : 'of every workspace'}: what was queued, sent and failed, which mail is being retried, how long the providers took, and the failures with the provider's answer. Mail is kept 30 days.${scope.admin ? '' : ' You see your workspace\'s applications only.'}</p>
+      ${filters}
+      <div class="stat-grid mail-queue-stats">
+        ${stat(totals.queued, 'queued now')}${stat(totals.retrying, 'retrying now')}${stat(totals.sending, 'being sent')}
+        ${stat(totals.sent, 'sent')}${stat(totals.failed, 'failed', totals.failed ? ' stat-bad' : '')}
+        ${stat(health === null ? null : `${health}%`, 'delivered')}${stat(totals.avg_ms === null ? null : `${totals.avg_ms} ms`, 'average provider call')}
+      </div>
+      <div class="columns">
+        ${region('Per hour', html`<div class="table-wrap"><table class="report report-reflow">
+          <thead><tr><th>Hour (UTC)</th><th class="num">Queued</th><th class="num">Sent</th><th class="num">Failed</th><th class="num">Retrying</th></tr></thead>
+          <tbody>${perHour.length ? perHour.map((h) => html`<tr><td data-label="Hour">${when(h.hour)}</td><td class="num" data-label="Queued">${h.queued}</td>
+              <td class="num" data-label="Sent">${h.sent}</td><td class="num" data-label="Failed">${h.failed ? html`<b>${h.failed}</b>` : 0}</td><td class="num" data-label="Retrying">${h.retrying}</td></tr>`)
+            : html`<tr><td colspan="5" class="empty">No mail in this period.</td></tr>`}</tbody></table></div>`)}
+        ${region('Slowest provider calls', html`<div class="table-wrap"><table class="report report-reflow">
+          <thead><tr><th>When (UTC)</th><th>Application</th><th>Service</th><th>Status</th><th class="num">Time</th></tr></thead>
+          <tbody>${slowest.length ? slowest.map((m) => html`<tr><td data-label="When">${when(m.last_attempt_at)}</td><td data-label="Application">${m.alias}</td>
+              <td data-label="Service" class="u-break">${m.service ?? ''}${m.provider ? html` <span class="muted">${m.provider}</span>` : ''}</td>
+              <td data-label="Status"><span class="tag${m.status === 'sent' ? ' tag-ok' : m.status === 'error' ? ' tag-error' : ''}">${m.status === 'queued' ? 'retrying' : m.status}</span></td>
+              <td class="num" data-label="Time">${m.duration_ms} ms</td></tr>`)
+            : html`<tr><td colspan="5" class="empty">No provider calls in this period.</td></tr>`}</tbody></table></div>`)}
+      </div>
+      ${region('Per workspace and application', html`<div class="table-wrap"><table class="report report-reflow">
+        <thead><tr><th>Workspace</th><th>Application</th><th class="num">Sent</th><th class="num">Failed</th><th class="num">Retrying</th><th class="num">Queued</th><th class="num">Average</th><th class="num">Slowest</th></tr></thead>
+        <tbody>${usage.length ? usage.map((u) => html`<tr><td data-label="Workspace">${u.workspace}</td>
+            <td data-label="Application"><a href="${BASE}/mail-queue?${new URLSearchParams({ period: String(hours), ...(scope.admin && ws ? { ws: String(ws) } : {}), app: String(u.app_id) }).toString()}">${u.alias}</a></td>
+            <td class="num" data-label="Sent">${u.sent}</td><td class="num" data-label="Failed">${u.failed ? html`<b>${u.failed}</b>` : 0}</td>
+            <td class="num" data-label="Retrying">${u.retrying}</td><td class="num" data-label="Queued">${u.queued}</td>
+            <td class="num" data-label="Average">${u.avg_ms === null ? '–' : `${u.avg_ms} ms`}</td><td class="num" data-label="Slowest">${u.max_ms === null ? '–' : `${u.max_ms} ms`}</td></tr>`)
+          : html`<tr><td colspan="8" class="empty">No mail in this period.</td></tr>`}</tbody></table></div>`)}
+      ${region('Failures', html`<p class="muted u-mt0">Mail that will not be tried again: the provider refused it, or it failed ${5} times. <b>Retry</b> queues it again with fresh attempts; <b>Purge</b> deletes it.</p>
+        <div class="table-wrap"><table class="report report-reflow mail-failures">
+        <thead><tr><th>Failed (UTC)</th><th>Application</th><th>To</th><th>Subject</th><th>Provider's answer</th><th>Actions</th></tr></thead>
+        <tbody>${failures.length ? failures.map((m) => html`<tr>
+            <td data-label="Failed">${when(m.failed_at)}<br><span class="muted small">queued ${when(m.created_at)} · ${m.attempts} attempt${m.attempts === 1 ? '' : 's'}</span></td>
+            <td data-label="Application">${m.alias}${scope.admin && !ws ? html`<br><span class="muted small">${m.workspace}</span>` : ''}</td>
+            <td data-label="To" class="u-break">${[...m.to_addresses, ...m.cc_addresses].join(', ')}${m.bcc_addresses.length ? html` <span class="muted">+${m.bcc_addresses.length} bcc</span>` : ''}</td>
+            <td data-label="Subject" class="u-break">${m.subject}</td>
+            <td data-label="Provider's answer" class="u-break">${m.service ? html`<span class="muted small">${m.service}${m.provider ? ` (${m.provider})` : ''}</span><br>` : ''}<span class="mail-answer">${m.message ?? ''}</span></td>
+            <td data-label="Actions"><div class="buttons u-mt0">${action(`/${m.id}/retry`, 'Retry', '')}${action(`/${m.id}/purge`, 'Purge', ' btn-danger', 'Delete this mail?')}</div></td></tr>`)
+          : html`<tr><td colspan="6" class="empty">No failed mail.</td></tr>`}</tbody></table></div>
+        ${failures.length ? html`<div class="buttons">${action('/retry', html`${icon('refresh')} Retry all failed`, '')}${action('/purge', html`${icon('trash')} Purge all failed`, ' btn-danger', 'Delete every failed mail shown here?')}</div>` : ''}`,
+        '', 'failures')}`;
+    return send(reply, s, shell(s, 'Mail queue', [['Administration'], ['Mail queue']], main, 'users'));
+  });
+
+  /** One mail's id when this developer may act on it (its application is in their scope). */
+  const mine = async (s: Session, mailId: unknown) => {
+    const scope = await queueScope(s);
+    if (!scope) return null;
+    const m = await owner.one<{ id: string; app_id: number }>(
+      'select q.id, q.app_id from meta.mail_queue q where q.id = $1 and ($2::int is null or meta.app_workspace(q.app_id) = $2)', [id(mailId), scope.workspace]);
+    return m;
+  };
+  const redirect = (req: Req) => {
+    const q = new URLSearchParams();
+    for (const k of ['period', 'ws', 'app']) if (id(req.query[k])) q.set(k, String(req.query[k]));
+    return `${BASE}/mail-queue${q.size ? `?${q}` : ''}#failures`;
+  };
+
+  app.post(`${BASE}/mail-queue/:mid(^\\d{1,12}$)/retry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s) return;
+    if (!(await queueScope(s))) return refuse(s, reply);
+    const m = await mine(s, req.params.mid);
+    if (!m) return reply.code(404).send('Not found');
+    const r = await owner.query(`update meta.mail_queue set status = 'queued', attempts = 0, not_before = now(), message = null where id = $1 and status in ('error', 'cancelled')`, [m.id]);
+    if (r.rowCount) await owner.query(`select pg_notify('pgkiln_mail', $1)`, [String(m.id)]);
+    await logActivity({ appId: m.app_id, username: s.username, event: 'mail_queue', ip: clientIp(req), detail: `retry mail ${m.id}` });
+    flash(s, r.rowCount ? 'The mail is queued again.' : 'The mail has changed meanwhile.', r.rowCount ? 'ok' : 'error');
+    return back(reply, s, redirect(req));
+  });
+
+  app.post(`${BASE}/mail-queue/:mid(^\\d{1,12}$)/purge`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s) return;
+    if (!(await queueScope(s))) return refuse(s, reply);
+    const m = await mine(s, req.params.mid);
+    if (!m) return reply.code(404).send('Not found');
+    // mail that is waiting or being sent is cancelled on the application's Mail page, not purged
+    const r = await owner.query(`delete from meta.mail_queue where id = $1 and status in ('error', 'cancelled', 'sent')`, [m.id]);
+    await logActivity({ appId: m.app_id, username: s.username, event: 'mail_queue', ip: clientIp(req), detail: `purged mail ${m.id}` });
+    flash(s, r.rowCount ? 'The mail is deleted.' : 'Only sent, failed or cancelled mail can be purged.', r.rowCount ? 'ok' : 'error');
+    return back(reply, s, redirect(req));
+  });
+
+  // all failed mail in the developer's scope (and the filter's workspace and application)
+  for (const what of ['retry', 'purge'] as const)
+    app.post(`${BASE}/mail-queue/${what}`, async (req: Req, reply) => {
+      const s = await developer(req, reply);
+      if (!s) return;
+      const scope = await queueScope(s);
+      if (!scope) return refuse(s, reply);
+      const ws = scope.admin ? id(req.query.ws) : scope.workspace;
+      const appId = id(req.query.app);
+      const r = what === 'retry'
+        ? await owner.query(`update meta.mail_queue q set status = 'queued', attempts = 0, not_before = now(), message = null where ${IN_SCOPE} and q.status = 'error'`, [ws, appId])
+        : await owner.query(`delete from meta.mail_queue q where ${IN_SCOPE} and q.status = 'error'`, [ws, appId]);
+      if (what === 'retry' && r.rowCount) await owner.query(`select pg_notify('pgkiln_mail', '')`);
+      await logActivity({ appId: appId, username: s.username, event: 'mail_queue', ip: clientIp(req),
+        detail: `${what === 'retry' ? 'retried' : 'purged'} ${r.rowCount} failed mail(s)${ws ? ` of workspace ${ws}` : ''}${appId ? ` of application ${appId}` : ''}` });
+      flash(s, `${r.rowCount} mail${r.rowCount === 1 ? '' : 's'} ${what === 'retry' ? 'queued again' : 'deleted'}.`);
+      return back(reply, s, redirect(req));
+    });
+}

package/src/builder/mail.ts

@@ -2,7 +2,7 @@ import type { FastifyInstance } from 'fastify';
 import { owner } from '../db.ts';
 import { html, raw } from '../html.ts';
 import { icon } from '../icons.ts';
-import { senderAllowed, sendWith, type MailService } from '../mail/mail.ts';
+import { senderAllowed, serviceSettings, verifyService, type MailService } from '../mail/mail.ts';
 import { deliver, MailError, PROVIDER_LABELS, PROVIDERS, settingsProblems, verifyConnection, type Provider, type ServiceSettings } from '../mail/providers.ts';
 import { decryptSecret, encryptSecret, secretKeyConfigured } from '../secrets.ts';
 import { clientIp, logActivity, type Session } from '../session.ts';
@@ -31,7 +31,7 @@ const SECRET_LABEL: Record<Provider, string> = {
 };
 
 /** What each provider is, for the choice of a new service. */
-const PROVIDER_TEXT: Record<Provider, [icon: string, text: string]> = {
+export const PROVIDER_TEXT: Record<Provider, [icon: string, text: string]> = {
   smtp: ['send', 'Any SMTP server or relay. Sign in with a password, or with OAuth2 (Exchange Online, Google Workspace).'],
   graph: ['building', 'Microsoft 365 through the Graph API, as an Entra application with Mail.Send.'],
   ses: ['database', 'Amazon Simple Email Service (API v2), with an IAM access key.'],
@@ -48,16 +48,21 @@ function providerChoice() {
     html`<a class="ab-tile" href="${BASE}/mail/new?provider=${p}"><span class="ab-tile-icon">${icon(PROVIDER_TEXT[p][0])}</span><strong>${PROVIDER_LABELS[p]}</strong><span>${PROVIDER_TEXT[p][1]}</span></a>`)}</div>`;
 }
 
-/** The settings of one provider only, in groups (a new service, or an existing one switching provider). */
-function serviceForm(x: Partial<MailService> & { has_secret?: boolean }, provider: Provider, action: string, csrfField: ReturnType<typeof csrf>, isNew: boolean,
-  tried: { ok: boolean; text: string; to?: string; from?: string } | null = null) {
+/**
+ * The settings of one provider only, in groups (a new service, or an existing one switching provider).
+ * (sprint 118) `ws`: a workspace's own service: no name (it is the workspace's), no base URL or token URL
+ * (the providers' own APIs only), and the checks go to the workspace's page.
+ */
+export function serviceForm(x: Partial<MailService> & { has_secret?: boolean }, provider: Provider, action: string, csrfField: ReturnType<typeof csrf>, isNew: boolean,
+  tried: { ok: boolean; text: string; to?: string; from?: string } | null = null, ws: { id: number } | null = null) {
+  const checkAction = ws ? `${BASE}/workspaces/${ws.id}/mail/check` : `${BASE}/mail/check`;
   const s: ServiceSettings = x.provider === provider ? x.settings ?? {} : { default_from: x.settings?.default_from };
   const group = (legend: string, body: unknown, help?: string) =>
     html`<fieldset class="prop-group"><legend>${legend}</legend>${help ? html`<p class="muted u-mt0">${help}</p>` : ''}${body}</fieldset>`;
   const oauth = (scope: string) => html`<div class="form-grid">
       ${input('tenant_id', 'Tenant ID (Microsoft Entra)', s.tenant_id, { placeholder: 'contoso.onmicrosoft.com or its GUID' })}
       ${input('client_id', 'Client (application) ID', s.client_id)}
-      ${input('token_url', 'Token URL', s.token_url, { placeholder: 'empty: Microsoft Entra of the tenant', help: 'Another identity provider\'s token endpoint (client credentials grant), e.g. Google\'s.' })}
+      ${ws ? '' : input('token_url', 'Token URL', s.token_url, { placeholder: 'empty: Microsoft Entra of the tenant', help: 'Another identity provider\'s token endpoint (client credentials grant), e.g. Google\'s.' })}
       ${input('scope', 'Scope', s.scope, { placeholder: `empty: ${scope}` })}
     </div>`;
   const settings =
@@ -92,11 +97,11 @@ function serviceForm(x: Partial<MailService> & { has_secret?: boolean }, provide
         help: x.has_secret && x.provider === provider ? 'A secret is stored (encrypted). It is never shown; type a new one to replace it.' : 'Stored encrypted; never shown again.' })}
       ${x.has_secret ? check('remove_secret', 'Remove the stored secret', false) : ''}
       ${secretKeyConfigured() ? '' : html`<div class="alert alert-error" role="alert">The server has no <code>PGKILN_SECRET_KEY</code>: secrets can't be stored until it is set.</div>`}`);
-  const viaHttp = provider !== 'smtp' && provider !== 'log';
+  const viaHttp = provider !== 'smtp' && provider !== 'log' && !ws;
   return html`<form method="post" action="${action}" class="mail-service-form">${csrfField}
     <input type="hidden" name="provider" value="${provider}">
     ${group('Service', html`<div class="form-grid">
-      ${isNew ? input('name', 'Name', '', { required: true, placeholder: 'e.g. OFFICE365', help: 'Upper case letters, digits and _.' }) : ''}
+      ${isNew && !ws ? input('name', 'Name', '', { required: true, placeholder: 'e.g. OFFICE365', help: 'Upper case letters, digits and _.' }) : ''}
       ${input('description', 'Description', x.description)}
     </div>${check('enabled', 'Enabled', x.enabled !== false, 'A disabled service sends nothing: mail stays queued until it is enabled, or fails after its retries.')}`)}
     ${group('Senders', html`<div class="form-grid">
@@ -110,20 +115,20 @@ function serviceForm(x: Partial<MailService> & { has_secret?: boolean }, provide
       ${input('base_url', 'API base URL', s.base_url, { placeholder: 'empty: the provider\'s own API', help: 'A gateway or proxy that speaks the provider\'s API. Administrators only.' })}</div></details>` : ''}
     ${group('Check before saving', html`
       ${x.id ? html`<input type="hidden" name="service_id" value="${x.id}">` : ''}
-      <div class="buttons mail-check-buttons"><button class="btn" formaction="${BASE}/mail/check" formnovalidate name="check" value="connection">${icon('play')} Test connection</button></div>
+      <div class="buttons mail-check-buttons"><button class="btn" formaction="${checkAction}" formnovalidate name="check" value="connection">${icon('play')} Test connection</button></div>
       <div class="form-grid">
         ${input('test_to', 'Send a test mail to', tried?.to ?? '', { type: 'email', placeholder: 'you@example.com' })}
         ${input('test_from', 'From', tried?.from ?? s.default_from ?? (x.senders ?? []).find((a) => !a.startsWith('@')) ?? '', { type: 'email', help: 'One of the allowed senders.' })}
       </div>
-      <div class="buttons mail-check-buttons"><button class="btn" formaction="${BASE}/mail/check" formnovalidate name="check" value="send">${icon('send')} Send test mail</button></div>
+      <div class="buttons mail-check-buttons"><button class="btn" formaction="${checkAction}" formnovalidate name="check" value="send">${icon('send')} Send test mail</button></div>
       <div class="mail-check-result" role="status" aria-live="polite">${tried ? html`<div class="alert ${tried.ok ? 'alert-success' : 'alert-error'}">${tried.text}</div>` : ''}</div>`,
       `Uses the settings above as they are filled in, without saving them.${x.has_secret ? ' An empty secret field uses the stored secret.' : ''} Test connection signs in (SMTP) or makes a read-only call with the key; a test mail is really sent.`)}
-    <div class="buttons">${isNew ? html`<a class="btn" href="${BASE}/mail/new">Choose another provider</a>` : ''}<button class="btn btn-hot">${isNew ? 'Add service' : 'Save'}</button></div>
+    <div class="buttons">${isNew && !ws ? html`<a class="btn" href="${BASE}/mail/new">Choose another provider</a>` : ''}<button class="btn btn-hot">${isNew ? 'Add service' : 'Save'}</button></div>
   </form>`;
 }
 
 /** The form's values for the chosen provider, as typed (not checked). */
-function parseForm(b: Record<string, string | undefined>) {
+export function parseForm(b: Record<string, string | undefined>) {
   const provider = (PROVIDERS as readonly string[]).includes(b.provider ?? '') ? (b.provider as Provider) : 'smtp';
   const t = (k: string) => (b[k] ?? '').trim() || undefined;
   const senders = [...new Set((b.senders ?? '').split(/[\s,;]+/).map((x) => x.trim().toLowerCase()).filter(Boolean))];
@@ -148,8 +153,13 @@ function parseForm(b: Record<string, string | undefined>) {
  * The form's values, checked; throws with a message. A check of the connection
  * (opts.connection) needs the provider's settings only, not yet the senders.
  */
-function formValues(b: Record<string, string | undefined>, hasSecret: boolean, opts: { connection?: boolean } = {}) {
+export function formValues(b: Record<string, string | undefined>, hasSecret: boolean, opts: { connection?: boolean; workspace?: boolean } = {}) {
   const v = parseForm(b);
+  // (sprint 118) a workspace's own service: the providers' own APIs only
+  if (opts.workspace) {
+    delete v.settings.base_url;
+    delete v.settings.token_url;
+  }
   if (!opts.connection || v.senders.length) {
     if (!v.senders.length) throw new Error('Name at least one allowed sender (an address, or @domain).');
     const bad = v.senders.find((x) => !SENDER.test(x));
@@ -185,13 +195,49 @@ async function adminOnly(s: Session, reply: any, title: string) {
 
 const MAIL_COLUMNS = 'q.id, q.app_id, q.to_addresses, q.cc_addresses, q.bcc_addresses, q.subject, q.template, q.from_address, q.attempts, q.status, q.message, q.created_at, q.sent_at';
 
+/**
+ * Test connection / Send test mail with the settings as filled in (the stored secret when none is typed).
+ * (sprint 118) `workspaceId`: a workspace's own service, with the checks of serviceSettings().
+ */
+export async function runCheck(b: Record<string, string | undefined>, stored: MailService | null, username: string | null, workspaceId: number | null = null): Promise<{ ok: boolean; text: string }> {
+  const typed = parseForm(b);
+  const to = String(b.test_to ?? '').trim();
+  const from = String(b.test_from ?? '').trim();
+  try {
+    const sameProvider = stored?.provider === typed.provider;
+    const v = formValues(b, !!(sameProvider && stored?.secret_enc), { connection: b.check !== 'send', workspace: workspaceId !== null });
+    let secret: string | null = b.secret || null;
+    if (!secret && sameProvider && stored?.secret_enc) {
+      try {
+        secret = decryptSecret(stored.secret_enc);
+      } catch (e) {
+        throw new MailError(`The stored secret can't be read: ${(e as Error).message}`, false);
+      }
+    }
+    const svc = { name: stored?.name ?? 'new', provider: v.provider, settings: v.settings, workspace_id: workspaceId };
+    if (b.check === 'send') {
+      if (!ADDRESS.test(to)) throw new MailError('Give the address to send the test mail to.', false);
+      if (!ADDRESS.test(from) || !senderAllowed(v.senders, from)) throw new MailError('The From address must be one of the allowed senders.', false);
+      const pid = await deliver(v.provider, await serviceSettings(svc), secret, {
+        from, to: [to], cc: [], bcc: [], replyTo: null, subject: `pgkiln: test of a mail service (${PROVIDER_LABELS[v.provider]})`,
+        text: `This is a test of a pgkiln mail service${stored ? ` (${stored.name})` : ''}, sent from the App Builder by ${username}.`, html: null, attachments: [],
+      });
+      return { ok: true, text: `The test mail was handed to ${PROVIDER_LABELS[v.provider]}${pid ? ` (its id: ${pid})` : ''}. Check the mailbox of ${to}.` };
+    }
+    return { ok: true, text: await verifyService(svc, secret) };
+  } catch (e) {
+    return { ok: false, text: e instanceof MailError || !(e instanceof TypeError) ? (e as Error).message : 'The check failed.' };
+  }
+}
+
 export async function mailRoutes(app: FastifyInstance) {
   app.get(`${BASE}/mail`, async (req: Req, reply) => {
     const s = await developer(req, reply);
     if (!s || !(await adminOnly(s, reply, 'Mail services'))) return;
     const services = (await owner.query(`select s.id, s.name, s.description, s.provider, s.senders, s.enabled, s.secret_enc is not null as has_secret,
-        (select string_agg(a.alias, ', ' order by a.alias) from meta.app_mail_service x join meta.app a on a.id = x.app_id where x.service_id = s.id) as apps
-      from meta.mail_service s order by s.name`)).rows;
+        (select string_agg(a.alias, ', ' order by a.alias) from meta.app_mail_service x join meta.app a on a.id = x.app_id where x.service_id = s.id) as apps,
+        (select w.name from meta.workspace w where w.id = s.workspace_id) as workspace
+      from meta.mail_service s order by s.workspace_id is not null, s.name`)).rows;
     const recent = (await owner.query(`select ${MAIL_COLUMNS}, a.alias as app from meta.mail_queue q left join meta.app a on a.id = q.app_id order by q.id desc limit 100`)).rows;
     const main = html`
       <div class="title-row"><h1>Mail services</h1></div>
@@ -203,7 +249,7 @@ export async function mailRoutes(app: FastifyInstance) {
             <td data-label="Provider">${PROVIDER_LABELS[x.provider as Provider]}</td>
             <td data-label="Senders" class="u-break">${x.senders.join(', ')}</td>
             <td data-label="Secret">${x.has_secret ? 'stored (encrypted)' : x.provider === 'log' ? '–' : html`<b>missing</b>`}</td>
-            <td data-label="Applications">${x.apps ?? html`<span class="muted">none</span>`}</td>
+            <td data-label="Applications">${x.workspace ? html`<span class="tag">workspace ${x.workspace}</span> ` : ''}${x.apps ?? (x.workspace ? html`<span class="muted">its applications without a service of their own</span>` : html`<span class="muted">none</span>`)}</td>
             <td data-label="Status">${x.enabled ? 'enabled' : html`<b>disabled</b>`}</td></tr>`)
             : html`<tr><td colspan="6" class="empty">No mail services yet.</td></tr>`}</tbody></table></div>`,
         html`<a class="btn btn-sm btn-hot" href="${BASE}/mail/new">${icon('plus')} Add service</a>`)}
@@ -275,12 +321,14 @@ export async function mailRoutes(app: FastifyInstance) {
     const s = await developer(req, reply);
     if (!s || !(await adminOnly(s, reply, 'Mail services'))) return;
     const x = await owner.one<MailService & { has_secret: boolean }>(
-      'select id, name, description, provider, settings, senders, enabled, secret_enc is not null as has_secret from meta.mail_service where id = $1', [req.params.id]);
+      'select id, name, description, provider, settings, senders, enabled, workspace_id, secret_enc is not null as has_secret from meta.mail_service where id = $1', [req.params.id]);
     if (!x) return reply.code(404).send('Not found');
     const provider = (PROVIDERS as readonly string[]).includes(String(req.query.provider)) ? (req.query.provider as Provider) : x.provider;
+    // (sprint 118) a workspace's own service serves that workspace's applications only
     const apps = (await owner.query(`select a.id, a.alias, a.name, x.service_id, x.from_address, x.max_per_day, o.name as other
       from meta.app a left join meta.app_mail_service x on x.app_id = a.id left join meta.mail_service o on o.id = x.service_id and o.id <> $1
-      order by a.alias`, [x.id])).rows;
+      where $2::int is null or meta.app_workspace(a.id) = $2
+      order by a.alias`, [x.id, x.workspace_id ?? null])).rows;
     const recent = (await owner.query(`select ${MAIL_COLUMNS}, a.alias as app from meta.mail_queue q left join meta.app a on a.id = q.app_id
        where q.service = $1 or q.app_id in (select app_id from meta.app_mail_service where service_id = $2) order by q.id desc limit 50`, [x.name, x.id])).rows;
     const main = html`
@@ -334,10 +382,11 @@ export async function mailRoutes(app: FastifyInstance) {
     const b = req.body as Record<string, string | string[] | undefined>;
     const allow = new Set([b.allow ?? []].flat().map(Number).filter(Number.isInteger));
     try {
-      const service = await owner.one<{ senders: string[] }>('select senders from meta.mail_service where id = $1', [req.params.id]);
+      const service = await owner.one<{ senders: string[]; workspace_id: number | null }>('select senders, workspace_id from meta.mail_service where id = $1', [req.params.id]);
       if (!service) throw new Error('Not found.');
       await owner.tx(async (c) => {
-        const apps = (await c.query<{ id: number }>('select id from meta.app')).rows.map((a) => a.id);
+        // (sprint 118) a workspace's own service: that workspace's applications only
+        const apps = (await c.query<{ id: number }>('select id from meta.app where $1::int is null or meta.app_workspace(id) = $1', [service.workspace_id])).rows.map((a) => a.id);
         await c.query('delete from meta.app_mail_service where service_id = $1 and not (app_id = any($2::int[]))', [req.params.id, [...allow]]);
         for (const id of apps.filter((a) => allow.has(a))) {
           const from = String(b[`from_${id}`] ?? '').trim() || null;
@@ -369,30 +418,7 @@ export async function mailRoutes(app: FastifyInstance) {
     const typed = parseForm(b);
     const to = String(b.test_to ?? '').trim();
     const from = String(b.test_from ?? '').trim();
-    let result: { ok: boolean; text: string };
-    try {
-      const sameProvider = stored?.provider === typed.provider;
-      const v = formValues(b, !!(sameProvider && stored?.secret_enc), { connection: b.check !== 'send' });
-      let secret: string | null = b.secret || null;
-      if (!secret && sameProvider && stored?.secret_enc) {
-        try {
-          secret = decryptSecret(stored.secret_enc);
-        } catch (e) {
-          throw new MailError(`The stored secret can't be read: ${(e as Error).message}`, false);
-        }
-      }
-      if (b.check === 'send') {
-        if (!ADDRESS.test(to)) throw new MailError('Give the address to send the test mail to.', false);
-        if (!ADDRESS.test(from) || !senderAllowed(v.senders, from)) throw new MailError('The From address must be one of the allowed senders.', false);
-        const pid = await deliver(v.provider, v.settings, secret, {
-          from, to: [to], cc: [], bcc: [], replyTo: null, subject: `pgkiln: test of a mail service (${PROVIDER_LABELS[v.provider]})`,
-          text: `This is a test of a pgkiln mail service${stored ? ` (${stored.name})` : ''}, sent from the App Builder by ${s.username}.`, html: null, attachments: [],
-        });
-        result = { ok: true, text: `The test mail was handed to ${PROVIDER_LABELS[v.provider]}${pid ? ` (its id: ${pid})` : ''}. Check the mailbox of ${to}.` };
-      } else result = { ok: true, text: await verifyConnection(v.provider, v.settings, secret) };
-    } catch (e) {
-      result = { ok: false, text: e instanceof MailError || !(e instanceof TypeError) ? (e as Error).message : 'The check failed.' };
-    }
+    const result = await runCheck(b, stored ?? null, s.username, stored?.workspace_id ?? null);
     await logActivity({ username: s.username, event: 'mail_check', ip: clientIp(req), detail: `${b.check === 'send' ? 'send' : 'connection'} ${typed.provider}${stored ? ` ${stored.name}` : ''}: ${result.ok ? 'ok' : 'failed'}` });
     if (String(req.headers.accept ?? '').includes('application/json')) return reply.header('cache-control', 'no-store').send(result);
     const x = { id: stored?.id, name: stored?.name, description: typed.description, provider: typed.provider, settings: typed.settings, senders: typed.senders, enabled: typed.enabled,
@@ -416,8 +442,9 @@ export async function mailRoutes(app: FastifyInstance) {
     if (!s) return;
     const a = await owner.one<{ id: number; alias: string; name: string }>('select id, alias, name from meta.app where id = $1', [req.params.id]);
     if (!a) return reply.code(404).send('Not found');
+    // the application's service, else (sprint 118) its workspace's own
     const service = await owner.one(`select s.name, s.provider, s.enabled, x.from_address, x.max_per_day, s.settings->>'default_from' as default_from
-      from meta.app_mail_service x join meta.mail_service s on s.id = x.service_id where x.app_id = $1`, [a.id]);
+      from meta.mail_service s left join meta.app_mail_service x on x.app_id = $1 and x.service_id = s.id where s.id = meta.mail_service_for_app($1)`, [a.id]);
     const today = (await owner.one(`select count(*)::int as n from meta.mail_queue where app_id = $1 and created_at >= date_trunc('day', now() at time zone 'UTC') at time zone 'UTC' and status <> 'cancelled'`, [a.id])).n;
     const status = String(req.query.status ?? '');
     const rows = (await owner.query(`select ${MAIL_COLUMNS} from meta.mail_queue q where q.app_id = $1 and ($2 = '' or q.status = $2) order by q.id desc limit 200`, [a.id, status])).rows;