pgkiln 0.71.0 → 0.72.0

package/src/builder/telemetry.ts

@@ -0,0 +1,206 @@
+import type { FastifyInstance, FastifyReply } from 'fastify';
+import { owner } from '../db.ts';
+import { html, raw, type Raw } from '../html.ts';
+import { newToken } from '../security.ts';
+import { clientIp, logActivity, type Session } from '../session.ts';
+import {
+  instanceTelemetry, refreshTelemetry, saveInstanceTelemetry, saveWorkspaceTelemetry, setMetricsToken, workspaceTelemetry,
+  type InstanceForm, type Origin, type WorkspaceForm,
+} from '../telemetry/config.ts';
+import { destinationStatus, sendTest } from '../telemetry/export.ts';
+import { isAdmin } from './locks.ts';
+import { back, BASE, csrf, developer, flash, input, region, select, send, shell, type Req } from './ui.ts';
+import { workspaceAdmin } from './workspaces.ts';
+
+// (sprint 139, AP-6; APEX: Workspace Utilities → OpenTelemetry) Workspace
+// utilities → OpenTelemetry (administrators): the instance's collector, what
+// it receives, GET /metrics, and each workspace's choice; a workspace's
+// administrators switch its applications off or add a collector of their own
+// on /builder/workspaces/:id/telemetry. Headers (an API key) are write-only:
+// stored encrypted, only their names are shown.
+
+const crumbs: [string, string?][] = [['App Builder', BASE], ['Workspace utilities', `${BASE}/utilities`], ['OpenTelemetry']];
+const ORIGIN: Record<Origin, string> = { builder: 'set here', environment: 'environment variable', default: 'default' };
+
+const check = (name: string, label: string, on: boolean, help?: string) =>
+  html`<div class="field"><label class="check"><input type="checkbox" name="${name}" value="true"${on ? raw(' checked') : ''}> ${label}</label>${help ? html`<small class="help">${help}</small>` : ''}</div>`;
+
+const headersField = (names: string[], help: string) => html`<div class="field"><label class="label" for="f_headers">Headers</label>
+  <textarea id="f_headers" name="headers" rows="2" maxlength="4000" spellcheck="false" autocomplete="off" placeholder="${names.length ? `${names.join(', ')} set: type new headers to replace them` : 'e.g. x-honeycomb-team=…, Authorization=Bearer …'}"></textarea>
+  <small class="help">${help}${names.length ? html` Set now: <b>${names.join(', ')}</b> (values hidden).` : ''}</small>
+  ${names.length ? html`<label class="check"><input type="checkbox" name="remove_headers" value="true"> Remove the headers</label>` : ''}</div>`;
+
+const when = (ms: number | null) => (ms ? new Date(ms).toISOString().replace('T', ' ').slice(0, 19) + ' UTC' : '—');
+
+function statusCell(key: string): Raw {
+  const st = destinationStatus(key);
+  return html`${st.sent} sent${st.dropped ? html`, <span class="otel-problem">${st.dropped} dropped</span>` : ''} · last success ${when(st.lastOkAt)}
+    ${st.lastError ? html`<br><span class="otel-problem">Last problem (${when(st.lastErrorAt)}): ${st.lastError}</span>` : ''}`;
+}
+
+async function adminOnly(s: Session, reply: FastifyReply) {
+  if (await isAdmin(s.username)) return true;
+  await send(reply.code(403), s, shell(s, 'OpenTelemetry', crumbs, html`<h1 class="u-mb1">OpenTelemetry</h1><div class="alert alert-error" role="alert">Only administrators set up OpenTelemetry for the installation.</div>`));
+  return false;
+}
+
+export async function telemetryBuilderRoutes(app: FastifyInstance) {
+  app.get(`${BASE}/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    await refreshTelemetry(true);
+    const t = instanceTelemetry();
+    const token = typeof s.state.__METRICS_TOKEN === 'string' ? s.state.__METRICS_TOKEN : null;
+    if (token) {
+      delete s.state.__METRICS_TOKEN;
+      const { saveState } = await import('../session.ts');
+      await saveState(s);
+    }
+    const workspaces = (await owner.query<{ id: number; name: string; telemetry: string; telemetry_endpoint: string | null; apps: number }>(
+      `select w.id, w.name, w.telemetry, w.telemetry_endpoint, (select count(*) from meta.app a where meta.app_workspace(a.id) = w.id)::int as apps
+         from meta.workspace w order by w.name`)).rows;
+    const from = (k: string) => html`<span class="muted">(${ORIGIN[t.origin[k] ?? 'default']})</span>`;
+    const pub = (process.env.PUBLIC_URL ?? '').replace(/\/+$/, '');
+    const main = html`<h1 class="u-mb1">OpenTelemetry</h1>
+      <p class="muted u-mt0">Send traces, logs and metrics of the applications to your own monitoring (Grafana, Jaeger, Datadog, Honeycomb, OCI APM,
+        or anything that speaks OTLP) through an OpenTelemetry Collector. A trace per page request, with a span for each SQL statement, REST data source call,
+        AI request and tool call; mail and workflow steps as traces of their own. pgkiln sends OTLP over HTTP as JSON. The standard <code>OTEL_*</code>
+        environment variables work too; a value set here wins. Chapter 1 of the guide (Monitoring with OpenTelemetry) explains every choice.</p>
+      ${t.protocolWarning ? html`<div class="alert alert-warning" role="status">${t.protocolWarning}</div>` : ''}
+      <div class="columns wide-left">
+        ${region('Collector', html`<form method="post" action="${BASE}/telemetry">${csrf(s)}
+          ${check('enabled', 'Send to this collector', t.enabled, 'Off: nothing leaves the server (workspaces with a collector of their own still send to it).')}
+          <div class="form-grid">
+            ${input('endpoint', 'Collector URL', t.origin.otel_endpoint === 'builder' ? t.endpoint : '', { placeholder: t.endpoint || 'http://otel-collector:4318', help: html`The OTLP/HTTP base address; /v1/traces, /v1/logs and /v1/metrics are added. In effect: <b>${t.endpoint || 'none'}</b> ${from('otel_endpoint')} (<code>OTEL_EXPORTER_OTLP_ENDPOINT</code>).` as unknown as string })}
+            ${input('service_name', 'Service name', t.origin.otel_service_name === 'builder' ? t.serviceName : '', { placeholder: t.serviceName, help: `service.name of everything sent (OTEL_SERVICE_NAME). In effect: ${t.serviceName}.` })}
+            ${input('sample', 'Sample ratio', t.origin.otel_sample === 'builder' ? String(t.sample) : '', { placeholder: String(t.sample), help: 'Share of traces sent, 0 to 1 (OTEL_TRACES_SAMPLER_ARG). A trace with an error is always sent. A caller\'s traceparent decides for its trace.' })}
+            ${select('user', 'User', t.user, [['hashed', 'Hashed (the same user gives the same value, the name is not sent)'], ['name', 'User name'], ['none', 'Not sent']], 'enduser.id on the page span and on logs.')}
+          </div>
+          ${headersField(t.headerNames, 'Sent with every request to the collector, name=value, one per line or comma separated (OTEL_EXPORTER_OTLP_HEADERS). Stored encrypted.')}
+          <fieldset class="field"><legend class="label">Send</legend><div class="radio-group">
+            ${check('traces', 'Traces', t.traces)}${check('logs', 'Logs: errors, refused access and failed sign-ins', t.logs)}${check('metrics', `Metrics, every ${Math.round(t.intervalMs / 1000)} seconds`, t.metrics)}
+          </div></fieldset>
+          ${check('values', 'Include values', t.values, 'SQL statements with their literals, bind values, query strings and AI tool arguments. Off: literals become ?, values are left out and error messages have quoted values removed.')}
+          ${check('builder', 'Trace builder requests too', t.builder, 'Applications only by default.')}
+          ${check('browser', 'Browser spans', t.browser, 'Pages report their load time and Ajax calls to pgkiln, which adds them to the page\'s trace. The browser never sees the collector or its headers.')}
+          ${check('prometheus', 'Prometheus endpoint /metrics', t.prometheus, 'For administrators signed in to the builder, or a scraper with the metrics token below (PGKILN_METRICS, PGKILN_METRICS_TOKEN).')}
+          <div class="buttons"><button class="btn btn-hot">Save</button></div></form>
+          <form method="post" action="${BASE}/telemetry/test" class="u-mt1">${csrf(s)}<button class="btn">Send a test span</button></form>`)}
+        ${region('Status', html`<p class="muted u-mt0">Since this server started. Each server of the installation sends its own.</p>
+          <div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Collector</th><th>Sending</th></tr></thead><tbody>
+            <tr><td data-label="Collector">This instance's${t.enabled ? '' : html` <span class="muted">(off)</span>`}</td><td data-label="Sending">${statusCell('instance')}</td></tr>
+            ${workspaces.filter((w) => w.telemetry === 'own').map((w) => html`<tr><td data-label="Collector">Workspace ${w.name}</td><td data-label="Sending">${statusCell(`ws:${w.id}`)}</td></tr>`)}
+          </tbody></table></div>`)}
+        ${region('Prometheus', html`<p class="muted u-mt0">${t.prometheus ? html`<code>GET ${pub || ''}/metrics</code> answers with the counters, durations and queue lengths in the Prometheus text format.` : 'Off: switch it on above.'}
+            A scraper sends <code>Authorization: Bearer &lt;token&gt;</code>. ${t.prometheusTokenHash ? (t.origin.metrics_token_hash === 'builder' || !process.env.PGKILN_METRICS_TOKEN ? 'A token is set.' : 'The token is PGKILN_METRICS_TOKEN.') : 'No token is set.'}</p>
+          ${token ? html`<div class="alert alert-success" role="status">The new token (shown once): <code class="cell-break">${token}</code></div>` : ''}
+          <form method="post" action="${BASE}/telemetry/token" class="buttons">${csrf(s)}<button class="btn" name="action" value="new">${t.prometheusTokenHash ? 'Replace the token' : 'Make a token'}</button>
+            ${t.prometheusTokenHash ? html`<button class="btn btn-danger" name="action" value="remove" data-confirm="Remove the metrics token? Scrapers that use it get 401.">Remove the token</button>` : ''}</form>`)}
+        ${region('Workspaces', html`<p class="muted u-mt0">A workspace's administrators may switch its applications off, or send them to a collector of their own as well.</p>
+          <div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Workspace</th><th>Applications</th><th>Sends</th></tr></thead><tbody>
+          ${workspaces.length ? workspaces.map((w) => html`<tr><td data-label="Workspace"><a href="${BASE}/workspaces/${w.id}/telemetry">${w.name}</a></td><td data-label="Applications" class="num">${w.apps}</td>
+            <td data-label="Sends" class="cell-break">${w.telemetry === 'off' ? 'nothing' : w.telemetry === 'own' ? html`to the instance's collector and ${w.telemetry_endpoint}` : 'to the instance\'s collector'}</td></tr>`)
+            : html`<tr><td colspan="3" class="empty">No workspaces.</td></tr>`}</tbody></table></div>`)}
+      </div>`;
+    return send(reply, s, shell(s, 'OpenTelemetry', crumbs, main));
+  });
+
+  app.post(`${BASE}/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    const problem = await saveInstanceTelemetry((req.body ?? {}) as InstanceForm);
+    if (!problem) {
+      const t = instanceTelemetry();
+      await logActivity({ username: s.username, event: 'instance_settings', ip: clientIp(req),
+        detail: `OpenTelemetry: ${t.enabled ? `on, ${t.endpoint}` : 'off'}, sample ${t.sample}, user ${t.user}, values ${t.values}, browser ${t.browser}, /metrics ${t.prometheus}` });
+    }
+    flash(s, problem ?? 'OpenTelemetry settings saved.', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/telemetry`);
+  });
+
+  app.post(`${BASE}/telemetry/test`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    const problem = await sendTest(null);
+    flash(s, problem ? `The collector did not take the test span: ${problem}` : 'The collector took the test span ("pgkiln test span").', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/telemetry`);
+  });
+
+  app.post(`${BASE}/telemetry/token`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await adminOnly(s, reply))) return;
+    const action = (req.body as { action?: string } | undefined)?.action;
+    if (action === 'new') {
+      const token = `pgkm_${newToken()}`;
+      await setMetricsToken(token);
+      s.state.__METRICS_TOKEN = token;
+      await logActivity({ username: s.username, event: 'instance_settings', ip: clientIp(req), detail: 'metrics token made' });
+    } else if (action === 'remove') {
+      await setMetricsToken(null);
+      await logActivity({ username: s.username, event: 'instance_settings', ip: clientIp(req), detail: 'metrics token removed' });
+      flash(s, 'The metrics token is removed.');
+    }
+    return back(reply, s, `${BASE}/telemetry`);
+  });
+
+  // ---------------------------------------------------------------- a workspace's own
+
+  const wsAllowed = async (s: Session, reply: FastifyReply, id: string) => {
+    if (await workspaceAdmin(s.username, id)) return true;
+    await send(reply.code(403), s, shell(s, 'OpenTelemetry', [['Administration'], ['OpenTelemetry']],
+      html`<h1 class="u-mb1">OpenTelemetry</h1><div class="alert alert-error" role="alert">Only the workspace's administrators change its OpenTelemetry setting.</div>`, 'users'));
+    return false;
+  };
+
+  app.get(`${BASE}/workspaces/:id(^\\d{1,9}$)/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await wsAllowed(s, reply, req.params.id))) return;
+    const w = await owner.one<{ id: number; name: string; telemetry: string; telemetry_endpoint: string | null; telemetry_values: boolean; has_headers: boolean }>(
+      'select id, name, telemetry, telemetry_endpoint, telemetry_values, telemetry_headers_enc is not null as has_headers from meta.workspace where id = $1', [req.params.id]);
+    if (!w) return reply.code(404).send('Not found');
+    await refreshTelemetry(true);
+    const t = instanceTelemetry();
+    const names = Object.keys(workspaceTelemetry(w.id)?.headers ?? {});
+    const radio = (v: string, label: string, help: string) => html`<label class="check"><input type="radio" name="mode" value="${v}"${w.telemetry === v ? raw(' checked') : ''}> ${label} <span class="muted small">${help}</span></label>`;
+    const main = html`<div class="title-row"><h1>OpenTelemetry · ${w.name}</h1></div>
+      <p class="muted u-mt0">Traces and logs of this workspace's applications. ${t.enabled ? 'The instance sends them to its collector.' : 'The instance sends nothing at the moment.'}
+        A collector of the workspace's own is reached through the installation's allow-list and this workspace's <a href="${BASE}/network?ws=${w.id}">network access</a> list.</p>
+      <div class="columns wide-left">
+        ${region('Setting', html`<form method="post" action="${BASE}/workspaces/${w.id}/telemetry">${csrf(s)}
+          <fieldset class="field"><legend class="label">The applications send</legend><div class="radio-group">
+            ${radio('instance', 'As the instance does', '(default)')}
+            ${radio('off', 'Nothing', 'not even to the instance\'s collector')}
+            ${radio('own', 'Also to our own collector', '')}
+          </div></fieldset>
+          <div class="form-grid">${input('endpoint', 'Own collector URL', w.telemetry_endpoint ?? '', { placeholder: 'https://otlp.example.com', help: 'The OTLP/HTTP base address; /v1/traces and /v1/logs are added.' })}</div>
+          ${headersField(names.length ? names : w.has_headers ? ['(unreadable)'] : [], 'Sent with every request to the own collector (an API key). Stored encrypted.')}
+          ${check('values', 'Include values for the own collector', w.telemetry_values, 'SQL statements with their literals, bind values and query strings.')}
+          <div class="buttons"><button class="btn btn-hot">Save</button></div></form>
+          ${w.telemetry === 'own' ? html`<form method="post" action="${BASE}/workspaces/${w.id}/telemetry/test" class="u-mt1">${csrf(s)}<button class="btn">Send a test span</button></form>` : ''}`)}
+        ${w.telemetry === 'own' ? region('Status', html`<p class="u-mt0">${statusCell(`ws:${w.id}`)}</p>`) : ''}
+      </div>`;
+    return send(reply, s, shell(s, `OpenTelemetry · ${w.name}`, [['Administration'], [w.name, `${BASE}/workspaces/${w.id}`], ['OpenTelemetry']], main, 'users'));
+  });
+
+  app.post(`${BASE}/workspaces/:id(^\\d{1,9}$)/telemetry`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await wsAllowed(s, reply, req.params.id))) return;
+    const id = Number(req.params.id);
+    if (!(await owner.one('select 1 from meta.workspace where id = $1', [id]))) return reply.code(404).send('Not found');
+    const f = (req.body ?? {}) as WorkspaceForm;
+    const problem = await saveWorkspaceTelemetry(id, f);
+    if (!problem)
+      await logActivity({ username: s.username, event: 'workspace', ip: clientIp(req), detail: `workspace ${id} OpenTelemetry: ${f.mode}${f.mode === 'own' ? ` ${f.endpoint}` : ''}` });
+    flash(s, problem ?? 'Saved.', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/workspaces/${id}/telemetry`);
+  });
+
+  app.post(`${BASE}/workspaces/:id(^\\d{1,9}$)/telemetry/test`, async (req: Req, reply) => {
+    const s = await developer(req, reply);
+    if (!s || !(await wsAllowed(s, reply, req.params.id))) return;
+    const id = Number(req.params.id);
+    const problem = workspaceTelemetry(id)?.mode === 'own' ? await sendTest(id) : 'The workspace has no collector of its own.';
+    flash(s, problem ? `The collector did not take the test span: ${problem}` : 'The collector took the test span ("pgkiln test span").', problem ? 'error' : 'ok');
+    return back(reply, s, `${BASE}/workspaces/${id}/telemetry`);
+  });
+}

package/src/builder/workspaces.ts

@@ -385,7 +385,8 @@ export async function workspaceRoutes(app: FastifyInstance) {
             <div class="buttons"><button class="btn">${tx('wsp.save_devs')}</button></div></form>`)}
       </div>
       ${serviceRegion(s, w)}
-      ${region(tx('wsp.mail_net'), html`<p class="u-mt0">${tx.rich('wsp.mail_net_text', { mail: html`<a href="${BASE}/workspaces/${w.id}/mail">${tx('wsp.mail_service')}</a>`, network: html`<a href="${BASE}/network?ws=${w.id}">${tx('wsp.network')}</a>`, queue: html`<a href="${BASE}/mail-queue${full ? `?ws=${w.id}` : ''}">${tx('wsp.queue')}</a>` })}</p>`, '', 'mail-network')}
+      ${region(tx('wsp.mail_net'), html`<p class="u-mt0">${tx.rich('wsp.mail_net_text', { mail: html`<a href="${BASE}/workspaces/${w.id}/mail">${tx('wsp.mail_service')}</a>`, network: html`<a href="${BASE}/network?ws=${w.id}">${tx('wsp.network')}</a>`, queue: html`<a href="${BASE}/mail-queue${full ? `?ws=${w.id}` : ''}">${tx('wsp.queue')}</a>` })}</p>
+        <p>${tx.rich('wsp.telemetry_text', { link: html`<a href="${BASE}/workspaces/${w.id}/telemetry">${tx('wsp.telemetry')}</a>` })}</p>`, '', 'mail-network')}
       <div class="columns">${preferencesRegion(s, w, without, unlinked)}${linksRegion(s, w, links)}</div>
       ${full ? isolationRegion(s, w, roles.rows) : region(tx('wsp.isolation'), html`<p class="u-mt0">${w.isolated ? tx('wsp.iso_short_on') : tx('wsp.iso_short_off')} ${tx('wsp.admins_change')}</p>`)}
       ${!full ? '' : region(tx('wsp.applications'), html`<div class="table-wrap"><table class="report report-reflow">

package/src/instance-checks.ts

@@ -36,16 +36,23 @@ export interface SecretColumn {
   schema: string;
   table: string;
   column: string;
+  /** (sprint 139) only the rows that match (meta.setting: the settings named *_enc) */
+  where?: string;
 }
 
+/** (sprint 139) instance settings that hold a secret: meta.instance_setting rows named *_enc (otel_headers_enc) */
+const SETTING_SECRETS: SecretColumn = { schema: 'meta', table: 'instance_setting', column: 'value', where: `name like '%\\_enc'` };
+
 export async function encryptedColumns(): Promise<SecretColumn[]> {
-  return (await owner.query<SecretColumn>(
+  return [...(await owner.query<SecretColumn>(
     `select table_schema as schema, table_name as table, column_name as column from information_schema.columns
       where table_schema = 'meta' and data_type = 'text'
         and (column_name like '%\\_enc' or (table_name = 'push_key' and column_name = 'private_key'))
-      order by 2, 3`)).rows;
+      order by 2, 3`)).rows, SETTING_SECRETS];
 }
 
+const rowsOf = (c: SecretColumn) => `${quoteIdent(c.column)} is not null${c.where ? ` and (${c.where})` : ''}`;
+
 const quoteIdent = (s: string) => `"${s.replace(/"/g, '""')}"`;
 const tableOf = (c: SecretColumn) => `${quoteIdent(c.schema)}.${quoteIdent(c.table)}`;
 
@@ -54,7 +61,7 @@ export async function secretStatus(withKey?: string) {
   const out: (SecretColumn & { stored: number; unreadable: number })[] = [];
   const usable = (withKey ?? process.env.PGKILN_SECRET_KEY ?? '').length >= 32;
   for (const c of await encryptedColumns()) {
-    const rows = (await owner.query<{ v: string }>(`select ${quoteIdent(c.column)} as v from ${tableOf(c)} where ${quoteIdent(c.column)} is not null`)).rows;
+    const rows = (await owner.query<{ v: string }>(`select ${quoteIdent(c.column)} as v from ${tableOf(c)} where ${rowsOf(c)}`)).rows;
     let unreadable = 0;
     for (const r of rows) {
       try {
@@ -228,7 +235,7 @@ export async function rotateKey(oldKey: string, newKey: string, { dryRun = false
     const failed: string[] = [];
     for (const col of cols) {
       const rows = (await c.query<{ ctid: string; v: string }>(
-        `select ctid::text as ctid, ${quoteIdent(col.column)} as v from ${tableOf(col)} where ${quoteIdent(col.column)} is not null for update`)).rows;
+        `select ctid::text as ctid, ${quoteIdent(col.column)} as v from ${tableOf(col)} where ${rowsOf(col)} for update`)).rows;
       let rotated = 0;
       let already = 0;
       for (const r of rows) {

package/src/layout.ts

@@ -1,5 +1,6 @@
 import { html, type Raw } from './html.ts';
 import { staticUrl } from './static-version.ts';
+import { browserTraceMeta } from './telemetry/browser.ts';
 
 /** Attributes of <html>: language, text direction and a fixed theme (light/dark; absent = follow the OS). */
 export interface RootAttrs {
@@ -20,7 +21,7 @@ export function documentShell(title: string, body: Raw, bodyClass = '', data: Re
 <title>${title}</title>
 <link rel="stylesheet" href="${staticUrl('app.css')}">
 <script src="${staticUrl('app.js')}" defer></script>
-${head}
+${browserTraceMeta()}${head}
 </head>
 <body class="${bodyClass}"${attrs}>
 ${body}

package/src/mail/mail.ts

@@ -14,6 +14,7 @@ import { owner } from '../db.ts';
 import { decryptSecret } from '../secrets.ts';
 import { mailHostAddress, scopeForWorkspace } from '../network-acl.ts';
 import { deliver, MailError, verifyConnection, type OutgoingMail, type Provider, type ServiceSettings } from './providers.ts';
+import { count, markError, rootSpan, setAttrs, traceApp } from '../telemetry/trace.ts';
 
 export interface MailService {
   id: number;
@@ -95,6 +96,14 @@ interface Claimed {
 }
 
 async function sendOne(m: Claimed) {
+  // (sprint 139, AP-6) each attempt a trace of its own (or a span of the request that sends at once)
+  return rootSpan('mail send', { 'pgkiln.mail.id': String(m.id), 'pgkiln.mail.attempt': m.attempts }, async () => {
+    traceApp({ appId: m.app_id });
+    await sendOneTraced(m);
+  });
+}
+
+async function sendOneTraced(m: Claimed) {
   // the application's service, else (sprint 118) its workspace's own
   const service = await owner.one<MailService>(`select s.* from meta.mail_service s where s.id = meta.mail_service_for_app($1)`, [m.app_id]);
   const t0 = performance.now();
@@ -112,10 +121,15 @@ async function sendOne(m: Claimed) {
     await owner.query(`update meta.mail_queue set status = 'sent', sent_at = now(), provider_id = $2, service = $3, message = null,
                               last_attempt_at = now(), duration_ms = $4, provider = $5 where id = $1`,
       [m.id, id ? String(id).slice(0, 500) : null, service.name, ms, provider]);
+    setAttrs({ 'pgkiln.mail.provider': provider, 'pgkiln.mail.service': service.name });
+    count('pgkiln.mail.sent', { status: 'sent' });
   } catch (e) {
     const retry = e instanceof MailError ? e.retry : true;
     const message = (e as Error).message.slice(0, 2000);
     const [ms, provider] = timing();
+    setAttrs({ 'pgkiln.mail.provider': provider, 'pgkiln.mail.service': service?.name ?? null, 'pgkiln.mail.retry': retry && m.attempts < ATTEMPTS });
+    markError(message);
+    count('pgkiln.mail.sent', { status: retry && m.attempts < ATTEMPTS ? 'retry' : 'error' });
     if (retry && m.attempts < ATTEMPTS)
       await owner.query(`update meta.mail_queue set status = 'queued', not_before = now() + make_interval(mins => $2), message = $3, service = $4,
                                 last_attempt_at = now(), duration_ms = $5, provider = $6 where id = $1`,

package/src/runtime/ai-agents.ts

@@ -8,6 +8,7 @@ import { isAuthorized, pageAllowed } from './authz.ts';
 import { bindValues, dbg, publicError, stripSemicolon, type PageContext } from './context.ts';
 import { txContext } from './routes.ts';
 import { retrieve } from './search.ts';
+import { recordSpan } from '../telemetry/trace.ts';
 
 // AI agents and AI tools (migration 121; APEX 26.1 / 26.2: Shared Components
 // → AI Agents). Both are lists in meta.app (ai_agents, ai_tools), exported
@@ -397,6 +398,14 @@ export async function logToolCall(
   ctx: PageContext, regionId: number, agent: string | null, tool: string, type: string, input: Record<string, unknown>, out: AgentOutcome, ms: number,
 ) {
   const status = out.refused ? 'refused' : out.isError ? 'error' : 'ok';
+  // (sprint 139, AP-6) the tool call as a span of the page's trace (its arguments only where values are included)
+  recordSpan(`AI tool ${tool}`.slice(0, 120), {
+    ms,
+    kind: 1,
+    attrs: { 'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': tool, 'gen_ai.tool.type': type, 'pgkiln.ai.agent': agent, 'pgkiln.ai.tool.status': status },
+    vals: { 'gen_ai.tool.call.arguments': JSON.stringify(input ?? {}).slice(0, 2000) },
+    error: status === 'ok' ? null : status,
+  });
   let args = JSON.stringify(input ?? {});
   if (args.length > MAX_ARG_CHARS) args = JSON.stringify({ cut_off: args.slice(0, MAX_ARG_CHARS) });
   dbg(ctx, 4, 'ai', () => `AI tool ${tool} (${type}${agent ? `, agent ${agent}` : ''}) ${args} → ${status} in ${Math.round(ms)} ms${out.isError ? `: ${out.content.slice(0, 300)}` : ''}`);

package/src/runtime/routes.ts

@@ -55,6 +55,7 @@ import { availabilityRefusal } from './availability.ts';
 import { feedbackRoutes } from './feedback.ts';
 import { dataItemRoutes, loadPreferences, savePreferences } from './data-items.ts';
 import { ajaxRoutes } from './ajax.ts';
+import { traceApp } from '../telemetry/trace.ts';
 
 const XLSX_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
 
@@ -266,6 +267,8 @@ export async function loadContext(req: Req, reply: FastifyReply, { json = false,
   if (!page) return simplePage(reply, 404, locale.t('error.not_found'), locale.t('error.page_not_found', { page: req.params.page, app: app.name }), base, locale), null;
   if (locale.lang !== app.language) translatePage(page, locale.tr);
   const user = app.authentication === 'none' ? 'nobody' : (session.username ?? 'nobody');
+  // (sprint 139, AP-6) the request's trace: which application (its workspace decides where it goes), page and user
+  await traceApp({ appId: app.id, alias: app.alias, page: page.page_no, user });
 
   if (app.authentication !== 'none' && (page.requires_auth || fixedPage) && !session.username) {
     if (json) reply.code(401).send({ error: locale.t('error.session_ended') });

package/src/session.ts

@@ -1,4 +1,6 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
+import { count, emitLog } from './telemetry/trace.ts';
+import { appOf } from './telemetry/config.ts';
 import { runtime, sharedRuntime } from './db.ts';
 import { runtimeForApp } from './workspace-db.ts';
 import { hashToken, loginMaxFailuresPerIp, loginMaxFailuresPerUser, loginWindowMinutes, newToken } from './security.ts';
@@ -121,6 +123,7 @@ export async function logActivity(entry: {
   elapsedMs?: number;
   detail?: string;
 }) {
+  activityTelemetry(entry);
   try {
     // (090) on the connection of the entry's application (an isolated workspace's own); without one, the shared one
     const db = entry.appId != null ? await runtimeForApp(entry.appId) : sharedRuntime;
@@ -152,3 +155,26 @@ export async function loginThrottled(appId: number | null, username: string, ip:
   );
   return !!counts && (counts.user_fails >= loginMaxFailuresPerUser() || counts.ip_fails >= loginMaxFailuresPerIp());
 }
+
+// (sprint 139, AP-6) errors, refusals and failed sign-ins as OpenTelemetry logs and metrics
+const OTEL_EVENTS: Record<string, ['ERROR' | 'WARN', string]> = {
+  error: ['ERROR', 'pgkiln.errors'],
+  forbidden: ['WARN', 'pgkiln.access_denied'],
+  login_failed: ['WARN', 'pgkiln.signin_failures'],
+  login_locked: ['WARN', 'pgkiln.signin_failures'],
+};
+
+function activityTelemetry(e: { appId?: number | null; pageNo?: number | null; username?: string | null; event: string; detail?: string }) {
+  const known = OTEL_EVENTS[e.event];
+  if (!known) return;
+  const app = e.appId == null ? 'builder' : appOf(e.appId)?.[0] ?? 'unknown';
+  count(known[1], e.event === 'login_locked' ? { app, reason: 'locked' } : { app });
+  emitLog({
+    severity: known[0],
+    body: e.event,
+    detail: e.detail,
+    attrs: { 'pgkiln.event': e.event, 'pgkiln.app.id': e.appId ?? null, 'pgkiln.app.alias': e.appId == null ? null : app, 'pgkiln.page': e.pageNo == null ? null : String(e.pageNo) },
+    appId: e.appId ?? null,
+    user: e.username ?? null,
+  });
+}

package/src/telemetry/browser.ts

@@ -0,0 +1,91 @@
+import { html, type Raw } from '../html.ts';
+import { checksumValid, signText } from '../security.ts';
+import { browserSpans } from './export.ts';
+import { currentCtx, now, SPAN_KIND, type Span } from './trace.ts';
+
+// (sprint 139, AP-6) Browser spans: a page whose trace is sent carries a signed
+// <meta name="pgkiln-trace"> (layout.ts); app.js reports the page load and the
+// page's Ajax calls to POST /a/:alias/telemetry (src/telemetry/http.ts), and
+// relay() turns them into spans of the page's trace. The token is bound to the
+// trace, the page's span, the time and the application, and is good for 30
+// minutes and 200 spans. No collector address or token reaches the browser.
+
+const TOKEN_MINUTES = 30;
+const MAX_PER_TOKEN = 200;
+const used = new Map<string, number>();
+let usedSince = Date.now();
+
+/** The page's <meta> for browser spans: its trace, span, time and a signature (empty when not wanted). */
+export function browserTraceMeta(): Raw | '' {
+  const ctx = currentCtx();
+  if (!ctx?.trace.browser || !ctx.trace.sampled || !ctx.trace.alias) return '';
+  const root = ctx.trace.spans[0];
+  const ts = Math.floor(Date.now() / 1000);
+  const payload = `${root.traceId}.${root.spanId}.${ts}`;
+  return html`<meta name="pgkiln-trace" content="${payload}.${signText('otel-browser', `${payload}.${ctx.trace.alias.toLowerCase()}`)}">`;
+}
+
+interface BrowserSpan {
+  k?: unknown;
+  path?: unknown;
+  method?: unknown;
+  status?: unknown;
+  start?: unknown;
+  dur?: unknown;
+  t?: Record<string, unknown>;
+}
+
+const num = (v: unknown, max: number) => (typeof v === 'number' && Number.isFinite(v) ? Math.min(Math.max(v, 0), max) : null);
+
+/** Check a page's token and turn its reports into spans. Returns the number kept, or null for a bad token. */
+export function relay(alias: string, token: unknown, spans: unknown): number | null {
+  const m = typeof token === 'string' ? /^([0-9a-f]{32})\.([0-9a-f]{16})\.(\d{1,12})\.([\w-]{22})$/.exec(token) : null;
+  if (!m) return null;
+  const [, traceId, spanId, ts, sig] = m;
+  if (!checksumValid(signText('otel-browser', `${traceId}.${spanId}.${ts}.${alias.toLowerCase()}`), sig)) return null;
+  if (Date.now() / 1000 - Number(ts) > TOKEN_MINUTES * 60) return null;
+  if (Date.now() - usedSince > 3_600_000) {
+    used.clear();
+    usedSince = Date.now();
+  }
+  const list = Array.isArray(spans) ? (spans as BrowserSpan[]).slice(0, 50) : [];
+  const room = MAX_PER_TOKEN - (used.get(`${traceId}.${spanId}`) ?? 0);
+  if (room <= 0 || !list.length) return 0;
+  const prefix = `/a/${alias}/`.toLowerCase();
+  const out: Span[] = [];
+  const t = now();
+  for (const b of list.slice(0, room)) {
+    const dur = num(b.dur, 600_000);
+    const start = num(b.start, 8.64e15);
+    if (dur === null || start === null || Math.abs(start - t) > 3_600_000) continue;
+    const path = typeof b.path === 'string' ? b.path.split('#')[0] : '';
+    const [p, query] = path.split('?');
+    const s: Span = {
+      traceId, spanId: '', parentId: spanId, kind: SPAN_KIND.client, start, end: start + dur, name: '', attrs: { 'pgkiln.browser': true },
+    };
+    if (b.k === 'load') {
+      s.kind = SPAN_KIND.internal;
+      s.name = 'browser page load';
+      for (const k of ['ttfb', 'dom_interactive', 'dom_complete', 'load']) {
+        const v = num(b.t?.[k], 600_000);
+        if (v !== null) s.attrs[`pgkiln.browser.${k}_ms`] = Math.round(v);
+      }
+    } else if (b.k === 'ajax' && p.toLowerCase().startsWith(prefix) && p.length <= 500) {
+      const method = typeof b.method === 'string' && /^(GET|POST|PUT|PATCH|DELETE)$/.test(b.method) ? b.method : 'GET';
+      s.name = `browser ${method}`;
+      s.attrs['http.request.method'] = method;
+      s.attrs['url.path'] = p;
+      const st = num(b.status, 599);
+      if (st !== null) {
+        s.attrs['http.response.status_code'] = Math.round(st);
+        if (st >= 500 || st === 0) s.error = st ? `HTTP ${Math.round(st)}` : 'network error';
+      }
+      if (query) s.vals = { 'url.query': query.slice(0, 2000) };
+    } else continue;
+    out.push(s);
+  }
+  used.set(`${traceId}.${spanId}`, MAX_PER_TOKEN - room + out.length);
+  if (out.length) browserSpans(traceId, alias, out);
+  return out.length;
+}
+