pgkiln 0.52.0 → 0.53.0

package/src/network-acl.ts

@@ -0,0 +1,126 @@
+import dns from 'node:dns/promises';
+import net from 'node:net';
+import { owner } from './db.ts';
+import { isPrivateAddress, matches, restHosts, scopeProblem, urlProblem, type OutboundScope } from './webclient.ts';
+
+// (sprint 118, WS-9; APEX 26.2 workspace network ACLs) A workspace's network
+// access list: when meta.workspace.network_restricted is on, the outgoing web
+// calls made for its applications must match one of meta.workspace.network_hosts
+// as well as the installation's lists (PGKILN_REST_ALLOWED_HOSTS /
+// PGKILN_REST_PRIVATE_HOSTS). It only narrows: the installation's checks
+// (allow-list, private addresses at connect time, redirects) stay as they are.
+//
+// Every caller of src/webclient.ts webRequest() passes a scope built here:
+// websources.ts call() (meta.web_request, REST data sources, write-back,
+// invoke API in pages, workflows and automations, the builder's Test), its
+// token requests, objectstore.ts and the REST source catalogs' "read from a
+// URL". A workspace's own mail service checks its SMTP host with
+// mailHostAddress(). Refusals go to meta.network_refusal (7 days).
+//
+// No cache: one indexed query per call, so a narrowed list applies at once on
+// every server.
+
+export const KEEP_DAYS = 7;
+/** At most this many refusals are kept per workspace. */
+const KEEP_PER_WORKSPACE = 2000;
+
+/** Entries of a network access list: host, *.domain, host:port, [IPv6]:port. */
+export const HOST_ENTRY = /^(?:(?:\*\.)?[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*|\[[0-9a-f:.]{2,45}\])(?::([1-9][0-9]{0,4}))?$/;
+
+/** The entries of a typed list (comma, space or line separated), checked; throws with a message. */
+export function parseHostList(text: string): string[] {
+  const entries = [...new Set(text.split(/[\s,;]+/).map((x) => x.trim().toLowerCase()).filter(Boolean))];
+  for (const e of entries) {
+    const m = HOST_ENTRY.exec(e);
+    if (!m || (m[1] && Number(m[1]) > 65535))
+      throw new Error(`"${e.slice(0, 80)}" is not a host: use api.example.com, *.example.com or api.example.com:8443 (no scheme, no path).`);
+  }
+  if (entries.length > 200) throw new Error('At most 200 entries.');
+  return entries;
+}
+
+interface WorkspaceRow {
+  id: number;
+  name: string;
+  network_restricted: boolean;
+  network_hosts: string[];
+}
+
+function scopeOf(w: WorkspaceRow, appId: number | null, what: string): OutboundScope {
+  return {
+    workspaceId: w.id,
+    workspaceName: w.name,
+    appId,
+    what,
+    hosts: w.network_restricted ? w.network_hosts : null,
+    // a method: a copy of the scope with another "what" (a token endpoint) reports under its own name
+    refused(url, list, message) {
+      void recordRefusal(this.workspaceId, this.appId, this.what, url, list, message);
+    },
+  };
+}
+
+/** The scope of an application's outgoing calls (its workspace's list); null for no application. */
+export async function scopeForApp(appId: number | null | undefined, what: string): Promise<OutboundScope | null> {
+  if (appId === null || appId === undefined || !Number.isInteger(Number(appId))) return null;
+  const w = await owner.one<WorkspaceRow>(
+    'select w.id, w.name, w.network_restricted, w.network_hosts from meta.workspace w where w.id = meta.app_workspace($1)', [appId]);
+  return w ? scopeOf(w, Number(appId), what) : null;
+}
+
+/** The scope of a workspace's own outgoing calls (no application: REST source catalogs, its mail service). */
+export async function scopeForWorkspace(workspaceId: number, what: string): Promise<OutboundScope | null> {
+  const w = await owner.one<WorkspaceRow>('select id, name, network_restricted, network_hosts from meta.workspace where id = $1', [workspaceId]);
+  return w ? scopeOf(w, null, what) : null;
+}
+
+/** Keep a refusal (never throws: a refusal is already an error for the caller). */
+export async function recordRefusal(workspaceId: number | null, appId: number | null, what: string, url: URL, list: 'installation' | 'workspace', message: string) {
+  try {
+    await owner.query(
+      `insert into meta.network_refusal (workspace_id, app_id, what, host, url, list, message) values ($1, $2, $3, $4, $5, $6, $7)`,
+      [workspaceId, appId, what.slice(0, 100), url.host.slice(0, 300), `${url.protocol}//${url.host}${url.pathname}`.slice(0, 1000), list, message.slice(0, 1000)]);
+  } catch {
+    // an application deleted meanwhile, or the database is away: nothing to keep
+  }
+}
+
+/** Forget old refusals (the scheduler's pass, with the web requests' purge). */
+export async function purgeRefusals() {
+  await owner.query(`delete from meta.network_refusal where at < now() - make_interval(days => ${KEEP_DAYS})`);
+  await owner.query(
+    `delete from meta.network_refusal r using (
+       select id, row_number() over (partition by workspace_id order by id desc) as n from meta.network_refusal
+     ) x where r.id = x.id and x.n > ${KEEP_PER_WORKSPACE}`);
+}
+
+/**
+ * Where a workspace's own SMTP service may connect: the host must pass the
+ * installation's lists and the workspace's, and it is resolved here once; the
+ * connection then goes to the checked address (no DNS rebinding), with the
+ * host name kept for TLS. Throws an Error with the reason.
+ */
+export async function mailHostAddress(scope: OutboundScope, host: string, port: number): Promise<{ address: string; servername: string | undefined }> {
+  let url: URL;
+  try {
+    url = new URL(`https://${net.isIPv6(host) ? `[${host}]` : host}:${port}/`);
+  } catch {
+    throw new Error(`The SMTP host ${host} is not valid.`);
+  }
+  const hosts = restHosts();
+  const problem = urlProblem(url, hosts) ?? scopeProblem(scope, url);
+  if (problem) {
+    scope.refused?.(url, urlProblem(url, hosts) ? 'installation' : 'workspace', problem);
+    throw new Error(problem);
+  }
+  const bare = url.hostname.replace(/^\[|\]$/g, '');
+  const addresses = net.isIP(bare) ? [{ address: bare }] : await dns.lookup(bare, { all: true }).catch(() => []);
+  if (!addresses.length) throw new Error(`The SMTP host ${host} has no address.`);
+  const priv = matches(hosts.private, url);
+  if (!priv && addresses.some((a) => isPrivateAddress(a.address))) {
+    const msg = `The SMTP host ${host} resolves to a private, loopback or link-local address (allow it with PGKILN_REST_PRIVATE_HOSTS).`;
+    scope.refused?.(url, 'installation', msg);
+    throw new Error(msg);
+  }
+  return { address: addresses[0].address, servername: net.isIP(bare) ? undefined : bare };
+}

package/src/objectstore.ts

@@ -10,6 +10,7 @@
 import { createHash, createHmac, randomUUID } from 'node:crypto';
 import { decryptSecret } from './secrets.ts';
 import { WebError, webRequest } from './webclient.ts';
+import { scopeForApp } from './network-acl.ts';
 import { credentialValidFor, loadCredential } from './websources.ts';
 
 export interface ObjectStoreConfig {
@@ -113,6 +114,8 @@ async function send(appId: number, o: ObjectStoreConfig, method: 'PUT' | 'GET' |
   };
   const authorization = signV4({ method, url, headers, payloadHash, accessKey: cred.username!, secretKey: decryptSecret(cred.secret_enc!), region: cred.scope || 'us-east-1', amzDate });
   const res = await webRequest(url.toString(), {
+    // (sprint 118) the workspace's network access list
+    scope: await scopeForApp(appId, `object storage (web credential ${cred.name})`),
     method,
     headers: { ...Object.fromEntries(Object.entries(headers).filter(([k]) => k !== 'host')), authorization, 'accept-encoding': 'identity' },
     secretHeaders: ['authorization'],

package/src/runtime/engine.ts

@@ -560,8 +560,10 @@ async function runOneStep(ctx: PageContext, p: Process, names: Set<string>, dept
       default:
         await runSql(ctx, p.code ?? '', names);
         // web requests the process queued (meta.web_request): made now, before the next process
+        // (sprint 118) a request an allow-list refused is a warning (level 2), the others trace (level 6)
         await runPending(ctx.client!, ctx.app.id, (r, res, ms) =>
-          dbg(ctx, 6, 'web', () => `web request ${r.id} (${r.source ? `source ${r.source}` : r.method}): ${res.status === 'ok' ? `HTTP ${res.statusCode}` : res.message} in ${ms} ms`));
+          dbg(ctx, res.status === 'error' && /allow-list|network access list/.test(res.message ?? '') ? 2 : 6, 'web',
+            () => `web request ${r.id} (${r.source ? `source ${r.source}` : r.method}): ${res.status === 'ok' ? `HTTP ${res.statusCode}` : res.message} in ${ms} ms`));
         // AI requests it queued (meta.ai_generate): likewise
         await runPendingAi(ctx.client!, { appId: ctx.app.id, pageNo: ctx.page.page_no, user: ctx.user, debug: (level, text) => dbg(ctx, level, 'ai', text) });
         return p.success_message;

package/src/runtime/render.ts

@@ -5,7 +5,7 @@ import { pushBody } from './push.ts';
 import { mapHead } from './maps.ts';
 import { staticHead } from './static-files.ts';
 import { pluginAttributes, pluginOf } from './plugins.ts';
-import { appStyles, chosenStyle, chosenStyleName, conditionalStyles, styleChoice, themeCss } from './styles.ts';
+import { appStyles, chosenStyle, chosenStyleName, conditionalStyles, navAppearance, styleChoice, themeCss } from './styles.ts';
 import { html, raw, type Raw } from '../html.ts';
 import { icon } from '../icons.ts';
 import { documentShell } from '../layout.ts';
@@ -260,6 +260,8 @@ export async function chrome(ctx: PageContext, main: Raw, title: string) {
 
   const signedIn = ctx.user !== 'nobody';
   const topNav = ctx.app.theme?.nav === 'top';
+  // (sprint 117, APEX 26.2) Pillar Light / Pillar Dark: classes on <body> from a fixed list (styles.ts)
+  const appearance = navAppearance(ctx.app.theme, chosenStyle(ctx.app, ctx.session, ctx.styleByCondition));
   const nav = await navTree(ctx, topNav);
   const listNodes = ctx.app.navbar_list ? await listTree(ctx, ctx.app.navbar_list) : null;
   // (084) the Feedback entry, after the application's own navigation bar list
@@ -311,7 +313,7 @@ export async function chrome(ctx: PageContext, main: Raw, title: string) {
         <footer class="t-footer">${ctx.app.version ? html`<span class="t-version">${ctx.locale.tr(ctx.app.version)}</span>` : ''}<a class="t-top" href="#top">${icon('arrow-up')} ${t('common.back_to_top')}</a></footer></main>
     </div>
     ${devbarHtml(ctx)}`,
-    `t-app${topNav ? ' nav-top' : ''}`,
+    `t-app${topNav ? ' nav-top' : ''}${appearance ? ` nav-pillar nav-${appearance === 'pillar_dark' ? 'pillar-dark' : 'pillar-light'}` : ''}`,
     { 'data-base': ctx.base, 'data-page': String(ctx.page.page_no), ...pwaBody(ctx.app, ctx.user), ...(await pushBody(ctx.app, ctx.session.username)) },
     html`${pageStyle(ctx)}${pwaHead(ctx.app)}${mapHead(ctx)}${staticHead(ctx)}`,
     root,

package/src/runtime/styles.ts

@@ -69,6 +69,22 @@ export const RADII: Record<string, { label: string; css: string }> = {
   large: { label: 'Large (14 px)', css: '14px' },
 };
 
+/**
+ * (sprint 117, APEX 26.2) Navigation appearances: how the header and the
+ * navigation menu look, on top of the base style. Pillar Light: the header
+ * and menu on the page's own surfaces; Pillar Dark: header and menu as one
+ * dark column. Both carry the header pillar strip (theme.strip / a style's
+ * strip, else the accent) and a filled "pill" for the current page. The
+ * page's <body> gets the classes nav-pillar and nav-pillar-light / -dark.
+ */
+export const NAV_APPEARANCES: Record<string, { label: string }> = {
+  pillar_light: { label: 'Pillar Light' },
+  pillar_dark: { label: 'Pillar Dark' },
+};
+export type NavAppearance = 'pillar_light' | 'pillar_dark';
+/** A style's appearance "default": the base style's own look, whatever the application's appearance. */
+export const APPEARANCE_DEFAULT = 'default';
+
 export interface StyleVariant {
   name: string;
   accent?: string;
@@ -76,6 +92,11 @@ export interface StyleVariant {
   /** (0.29) the colours in dark mode (else dark mode keeps the base style's dark palette) */
   accent_dark?: string;
   header_dark?: string;
+  /** (sprint 117) the navigation appearance (a key of NAV_APPEARANCES, or 'default'; absent: as the application) */
+  appearance?: string;
+  /** (sprint 117) the header pillar strip's colour, light and dark mode (absent: the accent) */
+  strip?: string;
+  strip_dark?: string;
   font?: string;
   font_size?: string;
   radius?: string;
@@ -93,8 +114,11 @@ export function parseStyle(v: Record<string, unknown>): StyleVariant | string {
   const name = typeof v.name === 'string' ? v.name.trim() : '';
   if (!STYLE_NAME.test(name)) return 'Name: 1 to 30 letters, digits, spaces, - or _, starting with a letter or digit.';
   const out: StyleVariant = { name };
-  const colourLabels = { accent: 'Accent colour', header: 'Header colour', accent_dark: 'Accent colour (dark mode)', header_dark: 'Header colour (dark mode)' };
-  for (const k of ['accent', 'header', 'accent_dark', 'header_dark'] as const) {
+  const colourLabels = {
+    accent: 'Accent colour', header: 'Header colour', accent_dark: 'Accent colour (dark mode)', header_dark: 'Header colour (dark mode)',
+    strip: 'Header strip colour', strip_dark: 'Header strip colour (dark mode)',
+  };
+  for (const k of ['accent', 'header', 'accent_dark', 'header_dark', 'strip', 'strip_dark'] as const) {
     const x = v[k];
     if (x === undefined || x === null || x === '') continue;
     if (typeof x === 'string' && ITEM_REF.test(x.trim().toUpperCase())) {
@@ -110,6 +134,11 @@ export function parseStyle(v: Record<string, unknown>): StyleVariant | string {
     if (!own(list, x)) return `${label}: choose from ${Object.keys(list).join(', ')}.`;
     out[k] = x as string;
   }
+  const ap = v.appearance;
+  if (ap !== undefined && ap !== null && ap !== '') {
+    if (ap !== APPEARANCE_DEFAULT && !own(NAV_APPEARANCES, ap)) return `Navigation appearance: choose from ${APPEARANCE_DEFAULT}, ${Object.keys(NAV_APPEARANCES).join(', ')}.`;
+    out.appearance = ap as string;
+  }
   if (typeof v.condition === 'string' && v.condition.trim()) {
     if (v.condition.length > MAX_CONDITION) return `Condition: at most ${MAX_CONDITION} characters.`;
     out.condition = v.condition.trim();
@@ -162,35 +191,48 @@ export const choosable = (app: Pick<App, 'theme'>, name: unknown): name is strin
 /** An item's value for &ITEM. colours (session state), when the CSS is made. */
 export type ItemValue = (name: string) => string | undefined;
 
-const colourVars = (accent?: string, header?: string, item?: ItemValue) => {
+const colourVars = (accent?: unknown, header?: unknown, item?: ItemValue, strip?: unknown) => {
   // an item reference becomes the item's value, used only when it is a #rrggbb colour
-  const value = (c?: string) => {
-    const ref = c ? ITEM_REF.exec(c) : null;
+  const value = (c?: unknown) => {
+    if (typeof c !== 'string') return undefined;
+    const ref = ITEM_REF.exec(c);
     return ref ? item?.(ref[1])?.trim() : c;
   };
-  accent = value(accent);
-  header = value(header);
+  const a = value(accent);
+  const h = value(header);
+  const st = value(strip);
   const vars: string[] = [];
-  if (accent && HEX.test(accent)) vars.push(`--accent:${accent};--accent-soft:color-mix(in srgb, ${accent} 14%, var(--surface))`);
-  if (header && HEX.test(header)) vars.push(`--header:${header};`);
+  if (a && HEX.test(a)) vars.push(`--accent:${a};--accent-soft:color-mix(in srgb, ${a} 14%, var(--surface));`);
+  if (h && HEX.test(h)) vars.push(`--header:${h};`);
+  // (sprint 117) the header pillar strip
+  if (st && HEX.test(st)) vars.push(`--strip:${st};`);
   return vars;
 };
 
+/**
+ * (sprint 117) The navigation appearance for this request: the style's own
+ * (or 'default' for none), else the application's (theme.appearance).
+ */
+export function navAppearance(theme: App['theme'] | undefined, style: StyleVariant | null): NavAppearance | null {
+  const v = style?.appearance ?? theme?.appearance;
+  return typeof v === 'string' && v !== APPEARANCE_DEFAULT && own(NAV_APPEARANCES, v) ? (v as NavAppearance) : null;
+}
+
 /**
  * The theme's CSS: the base colours (Settings → Theme), then the style
  * variant in use. Only checked hex values and constants from the lists above.
  */
 export function themeCss(theme: App['theme'] | undefined, style: StyleVariant | null, item?: ItemValue): string {
   const out: string[] = [];
-  const base = colourVars(theme?.accent, theme?.header);
+  const base = colourVars(theme?.accent, theme?.header, undefined, theme?.strip);
   // html:root ties with app.css's html[data-style] rules and comes later, so own colours win over a base style's
   if (base.length) out.push(`html:root{${base.join('')}}`);
   // dark mode: the same specificity as app.css's dark palettes (html[data-style]…), later in the page
-  const dark = [...colourVars(theme?.accent_dark, theme?.header_dark), ...(style ? colourVars(style.accent_dark, style.header_dark, item) : [])];
+  const dark = [...colourVars(theme?.accent_dark, theme?.header_dark, undefined, theme?.strip_dark), ...(style ? colourVars(style.accent_dark, style.header_dark, item, style.strip_dark) : [])];
   if (dark.length)
     out.push(`@media (prefers-color-scheme: dark){html:root:not([data-theme="light"]){${dark.join('')}}}html:root[data-theme="dark"]{${dark.join('')}}`);
   if (style) {
-    const colours = colourVars(style.accent, style.header, item);
+    const colours = colourVars(style.accent, style.header, item, style.strip);
     if (colours.length) out.push(`html:root{${colours.join('')}}`);
     const all: string[] = [];
     if (style.font && own(FONTS, style.font)) all.push(`--font:${FONTS[style.font].css};`);

package/src/webclient.ts

@@ -21,7 +21,12 @@ import zlib from 'node:zlib';
 //   checks, and request headers marked secret are dropped when a redirect
 //   leaves the origin;
 // - a time limit for the whole exchange and a size limit for the response
-//   (PGKILN_REST_MAX_BYTES, default 5 MB, also after decompression).
+//   (PGKILN_REST_MAX_BYTES, default 5 MB, also after decompression);
+// - (sprint 118) a workspace's network access list (src/network-acl.ts):
+//   when the caller passes the scope of the application's workspace, every
+//   hop must match that list as well; it only narrows the lists above (the
+//   address checks keep using PGKILN_REST_PRIVATE_HOSTS alone). Refusals are
+//   reported to the scope (Administration → Network access).
 //
 // node:http/https rather than fetch(): fetch can't pin the checked address.
 
@@ -45,6 +50,21 @@ export interface WebRequest {
   allowLarge?: boolean;
   /** an allow-list of its own instead of PGKILN_REST_ALLOWED_HOSTS / PGKILN_REST_PRIVATE_HOSTS (push services) */
   hosts?: HostLists;
+  /** (sprint 118) the workspace the call is made for: its network access list narrows the lists above */
+  scope?: OutboundScope | null;
+}
+
+/** (sprint 118) The workspace an outgoing call is made for (src/network-acl.ts builds it). */
+export interface OutboundScope {
+  workspaceId: number;
+  workspaceName: string;
+  appId: number | null;
+  /** what makes the call: "web request", "REST data source", … */
+  what: string;
+  /** the workspace's network access list; null = not restricted (the installation's lists only) */
+  hosts: string[] | null;
+  /** told about every refused URL (never throws; called as a method of the scope) */
+  refused?(url: URL, list: 'installation' | 'workspace', message: string): void;
 }
 
 export interface HostLists {
@@ -68,7 +88,7 @@ export const maxResponseBytes = () => {
 };
 
 /** Whether `url`'s host matches an allow-list entry. */
-function matches(entries: string[], url: URL) {
+export function matches(entries: string[], url: URL) {
   const host = url.hostname.toLowerCase();
   const port = url.port || (url.protocol === 'https:' ? '443' : '80');
   return entries.some((e) => {
@@ -83,7 +103,7 @@ function matches(entries: string[], url: URL) {
   });
 }
 
-const restHosts = (): HostLists => ({ allowed: list(process.env.PGKILN_REST_ALLOWED_HOSTS), private: list(process.env.PGKILN_REST_PRIVATE_HOSTS) });
+export const restHosts = (): HostLists => ({ allowed: list(process.env.PGKILN_REST_ALLOWED_HOSTS), private: list(process.env.PGKILN_REST_PRIVATE_HOSTS) });
 
 /** Why the URL may not be called (null: it may, so far as the host name goes). */
 export function urlProblem(raw: string | URL, hosts: HostLists = restHosts()): string | null {
@@ -103,6 +123,27 @@ export function urlProblem(raw: string | URL, hosts: HostLists = restHosts()): s
   return null;
 }
 
+/** (sprint 118) Why the workspace's network access list refuses the URL (null: it doesn't, or there is none). */
+export function scopeProblem(scope: OutboundScope | null | undefined, url: URL): string | null {
+  if (!scope?.hosts) return null;
+  if (matches(scope.hosts.filter((h) => h !== '*'), url)) return null;
+  return `The host ${url.host} is not on the network access list of workspace ${scope.workspaceName}.`;
+}
+
+/** The installation's and the scope's checks of one URL; a refusal is reported to the scope. */
+export function checkUrl(url: URL, req: Pick<WebRequest, 'hosts' | 'scope'>): string | null {
+  const own = urlProblem(url, req.hosts);
+  const problem = own ?? scopeProblem(req.scope, url);
+  if (problem && req.scope?.refused) {
+    try {
+      req.scope.refused(url, own ? 'installation' : 'workspace', problem);
+    } catch {
+      // reporting never fails the caller
+    }
+  }
+  return problem;
+}
+
 // ---------------------------------------------------------------- addresses
 
 const blocked = new net.BlockList();
@@ -225,7 +266,7 @@ export async function webRequest(raw: string, req: WebRequest = {}): Promise<Web
   let body = req.body;
   const origin = url.origin;
   for (let hop = 0; ; hop++) {
-    const problem = urlProblem(url, req.hosts);
+    const problem = checkUrl(url, req);
     if (problem) throw new WebError(problem);
     const res = await once(url, { ...req, method, headers, body }, deadline, maxBytes);
     const location = res.headers.location;

package/src/webrequests.ts

@@ -2,6 +2,7 @@ import { owner, type Client } from './db.ts';
 import { purgeUnpacked, unpackForSql } from './unpack.ts';
 import { call, invoke, loadCredential } from './websources.ts';
 import { WebError, type WebResponse } from './webclient.ts';
+import { purgeRefusals } from './network-acl.ts';
 
 // Web requests from SQL (APEX_WEB_SERVICE; migration 052): application SQL
 // queues a request with meta.web_request(url, …) or
@@ -77,6 +78,8 @@ export async function execute(appId: number, r: PendingRequest): Promise<Request
       );
       const credential = r.credential ? await loadCredential(appId, r.credential) : null;
       res = await call({
+        appId,
+        what: 'meta.web_request',
         url: r.url,
         method: r.method,
         headers: { accept: '*/*', ...headers },
@@ -138,6 +141,8 @@ export async function purgeWebRequests(force = false) {
       where status = 'queued' and requested_at < now() - interval '${KEEP_HOURS} hours'`,
   );
   await owner.query(`delete from meta.web_request_log where finished_at < now() - interval '${KEEP_HOURS} hours'`);
+  // (sprint 118) refusals of the network access lists: 7 days
+  await purgeRefusals();
   await owner.query(
     `delete from meta.web_request_log l using (
        select id, row_number() over (partition by app_id order by id desc) as n from meta.web_request_log where status in ('ok', 'error')

package/src/websources.ts

@@ -3,7 +3,8 @@ import pg from 'pg';
 import { owner, runtime } from './db.ts';
 import { runtimeForApp } from './workspace-db.ts';
 import { decryptSecret, encryptSecret } from './secrets.ts';
-import { WebError, webRequest, type WebResponse } from './webclient.ts';
+import { checkUrl, WebError, webRequest, type OutboundScope, type WebResponse } from './webclient.ts';
+import { scopeForApp } from './network-acl.ts';
 
 // Web credentials and REST data sources (meta.web_credential,
 // meta.rest_source): building the request (parameters, credential), OAuth2
@@ -466,9 +467,11 @@ export function tokenRequest(c: WebCredential, refresh: string | null) {
   return { headers, body: form.toString() };
 }
 
-async function requestToken(c: WebCredential, refresh: string | null, timeoutMs: number): Promise<Token> {
+async function requestToken(c: WebCredential, refresh: string | null, timeoutMs: number, scope: OutboundScope | null): Promise<Token> {
   const { headers, body } = tokenRequest(c, refresh);
   const res = await webRequest(c.token_url!, {
+    // (sprint 118) the token endpoint is on the workspace's network access list too
+    scope: scope ? { ...scope, what: `token endpoint of web credential ${c.name}` } : null,
     method: 'POST',
     headers,
     secretHeaders: ['authorization'],
@@ -515,7 +518,7 @@ function storedRefresh(c: WebCredential) {
   }
 }
 
-async function oauthToken(c: WebCredential, timeoutMs: number, fresh = false): Promise<string> {
+async function oauthToken(c: WebCredential, timeoutMs: number, scope: OutboundScope | null, fresh = false): Promise<string> {
   const key = `${c.id}:${credKey(c)}`;
   const cached = tokens.get(key);
   if (cached && !fresh && cached.expires > Date.now()) return cached.access;
@@ -525,14 +528,14 @@ async function oauthToken(c: WebCredential, timeoutMs: number, fresh = false): P
       const refresh = cached?.refresh ?? storedRefresh(c);
       if (refresh) {
         try {
-          return await requestToken(c, refresh, timeoutMs);
+          return await requestToken(c, refresh, timeoutMs, scope);
         } catch (e) {
           // the refresh token may have expired: ask again with the grant (a refresh token grant has nothing else)
           if (grantOf(c) === 'refresh_token')
             throw new WebError(`${(e as Error).message} The refresh token of web credential ${c.name} may have expired: enter a new one in the builder.`, (e as WebError).status);
         }
       }
-      return requestToken(c, null, timeoutMs);
+      return requestToken(c, null, timeoutMs, scope);
     })();
     pendingTokens.set(key, p);
   }
@@ -546,7 +549,7 @@ async function oauthToken(c: WebCredential, timeoutMs: number, fresh = false): P
 }
 
 /** Headers that sign a request with the credential; the names in secret are secrets. */
-async function credentialHeaders(c: WebCredential, url: string, timeoutMs: number, fresh = false) {
+async function credentialHeaders(c: WebCredential, url: string, timeoutMs: number, scope: OutboundScope | null, fresh = false) {
   if (!credentialValidFor(c, url)) throw new WebError(`Web credential ${c.name} is not valid for this URL (see its "Valid for" URLs).`);
   switch (c.type) {
     case 'basic':
@@ -559,7 +562,7 @@ async function credentialHeaders(c: WebCredential, url: string, timeoutMs: numbe
       return { [c.header_name!.toLowerCase()]: value };
     }
     case 'oauth2':
-      return { authorization: `Bearer ${await oauthToken(c, timeoutMs, fresh)}` };
+      return { authorization: `Bearer ${await oauthToken(c, timeoutMs, scope, fresh)}` };
     case 'aws_sigv4':
       throw new WebError(`Web credential ${c.name} (aws_sigv4) signs object storage requests of file items only.`);
     default:
@@ -570,6 +573,12 @@ async function credentialHeaders(c: WebCredential, url: string, timeoutMs: numbe
 // ---------------------------------------------------------------- calling
 
 export interface CallOptions {
+  /** (sprint 118) the application the call is made for: its workspace's network access list applies (null: none) */
+  appId: number | null;
+  /** the scope when the caller has it already (else built from appId) */
+  scope?: OutboundScope | null;
+  /** what makes the call, for refusals (default "web request") */
+  what?: string;
   url: string;
   method?: string;
   headers?: Record<string, string>;
@@ -583,9 +592,14 @@ export interface CallOptions {
 /** One request, signed with the credential; an OAuth2 token refused with 401 is renewed once. */
 export async function call(o: CallOptions): Promise<WebResponse> {
   const timeoutMs = o.timeoutMs ?? 10_000;
+  const scope = o.scope !== undefined ? o.scope : await scopeForApp(o.appId, o.what ?? 'web request');
+  // refused before a credential's token is asked for
+  const problem = checkUrl(new URL(o.url), { scope });
+  if (problem) throw new WebError(problem);
   const send = async (fresh: boolean) => {
-    const auth: Record<string, string> = o.credential ? await credentialHeaders(o.credential, o.url, timeoutMs, fresh) : {};
+    const auth: Record<string, string> = o.credential ? await credentialHeaders(o.credential, o.url, timeoutMs, scope, fresh) : {};
     return webRequest(o.url, {
+      scope,
       method: o.method ?? 'GET',
       headers: { accept: 'application/json', ...o.headers, ...auth },
       secretHeaders: [...Object.keys(auth), ...(o.secretHeaders ?? [])],
@@ -748,7 +762,7 @@ export function buildOperation(s: RestSource, name: OperationName, row: Record<s
 export async function callOperation(s: RestSource, name: OperationName, row: Record<string, unknown>, params: Record<string, string> = {}) {
   const req = buildOperation(s, name, row, params);
   const credential = s.credential ? await loadCredential(s.app_id, s.credential) : null;
-  const res = await call({ ...req, credential, timeoutMs: s.timeout_s * 1000 });
+  const res = await call({ ...req, appId: s.app_id, what: `REST data source ${s.name}`, credential, timeoutMs: s.timeout_s * 1000 });
   if (name === 'fetch' && res.status === 404) return { json: null, row: null, status: 404 };
   if (res.status < 200 || res.status > 299) throw new WebError(`REST data source ${s.name}: the ${name} operation was answered with ${res.status}.`, res.status);
   const json = parseJson(res, `REST data source ${s.name}`);
@@ -778,6 +792,10 @@ export function clearResponseCache(sourceId?: number) {
 /** The parsed JSON response of a source (from the cache when it is fresh enough). */
 export async function fetchSource(s: RestSource, values: Record<string, string | null | undefined>) {
   const req = buildRequest(s, values);
+  // (sprint 118) the workspace's network access list, before the cache: a narrowed list applies at once
+  const scope = await scopeForApp(s.app_id, `REST data source ${s.name}`);
+  const problem = checkUrl(new URL(req.url), { scope });
+  if (problem) throw new WebError(problem);
   const credential = s.credential ? await loadCredential(s.app_id, s.credential) : null;
   const key = s.cache_seconds > 0
     ? `${s.id}:${createHash('sha256').update(JSON.stringify([s, req, credential ? credKey(credential) : null])).digest('base64url')}`
@@ -786,7 +804,7 @@ export async function fetchSource(s: RestSource, values: Record<string, string |
     const hit = responses.get(key);
     if (hit && Date.now() - hit.at < s.cache_seconds * 1000) return { json: hit.json, status: 200, cached: true };
   }
-  const res = await call({ ...req, credential, timeoutMs: s.timeout_s * 1000 });
+  const res = await call({ ...req, appId: s.app_id, scope, credential, timeoutMs: s.timeout_s * 1000 });
   if (res.status < 200 || res.status > 299) throw new WebError(`REST data source ${s.name}: the web service answered ${res.status}.`, res.status);
   const json = parseJson(res, `REST data source ${s.name}`);
   if (key) {
@@ -878,7 +896,7 @@ export async function invoke(appId: number, conf: InvokeConfig, lookup: Lookup,
     const source = await loadSource(appId, conf.source);
     const req = buildRequest(source, sourceParamValues(source, conf.params, lookup));
     const credential = source.credential ? await loadCredential(appId, source.credential) : null;
-    return { res: await call({ ...req, credential, timeoutMs: (timeoutS ?? source.timeout_s) * 1000 }), source };
+    return { res: await call({ ...req, appId, what: `${what} (REST data source ${source.name})`, credential, timeoutMs: (timeoutS ?? source.timeout_s) * 1000 }), source };
   }
   // values are URL-encoded and only follow the host (invokeCallProblems), so the host is always the developer's
   const url = conf.url!.replace(SUBST, (m, name: string) => {
@@ -893,7 +911,7 @@ export async function invoke(appId: number, conf: InvokeConfig, lookup: Lookup,
     return JSON.stringify(v ?? m);
   }) : undefined;
   const credential = conf.credential ? await loadCredential(appId, conf.credential) : null;
-  const res = await call({ url, method, headers: body ? { 'content-type': 'application/json' } : {}, body, credential, timeoutMs: (timeoutS ?? 10) * 1000 });
+  const res = await call({ appId, what, url, method, headers: body ? { 'content-type': 'application/json' } : {}, body, credential, timeoutMs: (timeoutS ?? 10) * 1000 });
   return { res, source: null };
 }