pgkiln 0.48.0 → 0.49.0

package/src/runtime/rest-handlers.ts

@@ -0,0 +1,75 @@
+import type pg from 'pg';
+import { substitute, type BindValues } from '../binds.ts';
+
+// (sprint 104, Database Actions DA-11 part 2; migration 117) The handler source types REST modules gained with
+// SQL Workshop › RESTful Services, next to collection, item and sql (rest.ts):
+//
+//   plpgsql  a PL/pgSQL block run as an anonymous DO block as the application's role. A DO block takes no
+//            parameters, so the binds go in as transaction-local settings, set with query parameters (never part
+//            of the SQL text), and :NAME in the block becomes meta.rest_param('NAME'). The block answers with
+//            meta.rest_reply(body jsonb, status int); without a reply: 204.
+//   media    a query whose first row has content_type and content (bytea or text), optional filename: the
+//            response is that content. It is served sandboxed (CSP sandbox, nosniff), so an HTML or SVG
+//            "image" from the table can't run script in the application's origin.
+
+/** Bind names a PL/pgSQL handler can read (a setting name each): letters, digits and _. */
+const BIND_NAME = /^[A-Z_][A-Z0-9_]{0,62}$/;
+/** The dollar-quote tag around a PL/pgSQL handler's block. */
+export const DO_TAG = '$pgkiln_rest$';
+/** At most this many binds go into settings (a body with thousands of fields is not a handler's input). */
+const MAX_BINDS = 200;
+
+export class HandlerInputError extends Error {}
+
+/** The DO statement of a PL/pgSQL handler: its binds as meta.rest_param('NAME'), wrapped in begin … end if needed. */
+export function plpgsqlBlock(source: string): string {
+  if (source.includes(DO_TAG)) throw new Error(`A PL/pgSQL handler may not contain ${DO_TAG}.`);
+  let body = substitute(source.trim().replace(/;+\s*$/, ''), (name) => `meta.rest_param('${name}')`);
+  if (!/^(declare|begin|<<)/i.test(body.replace(/^(\s*(--[^\n]*(\n|$)|\/\*[\s\S]*?\*\/))*\s*/, ''))) body = `begin\n${body};\nend`;
+  return `do ${DO_TAG}\n${body}\n${DO_TAG}`;
+}
+
+/** Set the binds as transaction-local settings pgkiln_bind.<name>, as query parameters. */
+export async function setBinds(c: pg.PoolClient, binds: BindValues) {
+  const entries = Object.entries(binds).filter(([k]) => BIND_NAME.test(k)).slice(0, MAX_BINDS);
+  for (const [, v] of entries) if (typeof v === 'string' && v.includes('\0')) throw new HandlerInputError('A parameter contains a NUL byte.');
+  if (!entries.length) return;
+  await c.query(`select set_config('pgkiln_bind.' || lower(k), coalesce(v, ''), true) from unnest($1::text[], $2::text[]) as b(k, v)`,
+    [entries.map(([k]) => k), entries.map(([, v]) => v ?? null)]);
+}
+
+/** Run a PL/pgSQL handler; its reply (meta.rest_reply) or none. */
+export async function runPlpgsql(c: pg.PoolClient, source: string): Promise<{ body: unknown; status: number | null }> {
+  // a reply from an earlier statement of this transaction must not leak into this one
+  await c.query(`select set_config('pgkiln_rest.body', '', true), set_config('pgkiln_rest.status', '', true)`);
+  await c.query(plpgsqlBlock(source));
+  const r = (await c.query(`select current_setting('pgkiln_rest.body', true) as body, current_setting('pgkiln_rest.status', true) as status`)).rows[0];
+  return { body: r.body ? JSON.parse(r.body) : undefined, status: r.status ? Number(r.status) : null };
+}
+
+/** A content type safe to send as a header: type/subtype with optional parameters, no line breaks. */
+export function safeContentType(v: unknown): string {
+  const s = typeof v === 'string' ? v.trim() : '';
+  return /^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+(\s*;\s*[a-z0-9_-]+=("[^"\r\n]*"|[^;\s\r\n]+))*$/i.test(s) && s.length <= 200 ? s : 'application/octet-stream';
+}
+
+/** A file name for Content-Disposition: printable ASCII without quotes, slashes or backslashes. */
+export const safeFilename = (v: unknown) => (typeof v === 'string' ? v.replace(/[^\x20-\x7e]|["\\/;]/g, '_').slice(0, 200) : '');
+
+/** The response of a media handler's row. */
+export function mediaResponse(row: Record<string, unknown>) {
+  if (!('content' in row) || !('content_type' in row)) throw new Error('A media handler\'s query returns the columns content_type and content (and optionally filename).');
+  const c = row.content;
+  const body = c === null || c === undefined ? Buffer.alloc(0) : Buffer.isBuffer(c) ? c : Buffer.from(typeof c === 'string' ? c : JSON.stringify(c));
+  const name = safeFilename(row.filename);
+  return {
+    body,
+    headers: {
+      'content-type': safeContentType(row.content_type),
+      'x-content-type-options': 'nosniff',
+      'content-security-policy': "default-src 'none'; style-src 'unsafe-inline'; img-src data:; sandbox",
+      'cache-control': 'no-store',
+      ...(name ? { 'content-disposition': `inline; filename="${name}"` } : {}),
+    } as Record<string, string>,
+  };
+}

package/src/runtime/rest.ts

@@ -10,8 +10,10 @@ import { loadApp, type App } from '../metadata.ts';
 import { clientIp } from '../session.ts';
 import { PassThrough } from 'node:stream';
 import { publicError, writeOut } from './context.ts';
+import { DO_TAG, HandlerInputError, mediaResponse, runPlpgsql, setBinds } from './rest-handlers.ts';
 
-// REST modules (APEX: RESTful Services): handlers defined in the builder,
+// REST modules (APEX: RESTful Services): handlers defined in the builder (Shared Components, or (sprint 104)
+// SQL Workshop › RESTful Services › Modules as templates and handlers; plpgsql and media types: rest-handlers.ts),
 // served under /a/<alias>/rest/<module>/<path>.
 //
 //   {"method": "GET", "path": "employees", "type": "collection", "source": "select … order by empno"}
@@ -33,17 +35,36 @@ import { publicError, writeOut } from './context.ts';
 export interface Handler {
   method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
   path: string;
-  type: 'collection' | 'item' | 'sql';
+  /** (117) plpgsql: a PL/pgSQL block (rest-handlers.ts); media: a row's content_type and content */
+  type: 'collection' | 'item' | 'sql' | 'plpgsql' | 'media';
   source: string;
   roles?: string[];
   auth?: 'token' | 'public';
   description?: string;
   page_size?: number;
   status?: number;
+  /** (117) documented parameters: in the OpenAPI document and the Try it form; a required one missing is a 400 */
+  params?: HandlerParam[];
 }
 
-const METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'];
-const TYPES = ['collection', 'item', 'sql'];
+export interface HandlerParam {
+  name: string;
+  in: 'query' | 'body';
+  type?: 'string' | 'integer' | 'number' | 'boolean';
+  required?: boolean;
+  description?: string;
+}
+
+/** (117) A template of a module: its URI and description (handlers name it in "path"). */
+export interface Template {
+  uri: string;
+  description?: string;
+}
+
+export const METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'];
+export const TYPES = ['collection', 'item', 'sql', 'plpgsql', 'media'];
+export const PARAM_TYPES = ['string', 'integer', 'number', 'boolean'];
+const PARAM_NAME = /^[a-z_][a-z0-9_]{0,62}$/i;
 const SEGMENT = /^(:[a-z][a-z0-9_]*|[a-z0-9][a-z0-9._-]*)$/i;
 const MAX_PAGE = 500;
 /** Rows fetched from a collection's cursor at a time. */
@@ -65,7 +86,18 @@ export function handlerProblems(handlers: unknown): string[] {
       problems.push(`${at}: ${[...RESERVED_BINDS].join(', ')} are set by the server and can't be path parameters.`);
     if (!TYPES.includes(h.type)) problems.push(`${at}: "type" is one of ${TYPES.join(', ')}.`);
     if (typeof h.source !== 'string' || !h.source.trim()) problems.push(`${at}: "source" is the SQL.`);
-    if (h.type !== 'sql' && h.method !== 'GET') problems.push(`${at}: collection and item handlers answer GET; use "sql" for ${h.method}.`);
+    if ((h.type === 'collection' || h.type === 'item' || h.type === 'media') && h.method !== 'GET') problems.push(`${at}: collection, item and media handlers answer GET; use "sql" or "plpgsql" for ${h.method}.`);
+    if (h.type === 'plpgsql' && typeof h.source === 'string' && h.source.includes(DO_TAG)) problems.push(`${at}: a PL/pgSQL block may not contain ${DO_TAG}.`);
+    if (h.params !== undefined) {
+      if (!Array.isArray(h.params)) problems.push(`${at}: "params" is a list of {"name", "in": "query" or "body", "type", "required"}.`);
+      else h.params.forEach((p: any) => {
+        if (!p || typeof p !== 'object' || typeof p.name !== 'string' || !PARAM_NAME.test(p.name)) problems.push(`${at}: a parameter's name is letters, digits and _.`);
+        else if (RESERVED_BINDS.has(p.name.toUpperCase())) problems.push(`${at}: ${p.name} is set by the server and can't be a parameter.`);
+        else if (p.name.toUpperCase() === 'BODY') problems.push(`${at}: :BODY is the whole JSON body; name the parameter otherwise.`);
+        if (p && p.in !== 'query' && p.in !== 'body') problems.push(`${at}: a parameter is "in" query or body.`);
+        if (p?.type !== undefined && !PARAM_TYPES.includes(p.type)) problems.push(`${at}: a parameter's type is one of ${PARAM_TYPES.join(', ')}.`);
+      });
+    }
     if (h.roles !== undefined && !(Array.isArray(h.roles) && h.roles.every((r: unknown) => typeof r === 'string'))) problems.push(`${at}: "roles" is a list of role names.`);
     if (h.auth !== undefined && h.auth !== 'token' && h.auth !== 'public') problems.push(`${at}: "auth" is token (default) or public.`);
     if (h.status !== undefined && !(Number.isInteger(h.status) && h.status >= 200 && h.status < 300)) problems.push(`${at}: "status" is a 2xx status.`);
@@ -76,6 +108,28 @@ export function handlerProblems(handlers: unknown): string[] {
   return problems;
 }
 
+/** (117) Problems in a module's templates, or []. */
+export function templateProblems(templates: unknown): string[] {
+  if (!Array.isArray(templates)) return ['The templates must be a JSON array.'];
+  const problems: string[] = [];
+  const seen = new Set<string>();
+  templates.forEach((t: any, i) => {
+    const uri = t?.uri;
+    if (typeof uri !== 'string' || !uri || !uri.split('/').every((s: string) => SEGMENT.test(s)))
+      problems.push(`Template ${i + 1}: the URI is like emp or emp/:id (letters, digits, - _ . and :parameters).`);
+    else if (uri === 'openapi.json') problems.push(`Template ${i + 1}: openapi.json is reserved for the module's description.`);
+    else if (uri.split('/').some((s: string) => s.startsWith(':') && RESERVED_BINDS.has(s.slice(1).toUpperCase())))
+      problems.push(`Template ${i + 1}: ${[...RESERVED_BINDS].join(', ')} can't be URI parameters.`);
+    else {
+      const key = uri.replace(/:[^/]+/g, ':');
+      if (seen.has(key)) problems.push(`Template ${i + 1}: ${uri} is there twice.`);
+      seen.add(key);
+    }
+    if (t?.description !== undefined && typeof t.description !== 'string') problems.push(`Template ${i + 1}: the description is text.`);
+  });
+  return problems;
+}
+
 /** The handler for a method and path, with its path parameters. */
 export function matchHandler(handlers: Handler[], method: string, path: string) {
   const parts = path.split('/').filter(Boolean);
@@ -145,7 +199,7 @@ async function handle(req: FastifyRequest, reply: FastifyReply) {
   const { alias, module: name } = req.params as { alias: string; module: string };
   const sub = ((req.params as Record<string, string>)['*'] ?? '').replace(/\/+$/, '');
   const a = await loadApp(alias);
-  const mod = a ? await runtime.one<{ handlers: Handler[]; enabled: boolean }>('select handlers, enabled from meta.rest_module where app_id = $1 and name = $2', [a.id, name]) : undefined;
+  const mod = a ? await runtime.one<{ handlers: Handler[]; enabled: boolean; roles: string[] | null }>('select handlers, enabled, roles from meta.rest_module where app_id = $1 and name = $2', [a.id, name]) : undefined;
   if (!a || !mod?.enabled || !(await restServicesOn(a.id))) return reply.code(404).send({ error: 'Not found' });
   const m = matchHandler(mod.handlers ?? [], req.method, sub);
   if (m === 'method') return reply.code(405).send({ error: `${req.method} is not allowed here.` });
@@ -165,6 +219,9 @@ async function handle(req: FastifyRequest, reply: FastifyReply) {
       binds.BODY = JSON.stringify(body);
     }
     Object.assign(binds, params);
+    // (117) documented parameters marked required
+    const missing = (h.params ?? []).filter((p) => p.required && (binds[p.name.toUpperCase()] ?? '') === '').map((p) => p.name);
+    if (missing.length) throw new RestError(400, `Missing parameter${missing.length > 1 ? 's' : ''}: ${missing.join(', ')}.`);
     // the built-in names are the server's, never the caller's (?app_user=… must not set :APP_USER)
     for (const k of Object.keys(binds)) if (RESERVED_BINDS.has(k)) delete binds[k];
     Object.assign(binds, { APP_USER: who.user, APP_ID: String(a.id), APP_ALIAS: a.alias, APP_SESSION: null, APP_PAGE_ID: null, REQUEST: null });
@@ -174,11 +231,24 @@ async function handle(req: FastifyRequest, reply: FastifyReply) {
                 set_config('request.jwt.claims', $3, true), set_config('statement_timeout', '30s', true)`,
         [String(a.id), who.user, who.claims ? JSON.stringify(who.claims) : ''],
       );
-      if (h.roles?.length) {
-        const ok = (await c.query('select bool_or(meta.has_role(r)) as ok from unnest($1::text[]) r', [h.roles])).rows[0].ok;
+      // (117) the module's privilege, then the handler's own roles
+      for (const roles of [mod.roles, h.roles]) if (roles?.length) {
+        const ok = (await c.query('select bool_or(meta.has_role(r)) as ok from unnest($1::text[]) r', [roles])).rows[0].ok;
         if (!ok) throw new RestError(403, 'You do not have a role this endpoint needs.');
       }
+      if (h.type === 'plpgsql') await setBinds(c, binds);
       if (a.db_role) await c.query(`set local role ${pg.escapeIdentifier(a.db_role)}`);
+      // (117) handlers run as a constrained role: never a superuser, a role that bypasses row level security, or
+      // pgkiln's owner (a runtime connection without RUNTIME_DATABASE_URL is the owner's)
+      const role = (await c.query(
+        `select r.rolsuper or r.rolbypassrls or pg_has_role(current_user, n.nspowner, 'MEMBER') as bad
+           from pg_roles r, pg_namespace n where r.rolname = current_user and n.nspname = 'meta'`)).rows[0];
+      if (!role || role.bad) throw new RestError(403, 'This endpoint would run as a role that bypasses row level security or owns pgkiln: refused. Give the application its own database role.');
+      if (h.type === 'plpgsql') {
+        const out = await runPlpgsql(c, h.source);
+        const status = out.status ?? (out.body === undefined ? 204 : h.status ?? (req.method === 'POST' ? 201 : 200));
+        return { status, body: out.body };
+      }
       const sql = applyBinds(h.source.trim().replace(/;+\s*$/, ''), binds);
       if (h.type === 'collection') {
         const size = Math.max(1, Math.min(MAX_PAGE, Number(binds.LIMIT) || h.page_size || 25));
@@ -211,9 +281,10 @@ async function handle(req: FastifyRequest, reply: FastifyReply) {
         }
         return { status: 200, body: undefined, streamed: true };
       }
-      if (h.type === 'item') {
+      if (h.type === 'item' || h.type === 'media') {
         const rows = (await c.query(`select * from (\n${sql}\n) "__r" limit 2`)).rows;
         if (!rows.length) throw new RestError(404, 'Not found');
+        if (h.type === 'media') return { status: 200, body: undefined, media: mediaResponse(rows[0]) };
         return { status: 200, body: rows[0] };
       }
       let last: pg.QueryResult | undefined;
@@ -222,9 +293,11 @@ async function handle(req: FastifyRequest, reply: FastifyReply) {
       return row ? { status: h.status ?? (req.method === 'POST' ? 201 : 200), body: row } : { status: 204, body: undefined };
     });
     if ('streamed' in result) return reply;
-    return result.body === undefined ? reply.code(204).send() : reply.code(result.status).send(result.body);
+    if ('media' in result && result.media) return reply.code(200).headers(result.media.headers).send(result.media.body);
+    return result.body === undefined ? reply.code(result.status === 204 || result.status < 200 ? 204 : result.status).send() : reply.code(result.status).send(result.body);
   } catch (e) {
     if (e instanceof RestError) return reply.code(e.status).send({ error: e.message });
+    if (e instanceof HandlerInputError) return reply.code(400).send({ error: e.message });
     const err = e as pg.DatabaseError;
     const message = await publicError({ app: a, page: { page_no: 0 } as never, user: 'api', ip: clientIp(req) }, e, `REST ${req.method} ${name}/${sub}`);
     const status = err.code === '42501' ? 403 : err.code && /^(P0001|22|23)/.test(err.code) ? 400 : 500;
@@ -233,26 +306,47 @@ async function handle(req: FastifyRequest, reply: FastifyReply) {
 }
 
 /** The OpenAPI 3 description of a module. */
-export function openApi(a: Pick<App, 'alias' | 'name'>, mod: { name: string; title: string; description: string | null; handlers: Handler[] }) {
+export function openApi(
+  a: Pick<App, 'alias' | 'name'>,
+  mod: { name: string; title: string; description: string | null; handlers: Handler[]; roles?: string[] | null; templates?: Template[] | null },
+) {
   const paths: Record<string, Record<string, unknown>> = {};
+  const toPath = (uri: string) => `/${uri.replace(/:([a-z][a-z0-9_]*)/gi, '{$1}')}`;
+  const modRoles = mod.roles ?? [];
+  for (const t of Array.isArray(mod.templates) ? mod.templates : []) if (t?.uri && t.description) (paths[toPath(t.uri)] ??= {}).description = t.description;
   for (const h of mod.handlers) {
-    const p = `/${h.path.replace(/:([a-z][a-z0-9_]*)/gi, '{$1}')}`;
-    const params = [...h.path.matchAll(/:([a-z][a-z0-9_]*)/gi)].map((x) => ({ name: x[1], in: 'path', required: true, schema: { type: 'string' } }));
+    const p = toPath(h.path);
+    const params: Record<string, unknown>[] = [...h.path.matchAll(/:([a-z][a-z0-9_]*)/gi)].map((x) => ({ name: x[1], in: 'path', required: true, schema: { type: 'string' } }));
     if (h.type === 'collection')
-      params.push({ name: 'limit', in: 'query', required: false, schema: { type: 'integer' } } as never, { name: 'offset', in: 'query', required: false, schema: { type: 'integer' } } as never);
+      params.push({ name: 'limit', in: 'query', required: false, schema: { type: 'integer' } }, { name: 'offset', in: 'query', required: false, schema: { type: 'integer' } });
+    const documented = Array.isArray(h.params) ? h.params : [];
+    for (const q of documented.filter((x) => x.in === 'query'))
+      params.push({ name: q.name, in: 'query', required: !!q.required, schema: { type: q.type ?? 'string' }, ...(q.description ? { description: q.description } : {}) });
+    const bodyParams = documented.filter((x) => x.in === 'body');
+    const bodySchema = bodyParams.length
+      ? { type: 'object', properties: Object.fromEntries(bodyParams.map((q) => [q.name, { type: q.type ?? 'string', ...(q.description ? { description: q.description } : {}) }])),
+          ...(bodyParams.some((q) => q.required) ? { required: bodyParams.filter((q) => q.required).map((q) => q.name) } : {}) }
+      : { type: 'object' };
+    const roles = [modRoles.length ? `one of the roles ${modRoles.join(', ')}` : '', h.roles?.length ? `one of the roles ${h.roles.join(', ')}` : ''].filter(Boolean);
+    const ok = h.type === 'sql' || h.type === 'plpgsql' ? String(h.status ?? (h.method === 'POST' ? 201 : 200)) : '200';
     paths[p] ??= {};
     paths[p][h.method.toLowerCase()] = {
       summary: h.description ?? `${h.method} ${h.path || '/'}`,
+      ...(roles.length ? { description: `The caller needs ${roles.join(' and ')}.`, 'x-pgkiln-roles': [modRoles, h.roles ?? []].filter((r) => r.length) } : {}),
       ...(params.length ? { parameters: params } : {}),
-      ...(h.method !== 'GET' ? { requestBody: { required: false, content: { 'application/json': { schema: { type: 'object' } } } } } : {}),
+      ...(h.method !== 'GET' ? { requestBody: { required: bodyParams.some((q) => q.required), content: { 'application/json': { schema: bodySchema } } } } : {}),
       ...(h.auth === 'public' ? { security: [] } : {}),
       responses: {
-        [h.type === 'sql' ? String(h.status ?? (h.method === 'POST' ? 201 : 200)) : '200']: {
-          description: h.type === 'collection' ? '{items, offset, limit, has_more}' : 'The row as an object',
-          content: { 'application/json': { schema: { type: 'object' } } },
-        },
-        ...(h.type === 'item' ? { 404: { description: 'Not found' } } : {}),
-        ...(h.auth === 'public' ? {} : { 401: { description: 'No valid token' }, 403: { description: 'Not allowed' } }),
+        [ok]: h.type === 'media'
+          ? { description: 'The content (content type from the row)', content: { 'application/octet-stream': { schema: { type: 'string', format: 'binary' } } } }
+          : {
+            description: h.type === 'collection' ? '{items, offset, limit, has_more}' : h.type === 'plpgsql' ? 'The body of meta.rest_reply' : 'The row as an object',
+            content: { 'application/json': { schema: { type: 'object' } } },
+          },
+        ...(h.type === 'sql' || h.type === 'plpgsql' ? { 204: { description: 'No content' } } : {}),
+        ...(documented.some((q) => q.required) || h.type === 'plpgsql' || h.type === 'sql' ? { 400: { description: 'Missing or invalid input' } } : {}),
+        ...(h.type === 'item' || h.type === 'media' ? { 404: { description: 'Not found' } } : {}),
+        ...(h.auth === 'public' && !roles.length ? {} : { 401: { description: 'No valid token' }, 403: { description: 'Not allowed' } }),
       },
     };
   }
@@ -269,7 +363,7 @@ export function openApi(a: Pick<App, 'alias' | 'name'>, mod: { name: string; tit
 export async function restRoutes(app: FastifyInstance) {
   app.get<{ Params: { alias: string; module: string } }>('/a/:alias/rest/:module/openapi.json', async (req, reply) => {
     const a = await loadApp(req.params.alias);
-    const mod = a ? await runtime.one<{ name: string; title: string; description: string | null; handlers: Handler[]; enabled: boolean }>('select * from meta.rest_module where app_id = $1 and name = $2', [a.id, req.params.module]) : undefined;
+    const mod = a ? await runtime.one<{ name: string; title: string; description: string | null; handlers: Handler[]; enabled: boolean; roles: string[]; templates: Template[] }>('select * from meta.rest_module where app_id = $1 and name = $2', [a.id, req.params.module]) : undefined;
     if (!a || !mod?.enabled || !(await restServicesOn(a.id))) return reply.code(404).send({ error: english('error.not_found') });
     return reply.send(openApi(a, mod));
   });