pgkiln 0.72.0 → 0.74.0

package/src/websources.ts

@@ -5,6 +5,7 @@ import { runtimeForApp } from './workspace-db.ts';
 import { decryptSecret, encryptSecret } from './secrets.ts';
 import { checkUrl, WebError, webRequest, type OutboundScope, type WebResponse } from './webclient.ts';
 import { scopeForApp } from './network-acl.ts';
+import { resolveSource } from './remote-servers.ts';
 
 // Web credentials and REST data sources (meta.web_credential,
 // meta.rest_source): building the request (parameters, credential), OAuth2
@@ -34,6 +35,8 @@ export interface WebCredential {
   oauth_username?: string | null;
   password_enc?: string | null;
   refresh_token_enc?: string | null;
+  /** (141) a credential of the application's workspace (meta.workspace_credential) */
+  workspace?: boolean;
 }
 
 export type ParamLocation = 'query' | 'path' | 'header' | 'body';
@@ -56,9 +59,12 @@ export interface RestSource {
   id: number;
   app_id: number;
   name: string;
+  /** a full URL; (141) with a server, the path after the server's base URL (resolveSource makes it full) */
   url: string;
   method: string;
   credential: string | null;
+  /** (141) a remote server of the application's workspace (static id) */
+  server?: string | null;
   headers: Record<string, string>;
   params: RestParam[];
   body: string | null;
@@ -101,9 +107,14 @@ const RESERVED_HEADERS = new Set(['authorization', 'proxy-authorization', 'cooki
 // ---------------------------------------------------------------- checks (builder, import)
 
 /** Problems with a REST data source definition (empty: fine). */
-export function sourceProblems(s: { url?: unknown; params?: unknown; columns?: unknown; headers?: unknown; row_selector?: unknown; key_columns?: unknown; operations?: unknown }): string[] {
+export function sourceProblems(s: { url?: unknown; server?: unknown; params?: unknown; columns?: unknown; headers?: unknown; row_selector?: unknown; key_columns?: unknown; operations?: unknown }): string[] {
   const out: string[] = [];
-  if (typeof s.url !== 'string' || !/^https?:\/\/[^/\s]+/i.test(s.url)) out.push('The URL must start with http:// or https://.');
+  if (s.server != null && s.server !== '') {
+    // (141) on a remote server: a path after its base URL (or nothing)
+    if (typeof s.server !== 'string' || !/^[A-Z][A-Z0-9_]{0,59}$/.test(s.server)) out.push('The remote server is its static id, e.g. CRM.');
+    if (typeof s.url !== 'string' || !/^([/?][^\s]*)?$/.test(s.url)) out.push('With a remote server the URL is the path after its base URL, e.g. /v1/orders/{id} (or empty).');
+    else if (s.url.includes('//') || /(^|\/)\.\.?(\/|$|\?)/.test(s.url)) out.push('The path may not contain // or .. segments.');
+  } else if (typeof s.url !== 'string' || !/^https?:\/\/[^/\s]+/i.test(s.url)) out.push('The URL must start with http:// or https://.');
   else if (/[{&]/.test(/^https?:\/\/[^/?#]*/i.exec(s.url)![0])) out.push('The URL\'s host is fixed: parameters can only follow it.');
   else {
     try {
@@ -384,21 +395,29 @@ export const withRest = (rowsQuery: string, sql: string | null | undefined) =>
 export async function loadSource(appId: number, name: string) {
   // the application's own connection (an isolated workspace's), also from a workflow or job (090)
   const s = await (await runtimeForApp(appId)).one<RestSource>(
-    `select id, app_id, name, url, method, credential, headers, params, body, row_selector, columns, cache_seconds, timeout_s, max_rows, key_columns, operations
+    `select id, app_id, name, url, method, credential, server, headers, params, body, row_selector, columns, cache_seconds, timeout_s, max_rows, key_columns, operations
        from meta.rest_source where app_id = $1 and name = $2`,
     [appId, name.toUpperCase()],
   );
   if (!s) throw new WebError(`REST data source ${name} does not exist.`);
-  return s;
+  // (141) on a remote server: the server's URL of this moment, and its credential when the source names none
+  return resolveSource(s);
 }
 
-/** With the (encrypted) secret: the owner connection, never the runtime role. */
+const CREDENTIAL_COLUMNS = 'id, name, type, username, header_name, token_url, scope, valid_for, secret_enc, grant_type, oauth_username, password_enc, refresh_token_enc';
+
+/**
+ * With the (encrypted) secret: the owner connection, never the runtime role. The application's own
+ * credential of that name, else (141) one of its workspace's (never another workspace's).
+ */
 export async function loadCredential(appId: number, name: string) {
-  const c = await owner.one<WebCredential>(
-    `select id, app_id, name, type, username, header_name, token_url, scope, valid_for, secret_enc, grant_type, oauth_username, password_enc, refresh_token_enc
-       from meta.web_credential where app_id = $1 and name = $2`,
+  const c = (await owner.one<WebCredential>(
+    `select ${CREDENTIAL_COLUMNS}, app_id from meta.web_credential where app_id = $1 and name = $2`,
     [appId, name.toUpperCase()],
-  );
+  )) ?? (await owner.one<WebCredential>(
+    `select ${CREDENTIAL_COLUMNS}, $1::int as app_id, true as workspace from meta.workspace_credential where workspace_id = meta.app_workspace($1) and name = $2`,
+    [appId, name.toUpperCase()],
+  ));
   if (!c) throw new WebError(`Web credential ${name} does not exist.`);
   return c;
 }
@@ -428,15 +447,18 @@ interface Token {
 const tokens = new Map<string, Token>();
 const pendingTokens = new Map<string, Promise<Token>>();
 const credKey = (c: WebCredential) =>
-  createHash('sha256').update(JSON.stringify([c.id, c.username, c.token_url, c.scope, c.secret_enc, c.grant_type ?? null, c.oauth_username ?? null, c.password_enc ?? null])).digest('base64url');
+  createHash('sha256').update(JSON.stringify([tokenId(c), c.username, c.token_url, c.scope, c.secret_enc, c.grant_type ?? null, c.oauth_username ?? null, c.password_enc ?? null])).digest('base64url');
+
+/** The token cache's id of a credential: (141) a workspace credential's starts with w. */
+const tokenId = (c: Pick<WebCredential, 'id' | 'workspace'>) => (c.workspace ? `w${c.id}` : String(c.id));
 
-/** Forget cached tokens (all, or one credential's). */
-export function clearTokens(credentialId?: number) {
+/** Forget cached tokens (all, or one credential's: its id, or w<id> for a workspace credential). */
+export function clearTokens(credentialId?: number | string) {
   for (const k of [...tokens.keys()]) if (credentialId === undefined || k.startsWith(`${credentialId}:`)) tokens.delete(k);
 }
 
 const secretOf = (c: WebCredential) => {
-  if (!c.secret_enc) throw new WebError(`Web credential ${c.name} has no secret yet: enter it in the builder (Shared Components → Web credentials).`);
+  if (!c.secret_enc) throw new WebError(`Web credential ${c.name} has no secret yet: enter it in the builder (${c.workspace ? 'Workspace utilities' : 'Shared Components'} → Web credentials).`);
   return decryptSecret(c.secret_enc);
 };
 
@@ -504,7 +526,7 @@ async function requestToken(c: WebCredential, refresh: string | null, timeoutMs:
 /** Keep a refresh token (encrypted); written with the owner connection only. */
 async function storeRefreshToken(c: WebCredential, token: string) {
   const enc = encryptSecret(token);
-  await owner.query('update meta.web_credential set refresh_token_enc = $2, token_refreshed_at = now() where id = $1', [c.id, enc]);
+  await owner.query(`update meta.${c.workspace ? 'workspace_credential' : 'web_credential'} set refresh_token_enc = $2, token_refreshed_at = now() where id = $1`, [c.id, enc]);
   c.refresh_token_enc = enc;
 }
 
@@ -519,7 +541,7 @@ function storedRefresh(c: WebCredential) {
 }
 
 async function oauthToken(c: WebCredential, timeoutMs: number, scope: OutboundScope | null, fresh = false): Promise<string> {
-  const key = `${c.id}:${credKey(c)}`;
+  const key = `${tokenId(c)}:${credKey(c)}`;
   const cached = tokens.get(key);
   if (cached && !fresh && cached.expires > Date.now()) return cached.access;
   let p = pendingTokens.get(key);
@@ -609,7 +631,7 @@ export async function call(o: CallOptions): Promise<WebResponse> {
   };
   const res = await send(false);
   if (res.status === 401 && o.credential?.type === 'oauth2') {
-    clearTokens(o.credential.id);
+    clearTokens(tokenId(o.credential));
     return send(true);
   }
   return res;
@@ -759,7 +781,8 @@ export function buildOperation(s: RestSource, name: OperationName, row: Record<s
  * 200–299 fails. Returns the response's JSON (null when empty) and, for
  * insert and fetch, the row in it (by the operation's row selector).
  */
-export async function callOperation(s: RestSource, name: OperationName, row: Record<string, unknown>, params: Record<string, string> = {}) {
+export async function callOperation(source: RestSource, name: OperationName, row: Record<string, unknown>, params: Record<string, string> = {}) {
+  const s = await resolveSource(source);
   const req = buildOperation(s, name, row, params);
   const credential = s.credential ? await loadCredential(s.app_id, s.credential) : null;
   const res = await call({ ...req, appId: s.app_id, what: `REST data source ${s.name}`, credential, timeoutMs: s.timeout_s * 1000 });
@@ -790,7 +813,9 @@ export function clearResponseCache(sourceId?: number) {
 }
 
 /** The parsed JSON response of a source (from the cache when it is fresh enough). */
-export async function fetchSource(s: RestSource, values: Record<string, string | null | undefined>) {
+export async function fetchSource(source: RestSource, values: Record<string, string | null | undefined>) {
+  // (141) on a remote server: its URL now (a source from loadSource is resolved already)
+  const s = await resolveSource(source);
   const req = buildRequest(s, values);
   // (sprint 118) the workspace's network access list, before the cache: a narrowed list applies at once
   const scope = await scopeForApp(s.app_id, `REST data source ${s.name}`);

package/src/workspace-export.ts

@@ -2,6 +2,9 @@ import type { Client } from './db.ts';
 import { owner } from './db.ts';
 import { checkDataset, datasetOf, storeDataset } from './datasets.ts';
 import { describeTeam, exportTeam, importTeam } from './team.ts';
+import { serverProblems } from './remote-servers.ts';
+import { STATIC_MAX, staticType } from './runtime/static-files.ts';
+import { credentialProblems, GRANT_TYPES } from './websources.ts';
 import { catalogDocument, fromCatalogDocument, insertCatalog, loadCatalog, replaceCatalog, wholeCatalogProblems } from './restcatalog.ts';
 
 // Workspace utilities → Export / import (APEX: Workspace Utilities → Export,
@@ -21,6 +24,10 @@ import { catalogDocument, fromCatalogDocument, insertCatalog, loadCatalog, repla
 //   datasets         (sprint 110) Data Reporter datasets (pgkiln-dataset/1 definitions)
 //   team             (sprint 113) Team Development: label groups, labels, milestones, templates and issues
 //                    (with comments and history; applications by alias), an object (src/team.ts)
+//   workspace_files  (sprint 141) static workspace files (content in base64)
+//   remote_servers   (sprint 141) remote servers; an existing server keeps its own base URL on import
+//                    (the URL is per installation: development, test, production)
+//   web_credentials  (sprint 141) the workspace's web credentials, never their secrets (entered again after an import)
 //
 // Not exported: the workspace's name (the importer's workspace keeps its own),
 // isolation and its database roles (installation-specific), script runs and
@@ -29,7 +36,8 @@ import { catalogDocument, fromCatalogDocument, insertCatalog, loadCatalog, repla
 // installation has a developer of that name.
 
 export const WORKSPACE_FORMAT = 'pgkiln-workspace/1';
-export const PARTS = ['settings', 'developers', 'app_groups', 'extension_links', 'rest_catalogs', 'sql_scripts', 'blueprints', 'data_generators', 'datasets', 'team'] as const;
+export const PARTS = ['settings', 'developers', 'app_groups', 'extension_links', 'rest_catalogs', 'sql_scripts', 'blueprints', 'data_generators', 'datasets', 'team',
+  'workspace_files', 'remote_servers', 'web_credentials'] as const;
 export type Part = (typeof PARTS)[number];
 export const PART_LABELS: Record<Part, string> = {
   settings: 'Settings: description, message, banner and preferences',
@@ -42,6 +50,9 @@ export const PART_LABELS: Record<Part, string> = {
   data_generators: 'Data generators',
   datasets: 'Data Reporter datasets',
   team: 'Team Development: labels, milestones, templates and issues',
+  workspace_files: 'Static workspace files',
+  remote_servers: 'Remote servers (an existing server keeps its base URL)',
+  web_credentials: 'Web credentials (without their secrets)',
 };
 
 const MAX_ITEMS = 5000;
@@ -82,6 +93,14 @@ export async function exportWorkspace(workspaceId: number, parts: readonly Part[
   if (want('datasets'))
     doc.datasets = (await owner.query('select * from meta.dataset where workspace_id = $1 order by name', [workspaceId])).rows.map(datasetOf);
   if (want('team')) doc.team = await exportTeam(workspaceId);
+  if (want('workspace_files'))
+    doc.workspace_files = (await owner.query(`select name, mime, translate(encode(content, 'base64'), E'\n', '') as content from meta.workspace_file where workspace_id = $1 order by name`, [workspaceId])).rows;
+  if (want('remote_servers'))
+    doc.remote_servers = (await owner.query('select static_id, name, description, base_url, env_var, credential from meta.remote_server where workspace_id = $1 order by static_id', [workspaceId])).rows;
+  if (want('web_credentials'))
+    // secrets never leave the installation
+    doc.web_credentials = (await owner.query(
+      'select name, description, type, username, header_name, token_url, scope, valid_for, grant_type, oauth_username from meta.workspace_credential where workspace_id = $1 order by name', [workspaceId])).rows;
   return doc;
 }
 
@@ -271,5 +290,65 @@ export async function importWorkspace(c: Client, workspaceId: number, doc: unkno
     const t = await importTeam(c, workspaceId, d.team, opts);
     tally('Team Development', t.added, t.replaced, t.skipped, t.issues ? `; ${t.issues} issue(s) with their comments` : '');
   }
+
+  if (want('workspace_files')) {
+    // (sprint 141) the same names, types and sizes as an upload
+    const n = count();
+    for (const x of d.workspace_files as Json[]) {
+      const name = isObj(x) && typeof x.name === 'string' ? x.name : '';
+      const type = staticType(name);
+      if (!type) throw new Error(`Static workspace file ${JSON.stringify(name).slice(0, 110)}: not an allowed name or type.`);
+      const content = typeof x.content === 'string' ? Buffer.from(x.content, 'base64') : Buffer.alloc(0);
+      if (content.length > STATIC_MAX) throw new Error(`Static workspace file ${name}: larger than ${STATIC_MAX / 1024 / 1024} MB.`);
+      const existing = (await c.query<{ id: number }>('select id from meta.workspace_file where workspace_id = $1 and name = $2', [workspaceId, name])).rows[0];
+      n[await upsert(existing,
+        () => c.query('insert into meta.workspace_file (workspace_id, name, mime, content, updated_by) values ($1, $2, $3, $4, $5)', [workspaceId, name, type, content, opts.by]),
+        (id) => c.query('update meta.workspace_file set mime = $2, content = $3, updated_by = $4, updated_at = now() where id = $1', [id, type, content, opts.by]))]++;
+    }
+    tally('Static workspace files', n.added, n.replaced, n.skipped);
+  }
+
+  if (want('remote_servers')) {
+    // (sprint 141) the base URL is this installation's: an existing server keeps its own, a new one takes the file's
+    const n = count();
+    for (const x of d.remote_servers as Json[]) {
+      if (!isObj(x)) continue;
+      const v = { static_id: x.static_id, name: x.name, description: text(x.description, 2000), base_url: text(x.base_url, 2000), env_var: text(x.env_var, 80), credential: text(x.credential, 60) };
+      const problems = serverProblems(v);
+      if (problems.length) throw new Error(`Remote server ${typeof x.static_id === 'string' ? x.static_id : '?'}: ${problems.join(' ')}`);
+      const existing = (await c.query<{ id: number }>('select id from meta.remote_server where workspace_id = $1 and static_id = $2', [workspaceId, v.static_id])).rows[0];
+      n[await upsert(existing,
+        () => c.query('insert into meta.remote_server (workspace_id, static_id, name, description, base_url, env_var, credential, updated_by) values ($1, $2, $3, $4, $5, $6, $7, $8)',
+          [workspaceId, v.static_id, v.name, v.description, v.base_url, v.env_var, v.credential, opts.by]),
+        (id) => c.query(`update meta.remote_server set name = $2, description = $3, env_var = $4, credential = $5, base_url = coalesce(base_url, $6),
+                                updated_by = $7, updated_at = now() where id = $1`, [id, v.name, v.description, v.env_var, v.credential, v.base_url, opts.by]))]++;
+    }
+    tally('Remote servers', n.added, n.replaced, n.skipped, n.replaced ? '; existing servers kept their base URL' : '');
+  }
+
+  if (want('web_credentials')) {
+    // (sprint 141) without secrets: an existing credential keeps its own, a new one gets them entered in the builder
+    const n = count();
+    for (const x of d.web_credentials as Json[]) {
+      if (!isObj(x)) continue;
+      const v = {
+        name: String(x.name ?? ''), description: text(x.description, 2000),
+        type: ['basic', 'header', 'bearer', 'oauth2', 'aws_sigv4'].includes(x.type) ? x.type : 'basic',
+        username: text(x.username, 500), header_name: text(x.header_name, 64), token_url: text(x.token_url, 2000), scope: text(x.scope, 500),
+        valid_for: Array.isArray(x.valid_for) ? x.valid_for.filter((u: unknown) => typeof u === 'string') : [],
+        grant_type: GRANT_TYPES.includes(x.grant_type) ? x.grant_type : 'client_credentials', oauth_username: text(x.oauth_username, 500),
+      };
+      const problems = [...(/^[A-Z][A-Z0-9_]{0,59}$/.test(v.name) ? [] : ['the name is upper case letters, digits and _.']), ...credentialProblems(v)];
+      if (problems.length) throw new Error(`Web credential ${v.name || '?'}: ${problems.join(' ')}`);
+      const vals = [v.description, v.type, v.username, v.header_name, v.token_url, v.scope, v.valid_for, v.grant_type, v.oauth_username];
+      const existing = (await c.query<{ id: number }>('select id from meta.workspace_credential where workspace_id = $1 and name = $2', [workspaceId, v.name])).rows[0];
+      n[await upsert(existing,
+        () => c.query(`insert into meta.workspace_credential (workspace_id, name, description, type, username, header_name, token_url, scope, valid_for, grant_type, oauth_username, updated_by)
+                       values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, [workspaceId, v.name, ...vals, opts.by]),
+        (id) => c.query(`update meta.workspace_credential set description = $2, type = $3, username = $4, header_name = $5, token_url = $6, scope = $7, valid_for = $8,
+                                grant_type = $9, oauth_username = $10, updated_by = $11, updated_at = now() where id = $1`, [id, ...vals, opts.by]))]++;
+    }
+    tally('Web credentials', n.added, n.replaced, n.skipped, n.added ? '; enter their secrets under Workspace utilities → Web credentials' : '');
+  }
   return r;
 }