pgkiln 0.73.0 → 0.74.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/ai/service.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { owner } from '../db.ts';
2
2
  import { decryptSecret } from '../secrets.ts';
3
+ import { aiServerUrl } from '../remote-servers.ts';
3
4
  import { ClaudeProvider } from './anthropic.ts';
4
5
  import { OpenAiProvider } from './openai.ts';
5
6
  import { AiError, type AiProvider, type GenerateRequest, type GenerateResult, type ProviderName } from './types.ts';
@@ -27,6 +28,9 @@ export interface AiService {
27
28
  base_url: string | null;
28
29
  api_key_enc: string | null;
29
30
  enabled: boolean;
31
+ /** (141) a remote server whose base URL (plus server_path) the service calls instead of base_url */
32
+ server_id?: number | null;
33
+ server_path?: string | null;
30
34
  }
31
35
 
32
36
  /** The default model of a new Claude service; OpenAI services name theirs. */
@@ -78,7 +82,12 @@ export function serviceKey(s: AiService): string {
78
82
  }
79
83
 
80
84
  export async function loadService(name: string): Promise<AiService | undefined> {
81
- return owner.one<AiService>('select * from meta.ai_service where name = $1', [name.toUpperCase()]);
85
+ const s = await owner.one<AiService>('select * from meta.ai_service where name = $1', [name.toUpperCase()]);
86
+ if (!s?.server_id) return s;
87
+ // (141) on a remote server: its URL now (base_url is then what baseUrlFor gives the SDK)
88
+ const r = await aiServerUrl(s.server_id, s.server_path);
89
+ if (!r.url) throw new AiError('config', `AI service ${s.name}: ${r.problem}`);
90
+ return { ...s, base_url: r.url };
82
91
  }
83
92
 
84
93
  /** The provider of a service, with its key (decrypted only here, never stored elsewhere). */
package/src/app.ts CHANGED
@@ -96,6 +96,7 @@ import { findWorkspaceRoutes } from './builder/find-workspace.ts';
96
96
  import { mailQueueRoutes } from './builder/mail-queue.ts';
97
97
  import { networkAccessRoutes } from './builder/network-access.ts';
98
98
  import { workspaceMailRoutes } from './builder/workspace-mail.ts';
99
+ import { workspaceResourceRoutes } from './builder/workspace-resources.ts';
99
100
  import { runtimeScope } from './db.ts';
100
101
  // (sprint 139, AP-6) OpenTelemetry: a trace per request, /metrics, the browser relay
101
102
  import { startTelemetry } from './telemetry/export.ts';
@@ -281,6 +282,7 @@ export async function buildApp(opts: { logger?: boolean } = {}) {
281
282
  await app.register(mailQueueRoutes);
282
283
  await app.register(networkAccessRoutes);
283
284
  await app.register(workspaceMailRoutes);
285
+ await app.register(workspaceResourceRoutes);
284
286
  // (sprint 138, AP-5) an application's Utilities
285
287
  await app.register(utilityRoutes);
286
288
  await app.register(dependencyRoutes);
@@ -387,6 +387,10 @@ export async function advise(appId: number, tx?: pg.ClientBase, opts: { db?: Db
387
387
  // (104) a document button names a document template or a report query
388
388
  const documents = new Set(all.filter((x) => x.kind === 'document_template' || x.kind === 'report_query').map((x) => x.row.name.toUpperCase()));
389
389
  const missing = (entry: Entry | null, field: string, message: string, severity: Severity = 'error') => findings.push({ severity, entry, field, message });
390
+ // (141) remote servers and web credentials of the application's workspace (sources and invoke steps may name them)
391
+ const wsServers = new Map((await query('select static_id, credential from meta.remote_server where workspace_id = meta.app_workspace($1)', [appId])).rows.map((r) => [r.static_id as string, r.credential as string | null]));
392
+ const wsCredentials = new Set((await query('select name from meta.workspace_credential where workspace_id = meta.app_workspace($1)', [appId])).rows.map((r) => r.name as string));
393
+ const appCredentials = new Set(all.filter((x) => x.kind === 'web_credential').map((x) => String(x.row.name).toUpperCase()));
390
394
 
391
395
  for (const e of entries) {
392
396
  for (const f of e.fields) {
@@ -457,6 +461,14 @@ export async function advise(appId: number, tx?: pg.ClientBase, opts: { db?: Db
457
461
  if (kind === 'button' && row.action === 'document' && !documents.has(String(row.document ?? '').toUpperCase()))
458
462
  missing(e, 'Document template', row.document ? `Document template or report query ${row.document} doesn't exist.` : 'A document button needs a document template or a report query.');
459
463
  if (kind === 'rest_module') for (const problem of handlerProblems(row.handlers)) missing(e, 'Handlers (JSON)', problem);
464
+ if (kind === 'rest_source') {
465
+ // (141) an export may name a server or credential this workspace doesn't have (yet): it imports, and says so here
466
+ if (row.server && !wsServers.has(row.server))
467
+ missing(e, 'Remote server', `Remote server ${row.server} doesn't exist in this application's workspace: add it under Workspace utilities → Remote servers (calls fail until then).`, 'warning');
468
+ const credential = row.credential || (row.server ? wsServers.get(row.server) : null);
469
+ if (credential && !appCredentials.has(credential) && !wsCredentials.has(credential))
470
+ missing(e, 'Web credential', `Web credential ${credential} is neither the application's nor its workspace's (calls fail until it exists).`, 'warning');
471
+ }
460
472
  if (kind === 'workflow_definition') {
461
473
  const tasks = new Set(all.filter((x) => x.kind === 'task_definition').map((x) => x.row.name.toUpperCase()));
462
474
  const outcomes = outcomeMap(all.filter((x) => x.kind === 'task_definition').map((x) => x.row));
@@ -465,7 +477,7 @@ export async function advise(appId: number, tx?: pg.ClientBase, opts: { db?: Db
465
477
  // invoke_api steps: REST data sources, parameters, web credentials, variables (sprint 32)
466
478
  const refs = {
467
479
  sources: new Map(all.filter((x) => x.kind === 'rest_source').map((x) => [String(x.row.name).toUpperCase(), x.row])),
468
- credentials: new Set(all.filter((x) => x.kind === 'web_credential').map((x) => String(x.row.name).toUpperCase())),
480
+ credentials: new Set([...appCredentials, ...wsCredentials]),
469
481
  startVars: [...String(row.title ?? '').matchAll(/&([A-Za-z][A-Za-z0-9_]*)\./g)].map((m) => m[1]),
470
482
  };
471
483
  const invokeRefs = invokeStepReferences(row.steps, refs);
package/src/builder/ai.ts CHANGED
@@ -55,7 +55,7 @@ function serviceForm(s: Partial<AiService>, action: string, csrfField: ReturnTyp
55
55
  }
56
56
 
57
57
  /** The form's values, checked; throws with a message. */
58
- function formValues(b: Record<string, string | undefined>) {
58
+ function formValues(b: Record<string, string | undefined>, hasServer = false) {
59
59
  const provider: ProviderName = (['openai', 'gemini', 'openai_compatible'] as const).find((p) => p === b.provider) ?? 'anthropic';
60
60
  const model = (b.model ?? '').trim();
61
61
  if (!/^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,199}$/.test(model)) throw new Error('The model is a name like claude-opus-5-5 (letters, digits and . _ : / @ -).');
@@ -65,7 +65,7 @@ function formValues(b: Record<string, string | undefined>) {
65
65
  if (!Number.isInteger(timeout) || timeout < 5 || timeout > 600) throw new Error('Time limit: 5 to 600 seconds.');
66
66
  const baseUrl = (b.base_url ?? '').trim() || null;
67
67
  if (baseUrl && (!/^https?:\/\/[^\s]+$/i.test(baseUrl) || baseUrl.length > 500)) throw new Error('The base URL starts with http:// or https://.');
68
- if (!baseUrl && provider === 'openai_compatible') throw new Error('An OpenAI-compatible service needs a base URL (e.g. http://localhost:11434/v1 for Ollama).');
68
+ if (!baseUrl && provider === 'openai_compatible' && !hasServer) throw new Error('An OpenAI-compatible service needs a base URL (e.g. http://localhost:11434/v1 for Ollama).');
69
69
  const effort = ['low', 'medium', 'high', 'xhigh', 'max'].includes(b.effort ?? '') ? b.effort! : null;
70
70
  return [(b.description ?? '').trim() || null, provider, model, provider === 'anthropic' ? effort : null, b.refusal_fallback === 'true', maxTokens, timeout, baseUrl, b.enabled === 'true'];
71
71
  }
@@ -157,12 +157,20 @@ export async function aiRoutes(app: FastifyInstance) {
157
157
  delete s.state[TEST_RESULT];
158
158
  const recent = (await owner.query(`select u.*, a.alias as app from meta.ai_usage u left join meta.app a on a.id = u.app_id where u.service_id = $1 order by u.id desc limit 50`, [x.id])).rows;
159
159
  const view = { ...x, api_key_enc: x.api_key_enc ? 'stored' : null }; // the key itself never reaches the page
160
+ // (141) the remote servers of every workspace (administrators choose one)
161
+ const servers = (await owner.query<{ id: number; static_id: string; workspace: string }>(
162
+ 'select r.id, r.static_id, w.name as workspace from meta.remote_server r join meta.workspace w on w.id = r.workspace_id order by lower(w.name), r.static_id')).rows;
160
163
  const main = html`
161
164
  <div class="title-row"><h1 class="ai-name">${x.name}</h1></div>
162
165
  <div class="columns wide-left">
163
166
  ${region('Settings', html`${serviceForm(view, `${BASE}/ai/${x.id}`, csrf(s), false)}
164
167
  <form method="post" action="${BASE}/ai/${x.id}/delete" class="danger-zone">${csrf(s)}
165
168
  <button class="btn btn-danger" data-confirm="Delete AI service ${x.name}? Processes that use it fail until another service gets its name.">Delete service</button></form>`)}
169
+ ${region('Remote server', html`<form method="post" action="${BASE}/ai/${x.id}/server">${csrf(s)}
170
+ <p class="muted u-mt0">Instead of the base URL: a remote server's URL (Workspace utilities → Remote servers) plus a path, so moving the endpoint changes one value. The API key is sent there: choose a server whose workspace administrators you trust with it.</p>
171
+ <div class="form-grid">${select('server_id', 'Remote server', x.server_id ? String(x.server_id) : '', [['', 'none: the base URL above'], ...servers.map((r): [string, string] => [String(r.id), `${r.static_id} (${r.workspace})`])])}
172
+ ${input('server_path', 'Path', x.server_path ?? '', { placeholder: '/v1', help: 'After the server\'s base URL, e.g. /v1; empty: the base URL itself.' })}</div>
173
+ <div class="buttons"><button class="btn">Save</button></div></form>`)}
166
174
  ${region('Test', html`
167
175
  <p class="muted u-mt0">Sends a prompt to the service (counted in the usage log as a builder test).</p>
168
176
  <form method="post" action="${BASE}/ai/${x.id}/test">${csrf(s)}
@@ -194,7 +202,7 @@ export async function aiRoutes(app: FastifyInstance) {
194
202
  `update meta.ai_service set (${COLUMNS}) = ($2, $3, $4, $5, $6, $7, $8, $9, $10), updated_at = now(),
195
203
  api_key_enc = case when $12 then null when $11::text is null then api_key_enc else $11 end
196
204
  where id = $1`,
197
- [req.params.id, ...formValues(b), key, b.remove_key === 'true'],
205
+ [req.params.id, ...formValues(b, !!(await owner.one('select 1 as ok from meta.ai_service where id = $1 and server_id is not null', [req.params.id]))), key, b.remove_key === 'true'],
198
206
  );
199
207
  flash(s, r.rowCount ? 'AI service saved.' : 'Not found.', r.rowCount ? 'ok' : 'error');
200
208
  } catch (e) {
@@ -203,6 +211,24 @@ export async function aiRoutes(app: FastifyInstance) {
203
211
  return back(reply, s, `${BASE}/ai/${req.params.id}`);
204
212
  });
205
213
 
214
+ // (141) the service's remote server and path
215
+ app.post(`${BASE}/ai/:id(^\\d+$)/server`, async (req: Req, reply) => {
216
+ const s = await developer(req, reply);
217
+ if (!s || !(await adminOnly(s, reply, 'AI services'))) return;
218
+ const b = req.body ?? {};
219
+ try {
220
+ const serverId = /^\d{1,9}$/.test(String(b.server_id ?? '')) ? Number(b.server_id) : null;
221
+ const path = String(b.server_path ?? '').trim() || null;
222
+ if (path && (!/^\/[^\s]*$/.test(path) || path.length > 500)) throw new Error('The path starts with /, without spaces, e.g. /v1.');
223
+ if (serverId && !(await owner.one('select 1 as ok from meta.remote_server where id = $1', [serverId]))) throw new Error('No such remote server.');
224
+ const r = await owner.query('update meta.ai_service set server_id = $2, server_path = $3, updated_at = now() where id = $1', [req.params.id, serverId, serverId ? path : null]);
225
+ flash(s, r.rowCount ? (serverId ? 'The service now calls its remote server: test it.' : 'The service calls its base URL again.') : 'Not found.', r.rowCount ? 'ok' : 'error');
226
+ } catch (e) {
227
+ flash(s, (e as Error).message.includes('ai_service_compatible_url') ? 'An OpenAI-compatible service needs a base URL or a remote server.' : (e as Error).message, 'error');
228
+ }
229
+ return back(reply, s, `${BASE}/ai/${req.params.id}`);
230
+ });
231
+
206
232
  app.post(`${BASE}/ai/:id(^\\d+$)/apps`, async (req: Req, reply) => {
207
233
  const s = await developer(req, reply);
208
234
  if (!s || !(await adminOnly(s, reply, 'AI services'))) return;
@@ -391,7 +391,7 @@ export async function designerRoutes(app: FastifyInstance) {
391
391
  ${select('page_template', 'Page template', p.page_template ?? 'standard', [['standard', 'Standard'], ['left_side_column', 'Left side column'], ['right_side_column', 'Right side column'], ['minimal', 'Minimal (no navigation)']], 'Side column: regions with position left (or right) show in a column beside the body (under it on narrow screens). Minimal: no header and navigation menu, e.g. for a login page (Application settings → Login page).')}
392
392
  ${select('dialog_position', 'Dialog position', p.dialog_position ?? 'center', [['center', 'Centred dialog'], ['right', 'Drawer from the right'], ['left', 'Drawer from the left'], ['top', 'Drawer from the top'], ['bottom', 'Drawer from the bottom']], 'Modal pages only. On phones dialogs and side drawers fill the screen.')}
393
393
  ${select('dialog_size', 'Dialog size', p.dialog_size ?? 'medium', [['small', 'Small'], ['medium', 'Medium'], ['large', 'Large']], 'The width (the height of top and bottom drawers).')}
394
- ${input('static_includes', 'JavaScript and CSS files', (p.static_includes ?? []).join(', '), { placeholder: 'e.g. page-3.js, page-3.css', help: 'Static application files (.js, .css) this page loads after the application\'s own, in this order.' })}
394
+ ${input('static_includes', 'JavaScript and CSS files', (p.static_includes ?? []).join(', '), { placeholder: 'e.g. page-3.js, page-3.css', help: 'Static application files (.js, .css) this page loads after the application\'s own, in this order; #WORKSPACE_FILES#name for a static workspace file.' })}
395
395
  <div class="field"><span class="label" aria-hidden="true"></span><label class="check"><input type="checkbox" name="warn_unsaved" value="true"${p.warn_unsaved ? raw(' checked') : ''}> Warn on unsaved changes</label>
396
396
  <small class="help">Ask before the user leaves the page (a link, a redirect button, closing the tab) while an item has a changed value. Submitting doesn't ask. Leave an item out with {"warn_unsaved": false} in its attributes.</small></div>
397
397
  ${select('parent_page', 'Breadcrumb parent', p.parent_page ?? '', [['', '- none -'], ...lk.pages.filter((x) => x.page_no !== p.page_no).map((x): [string, string] => [String(x.page_no), `${x.page_no}. ${x.name}`])])}
@@ -286,6 +286,9 @@ export async function homeRoutes(app: FastifyInstance) {
286
286
  ${card(`${BASE}/developers`, icon('users'), 'Developers', 'Who may use this builder.')}
287
287
  ${s.workspace?.sql_workshop === false ? '' : card(`${BASE}/sql`, icon('database'), 'SQL Workshop', 'Run SQL, browse objects and load data.')}
288
288
  ${card(`${BASE}/backups`, icon('archive'), 'Manage backups', 'The backups of every application of this workspace; back them all up at once.')}
289
+ ${card(`${BASE}/workspace-files`, icon('file'), 'Static workspace files', 'CSS, JavaScript, images and fonts every application of this workspace can load (#WORKSPACE_FILES#).')}
290
+ ${card(`${BASE}/remote-servers`, icon('server'), 'Remote servers', 'Named base URLs for REST data sources, catalogs and AI services: change one value to move them all.')}
291
+ ${card(`${BASE}/workspace-credentials`, icon('key'), 'Web credentials', 'Keys and OAuth2 clients the applications of this workspace use by name (secrets encrypted, write-only).')}
289
292
  ${card(`${BASE}/rest-catalogs`, icon('globe'), 'REST Source Catalogs', 'Web services described once for the workspace (e.g. from an OpenAPI document); applications make REST data sources from them.')}
290
293
  ${s.workspace?.admin ? card(`${BASE}/workspace-export`, icon('download'), 'Export and import', 'This workspace\'s components as one file: settings, developers, groups, links, catalogs, scripts, blueprints (workspace administrators).') : ''}
291
294
  ${card(`${BASE}/dashboard`, icon('activity'), 'Dashboard', 'Usage and problems across the workspace.')}
@@ -3,6 +3,7 @@ import { owner } from '../db.ts';
3
3
  import { html, raw } from '../html.ts';
4
4
  import { icon } from '../icons.ts';
5
5
  import { parseHostList, scopeForWorkspace } from '../network-acl.ts';
6
+ import { serverUrl, type RemoteServer } from '../remote-servers.ts';
6
7
  import { clientIp, logActivity, type Session } from '../session.ts';
7
8
  import { matches, restHosts, scopeProblem, urlProblem } from '../webclient.ts';
8
9
  import { clearResponseCache } from '../websources.ts';
@@ -53,6 +54,29 @@ export async function networkAccessRoutes(app: FastifyInstance) {
53
54
  `select r.at, r.what, r.host, r.url, r.list, r.message, a.alias from meta.network_refusal r left join meta.app a on a.id = r.app_id
54
55
  where r.workspace_id = $1 order by r.id desc limit 100`, [w.id]).then((r) => r.rows),
55
56
  ]);
57
+ // (141) the hosts the workspace's remote servers stand for now, and whether a call to them would pass
58
+ const scopeNow = await scopeForWorkspace(w.id, 'check');
59
+ const servers = (await owner.query<RemoteServer>('select * from meta.remote_server where workspace_id = $1 order by static_id', [w.id])).rows.map((x) => {
60
+ const u = serverUrl(x);
61
+ let host = '';
62
+ let problem: string | null = u.problem;
63
+ if (u.url)
64
+ try {
65
+ const url = new URL(u.url);
66
+ host = url.host;
67
+ problem = urlProblem(url) ?? scopeProblem(scopeNow, url);
68
+ } catch {
69
+ problem = 'not a valid URL';
70
+ }
71
+ return { ...x, url: u.url, from: u.from, host, problem };
72
+ });
73
+ const serversOf = (entry: string) => servers.filter((x) => x.host && (() => {
74
+ try {
75
+ return matches([entry], new URL(`https://${x.host}`));
76
+ } catch {
77
+ return false;
78
+ }
79
+ })()).map((x) => x.static_id);
56
80
  // entries the installation's list never lets through (they narrow nothing: worth a warning)
57
81
  const unreachable = w.network_hosts.filter((h) => {
58
82
  const host = h.replace(/^\*\./, 'x.');
@@ -90,11 +114,20 @@ export async function networkAccessRoutes(app: FastifyInstance) {
90
114
  <textarea id="f_hosts" name="hosts" rows="6" spellcheck="false">${w.network_hosts.join('\n')}</textarea>
91
115
  <small class="help">One per line: <code>api.example.com</code>, <code>*.example.com</code> for its subdomains, <code>api.example.com:8443</code> for one port only. No scheme or path. Restricted with no hosts: no outgoing calls at all.</small></div>
92
116
  ${unreachable.length ? html`<div class="alert alert-warning" role="status">Not on the installation's list, so never reachable: ${unreachable.join(', ')}.</div>` : ''}
117
+ ${w.network_hosts.some((h) => serversOf(h).length) ? html`<p class="muted small">Remote servers on these hosts: ${w.network_hosts.filter((h) => serversOf(h).length).map((h) => `${h} (${serversOf(h).join(', ')})`).join('; ')}.</p>` : ''}
93
118
  <div class="buttons"><button class="btn btn-hot">Save</button></div></form>`, '', 'workspace-list')}
94
119
  ${region('Installation', html`<p class="muted u-mt0">Set in the server's environment by its administrators (<code>PGKILN_REST_ALLOWED_HOSTS</code>).</p>
95
120
  ${inst.allowed.includes('*') ? html`<p>Any public host (<code>*</code>).</p>` : entries(inst.allowed)}
96
121
  ${m.admin ? html`<h3 class="u-mt1">Private hosts</h3><p class="muted small">Hosts on private or loopback addresses that may be called (<code>PGKILN_REST_PRIVATE_HOSTS</code>).</p>${entries(inst.private)}` : ''}`)}
97
122
  </div>
123
+ ${region('Remote servers', html`<p class="muted u-mt0">The hosts this workspace's <a href="${BASE}/remote-servers">remote servers</a> stand for now: the REST data sources, catalogs and AI services on a server call this host.</p>
124
+ <div class="table-wrap"><table class="report report-reflow">
125
+ <thead><tr><th>Server</th><th>Host</th><th>From</th><th>Would be called</th></tr></thead>
126
+ <tbody>${servers.length ? servers.map((x) => html`<tr><td data-label="Server">${x.static_id}</td>
127
+ <td data-label="Host" class="u-break">${x.host ? html`<code>${x.host}</code>` : html`<span class="muted">none</span>`}</td>
128
+ <td data-label="From">${x.from === 'env' ? x.env_var : x.from === 'stored' ? 'base URL' : ''}</td>
129
+ <td data-label="Would be called">${x.problem ? html`<span class="tag tag-error">no</span> <span class="muted small">${x.problem}</span>` : html`<span class="tag tag-ok">yes</span>`}</td></tr>`)
130
+ : html`<tr><td colspan="4" class="empty">No remote servers.</td></tr>`}</tbody></table></div>`, '', 'servers')}
98
131
  ${region('Check a URL', html`<form method="get" action="${BASE}/network" class="search u-mwnone">
99
132
  <input type="hidden" name="ws" value="${w.id}">
100
133
  ${input('url', 'URL', tryUrl, { type: 'url', placeholder: 'https://api.example.com/v1/items' })}
@@ -144,7 +144,7 @@ export async function restCatalogRoutes(app: FastifyInstance) {
144
144
  const s = await developer(req, reply);
145
145
  if (!s) return;
146
146
  const rows = (await owner.query(
147
- `select c.id, c.name, c.description, c.base_url, c.source, c.api_version, c.updated_at,
147
+ `select c.id, c.name, c.description, c.base_url, c.server, c.source, c.api_version, c.updated_at,
148
148
  (select count(*)::int from meta.rest_catalog_service x where x.catalog_id = c.id) as services
149
149
  from meta.rest_catalog c where c.workspace_id = $1 order by lower(c.name)`, [currentWorkspace(s)])).rows;
150
150
  const main = html`<div class="title-row"><h1>REST Source Catalogs</h1></div>
@@ -154,7 +154,7 @@ export async function restCatalogRoutes(app: FastifyInstance) {
154
154
  <thead><tr><th>Catalog</th><th>Base URL</th><th class="num">Services</th><th>From</th><th>Updated (UTC)</th></tr></thead>
155
155
  <tbody>${rows.length ? rows.map((c) => html`<tr>
156
156
  <td data-label="Catalog"><a href="${root}/${c.id}">${c.name}</a>${c.description ? html`<br><span class="muted small">${String(c.description).slice(0, 160)}</span>` : ''}</td>
157
- <td data-label="Base URL"><code class="wrap-anywhere">${c.base_url}</code></td>
157
+ <td data-label="Base URL">${c.server ? html`<span class="tag">${icon('server')} ${c.server}</span> ` : ''}<code class="wrap-anywhere">${c.base_url ?? ''}</code></td>
158
158
  <td class="num" data-label="Services">${c.services}</td>
159
159
  <td data-label="From">${c.source === 'openapi' ? html`OpenAPI${c.api_version ? html` <span class="muted">${c.api_version}</span>` : ''}` : 'by hand'}</td>
160
160
  <td data-label="Updated">${when(c.updated_at)}</td></tr>`)
@@ -225,7 +225,7 @@ export async function restCatalogRoutes(app: FastifyInstance) {
225
225
  const sel = req.query.service ? c.services.find((x) => String(x.id) === req.query.service) : null;
226
226
  const main = html`<div class="title-row"><h1>${c.name}</h1>
227
227
  <a class="btn" href="${base}/download">${icon('download')} Download</a></div>
228
- <p class="muted u-mt0"><code class="wrap-anywhere">${c.base_url}</code>${c.source === 'openapi' ? html` · from an OpenAPI document${c.api_version ? html` (version ${c.api_version})` : ''}` : ''} · updated ${when(c.updated_at)} UTC</p>
228
+ <p class="muted u-mt0">${c.server ? html`On remote server <a href="${BASE}/remote-servers">${c.server}</a> · ` : ''}<code class="wrap-anywhere">${c.base_url ?? ''}</code>${c.source === 'openapi' ? html` · from an OpenAPI document${c.api_version ? html` (version ${c.api_version})` : ''}` : ''} · updated ${when(c.updated_at)} UTC</p>
229
229
  ${notesHtml(s)}
230
230
  ${region('Services', html`<div class="table-wrap"><table class="report report-reflow">
231
231
  <thead><tr><th>Service</th><th>Method</th><th>Path</th><th class="num">Parameters</th><th class="num">Columns</th><th>Write back</th></tr></thead>
@@ -241,12 +241,13 @@ export async function restCatalogRoutes(app: FastifyInstance) {
241
241
  ${req.query.service === 'new'
242
242
  ? region('New service', serviceForm(s, `${base}/services`, { method: 'GET', path: '/' }, 'Add'), '', 'service')
243
243
  : sel
244
- ? region(`Service ${sel.name}`, html`<p class="muted u-mt0">URL: <code class="wrap-anywhere">${serviceUrl(c.base_url, sel.path)}</code></p>
244
+ ? region(`Service ${sel.name}`, html`<p class="muted u-mt0">URL: <code class="wrap-anywhere">${c.server ? `${c.server} + ${sel.path}` : serviceUrl(c.base_url ?? '', sel.path)}</code></p>
245
245
  ${serviceForm(s, `${base}/services/${sel.id}`, sel, 'Save')}
246
246
  <form method="post" action="${base}/services/${sel.id}/delete" class="danger-zone">${csrf(s)}<button class="btn btn-danger" data-confirm="Delete the service ${sel.name}?">Delete service</button></form>`, '', 'service')
247
247
  : ''}
248
248
  ${region('Catalog', html`<form method="post" action="${base}">${csrf(s)}
249
- <div class="form-grid">${input('name', 'Name', c.name, { required: true })}${input('base_url', 'Base URL', c.base_url, { required: true, help: 'e.g. a test system\'s URL. Data sources made earlier keep the URL they were made with.' })}
249
+ <div class="form-grid">${input('name', 'Name', c.name, { required: true })}${input('base_url', 'Base URL', c.base_url ?? '', { help: 'e.g. a test system\'s URL (optional on a remote server). Data sources made earlier keep the URL they were made with.' })}
250
+ ${input('server', 'Remote server', c.server ?? '', { placeholder: 'e.g. CRM', help: 'The static id of a remote server of this workspace (Workspace utilities → Remote servers): data sources made from the catalog then name the server and the service\'s path, so they move with it.' })}
250
251
  ${input('api_version', 'API version', c.api_version ?? '')}
251
252
  <div class="field" data-wide><label class="label" for="f_cdescription">Description</label><textarea id="f_cdescription" name="description" rows="2">${c.description ?? ''}</textarea></div></div>
252
253
  <div class="buttons"><button class="btn btn-hot">${icon('save')} Save</button></div></form>`)}
@@ -262,14 +263,19 @@ export async function restCatalogRoutes(app: FastifyInstance) {
262
263
  if (!c) return reply.code(404).send('Not found');
263
264
  const v = {
264
265
  name: String(req.body?.name ?? '').trim(), base_url: trimBase(String(req.body?.base_url ?? '')),
266
+ server: String(req.body?.server ?? '').trim().toUpperCase() || null,
265
267
  description: String(req.body?.description ?? '').trim() || null, api_version: String(req.body?.api_version ?? '').trim() || null,
266
268
  };
267
269
  // the services must still make sense on the new base URL
268
- const problems = [...catalogProblems(v), ...c.services.flatMap((x) => serviceProblems(v.base_url, x))];
270
+ const problems = [...catalogProblems(v), ...c.services.flatMap((x) => serviceProblems(v.base_url || 'https://x', x))];
269
271
  if (problems.length) flash(s, problems.slice(0, 6).join(' '), 'error');
270
272
  else
271
273
  try {
272
- await owner.query('update meta.rest_catalog set name = $2, base_url = $3, description = $4, api_version = $5, updated_at = now() where id = $1', [c.id, v.name, v.base_url, v.description, v.api_version]);
274
+ await owner.query('update meta.rest_catalog set name = $2, base_url = $3, description = $4, api_version = $5, server = $6, updated_at = now() where id = $1',
275
+ [c.id, v.name, v.base_url || null, v.description, v.api_version, v.server]);
276
+ if (v.server && !(await owner.one('select 1 as ok from meta.remote_server where workspace_id = $1 and static_id = $2', [c.workspace_id, v.server])))
277
+ flash(s, `Catalog saved. This workspace has no remote server ${v.server} yet: add it under Workspace utilities → Remote servers.`, 'error');
278
+ else
273
279
  flash(s, 'Catalog saved.');
274
280
  } catch (e) {
275
281
  flash(s, uniqueError(e), 'error');
@@ -292,6 +298,8 @@ export async function restCatalogRoutes(app: FastifyInstance) {
292
298
  if (doc instanceof Error) throw doc;
293
299
  doc = await documentOf(doc, s);
294
300
  const { catalog, notes } = readCatalogText(doc.text, { baseUrl: req.body?.base_url, from: doc.from, name: c.name });
301
+ // (141) the catalog stays on its remote server
302
+ catalog.server = c.server ?? null;
295
303
  const problems = wholeCatalogProblems(catalog);
296
304
  if (problems.length) throw new Error(problems.slice(0, 8).join(' '));
297
305
  await owner.tx((db) => replaceCatalog(db, c.id, catalog));
@@ -338,7 +346,7 @@ export async function restCatalogRoutes(app: FastifyInstance) {
338
346
  const retry = `${root}/${c.id}?service=${sid ?? 'new'}#service`;
339
347
  try {
340
348
  const v = serviceFromBody(req.body ?? {});
341
- const problems = serviceProblems(c.base_url, v);
349
+ const problems = serviceProblems(c.base_url || 'https://x', v);
342
350
  if (problems.length) throw new Error(problems.join(' '));
343
351
  const vals = [v.name, v.description, v.path, v.method, JSON.stringify(v.params), JSON.stringify(v.headers), v.body, v.row_selector, JSON.stringify(v.columns), v.key_columns, JSON.stringify(v.operations)];
344
352
  let id = sid;
@@ -388,14 +396,14 @@ export async function restCatalogRoutes(app: FastifyInstance) {
388
396
  const c = (await loadCatalog(svc.catalog_id))!;
389
397
  const v = sourceFromService(c, c.services.find((x) => x.id === svc.id)!, { name: req.body?.name, credential: req.body?.credential });
390
398
  if (!/^[A-Z][A-Z0-9_]{0,59}$/.test(v.name)) throw new Error('The name is upper case letters, digits and _, starting with a letter, e.g. COUNTRIES.');
391
- if (v.credential && !(await owner.one('select 1 as ok from meta.web_credential where app_id = $1 and name = $2', [appId, v.credential])))
392
- throw new Error(`This application has no web credential ${v.credential}.`);
399
+ if (v.credential && !(await owner.one('select 1 as ok from meta.web_credential where app_id = $1 and name = $2 union all select 1 from meta.workspace_credential where workspace_id = meta.app_workspace($1) and name = $2', [appId, v.credential])))
400
+ throw new Error(`This application has no web credential ${v.credential} (nor does its workspace).`);
393
401
  const problems = sourceProblems(v);
394
402
  if (problems.length) throw new Error(problems.join(' '));
395
403
  const r = await owner.one<{ id: number }>(
396
- `insert into meta.rest_source (app_id, name, description, url, method, credential, params, headers, body, row_selector, columns, key_columns, operations)
397
- values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) returning id`,
398
- [appId, v.name, v.description, v.url, v.method, v.credential, JSON.stringify(v.params), JSON.stringify(v.headers), v.body, v.row_selector, JSON.stringify(v.columns), v.key_columns, JSON.stringify(v.operations)]);
404
+ `insert into meta.rest_source (app_id, name, description, url, method, credential, params, headers, body, row_selector, columns, key_columns, operations, server)
405
+ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) returning id`,
406
+ [appId, v.name, v.description, v.url, v.method, v.credential, JSON.stringify(v.params), JSON.stringify(v.headers), v.body, v.row_selector, JSON.stringify(v.columns), v.key_columns, JSON.stringify(v.operations), v.server]);
399
407
  flash(s, `REST data source ${v.name} made from ${c.name}. Test it below${v.credential ? '' : '; add a web credential if the service needs one'}.`);
400
408
  return back(reply, s, `${BASE}/apps/${appId}/shared?c=rest_source-${r!.id}`);
401
409
  } catch (e) {
@@ -423,7 +431,7 @@ export async function catalogPicker(appId: number, s: Session): Promise<Raw | ''
423
431
  where c.workspace_id = meta.app_workspace($1) order by lower(c.name), x.position, x.name`, [appId])).rows;
424
432
  if (!services.length)
425
433
  return html`<p class="muted u-mt0 u-mb1">${icon('globe')} Services described once for the workspace: <a href="${root}">REST Source Catalogs</a>.</p>`;
426
- const credentials = (await owner.query('select name from meta.web_credential where app_id = $1 order by name', [appId])).rows;
434
+ const credentials = (await owner.query('select name from meta.web_credential where app_id = $1 union select name from meta.workspace_credential where workspace_id = meta.app_workspace($1) order by name', [appId])).rows;
427
435
  const groups = new Map<string, typeof services>();
428
436
  for (const x of services) groups.set(x.catalog, [...(groups.get(x.catalog) ?? []), x]);
429
437
  return region('From a REST source catalog', html`<form method="post" action="${BASE}/apps/${appId}/rest-sources/from-catalog">${csrf(s)}
@@ -603,7 +603,11 @@ export async function builderRoutes(app: FastifyInstance) {
603
603
  });
604
604
  // supporting objects are never run on import: the developer reviews them and chooses
605
605
  const scripts = (await owner.one('select count(*)::int as n from meta.supporting_script where app_id = $1', [r.id])).n;
606
- flash(s, `Application imported. Check its database role and users under Settings / Shared Components.${scripts ? ` It has ${scripts} supporting object script(s): they were not run.` : ''}`);
606
+ // (141) remote servers its REST data sources name that this workspace doesn't have (yet)
607
+ const servers = (await owner.query<{ server: string }>(
608
+ `select distinct x.server from meta.rest_source x where x.app_id = $1 and x.server is not null
609
+ and not exists (select 1 from meta.remote_server r where r.workspace_id = meta.app_workspace($1) and r.static_id = x.server) order by 1`, [r.id])).rows.map((x) => x.server);
610
+ flash(s, `Application imported. Check its database role and users under Settings / Shared Components.${scripts ? ` It has ${scripts} supporting object script(s): they were not run.` : ''}${servers.length ? ` Its REST data sources use remote server${servers.length > 1 ? 's' : ''} ${servers.join(', ')}, which this workspace doesn't have: add ${servers.length > 1 ? 'them' : 'it'} under Workspace utilities → Remote servers.` : ''}`);
607
611
  return back(reply, s, scripts ? `${BASE}/apps/${r.id}/supporting-objects?imported=1` : `${BASE}/apps/${r.id}`);
608
612
  } catch (e) {
609
613
  flash(s, `Import failed: ${(e as Error).message}`, 'error');
@@ -5,6 +5,7 @@ import { icon } from '../icons.ts';
5
5
  import { readMultipart } from '../runtime/files.ts';
6
6
  import { includable, STATIC_MAX, STATIC_TYPES, staticType } from '../runtime/static-files.ts';
7
7
  import type { Session } from '../session.ts';
8
+ import { WORKSPACE_FILES } from '../metadata.ts';
8
9
  import { appOr404 } from './forms.ts';
9
10
  import { appHeader, back, BASE, csrf, developer, flash, input, region, send, shell, type Body, type Req } from './ui.ts';
10
11
 
@@ -17,9 +18,15 @@ import { appHeader, back, BASE, csrf, developer, flash, input, region, send, she
17
18
  /** Files a list may name (every page, or one page). */
18
19
  const MAX_INCLUDES = 20;
19
20
 
20
- /** "a.js, b.css" → the valid, includable, distinct names, in order. */
21
+ /**
22
+ * "a.js, b.css" → the valid, includable, distinct names, in order. (141) "#WORKSPACE_FILES#theme.css"
23
+ * is a static workspace file; "#APP_FILES#a.js" is the same as "a.js".
24
+ */
21
25
  export function parseIncludes(text: string | undefined) {
22
- const names = (text ?? '').split(/[\s,]+/).map((n) => n.trim()).filter((n) => n && staticType(n) && includable(n));
26
+ const names = (text ?? '').split(/[\s,]+/).map((n) => n.trim().replace(/^#APP_FILES#/, '')).filter((n) => {
27
+ const file = n.startsWith(WORKSPACE_FILES) ? n.slice(WORKSPACE_FILES.length) : n;
28
+ return file && staticType(file) && includable(file);
29
+ });
23
30
  return [...new Set(names)].slice(0, MAX_INCLUDES);
24
31
  }
25
32
 
@@ -84,7 +91,7 @@ async function page(s: Session, a: any, query: Record<string, string | undefined
84
91
  <small class="help">A file with the same name is replaced.</small>
85
92
  </form>`)}
86
93
  ${region('Every page loads', html`<form method="post" action="${base}/includes">${csrf(s)}
87
- <div class="form-grid">${input('includes', 'Scripts and stylesheets, in order', (a.static_includes ?? []).join(', '), { placeholder: 'e.g. library.js, app.js, app-styles.css', help: `Up to ${MAX_INCLUDES} .js and .css files, loaded in this order after pgkiln's own (scripts deferred). A page adds its own files under Page → Appearance.` })}</div>
94
+ <div class="form-grid">${input('includes', 'Scripts and stylesheets, in order', (a.static_includes ?? []).join(', '), { placeholder: 'e.g. library.js, app.js, app-styles.css', help: `Up to ${MAX_INCLUDES} .js and .css files, loaded in this order after pgkiln's own (scripts deferred). #WORKSPACE_FILES#name is a static workspace file (Workspace utilities). A page adds its own files under Page → Appearance.` })}</div>
88
95
  <div class="buttons"><button class="btn btn-hot">Save</button></div>
89
96
  </form>`)}`;
90
97
  }
@@ -191,7 +198,10 @@ export async function staticFileBuilderRoutes(app: FastifyInstance) {
191
198
  if (!a) return reply.code(404).send('Not found');
192
199
  const names = parseIncludes(req.body?.includes);
193
200
  await owner.query('update meta.app set static_includes = $2, updated_at = now() where id = $1', [a.id, names]);
194
- const missing = (await owner.query<{ n: string }>('select n from unnest($2::text[]) n where not exists (select 1 from meta.static_file f where f.app_id = $1 and f.name = n)', [a.id, names])).rows.map((r) => r.n);
201
+ const missing = (await owner.query<{ n: string }>(
202
+ `select n from unnest($2::text[]) n
203
+ where not exists (select 1 from meta.static_file f where f.app_id = $1 and f.name = n)
204
+ and not exists (select 1 from meta.workspace_file w where w.workspace_id = meta.app_workspace($1) and $3 || w.name = n)`, [a.id, names, WORKSPACE_FILES])).rows.map((r) => r.n);
195
205
  flash(s, missing.length ? `Saved. Not uploaded yet (left out until they are): ${missing.join(', ')}.` : 'Saved.');
196
206
  return back(reply, s, `${BASE}/apps/${a.id}/static-files`);
197
207
  });
@@ -9,6 +9,7 @@ import { runSync, syncLog, syncProblems } from '../restsync.ts';
9
9
  import { buildRequest, call, clearResponseCache, clearTokens, credentialProblems, guessColumns, loadCredential, parseJson, rowsSql, select, sourceProblems, toRows, withRest, type RestColumn, type RestSource } from '../websources.ts';
10
10
  import type { ComponentSpec } from './components.ts';
11
11
  import { back, BASE, csrf, developer, flash, type Req } from './ui.ts';
12
+ import { resolveSource } from '../remote-servers.ts';
12
13
 
13
14
  // Shared Components → Web credentials and REST data sources: their property
14
15
  // specs, the secrets' status (never a secret), "Test" for a source (its
@@ -75,7 +76,7 @@ export const REST_SOURCE_SPEC: ComponentSpec = {
75
76
  if (v.sync_time_zone === null) v.sync_time_zone = 'UTC';
76
77
  if (v.sync_mode === null) v.sync_mode = 'merge';
77
78
  const problems = [
78
- ...sourceProblems({ url: v.url, params: json(v.params), columns: json(v.columns), headers: json(v.headers), row_selector: v.row_selector, key_columns: v.key_columns, operations: json(v.operations) }),
79
+ ...sourceProblems({ url: v.url, server: v.server, params: json(v.params), columns: json(v.columns), headers: json(v.headers), row_selector: v.row_selector, key_columns: v.key_columns, operations: json(v.operations) }),
79
80
  ...syncProblems(v),
80
81
  ];
81
82
  return problems.length ? problems.join(' ') : null;
@@ -88,10 +89,12 @@ export const REST_SOURCE_SPEC: ComponentSpec = {
88
89
  fields: [
89
90
  { name: 'name', label: 'Name', kind: 'upper', group: 'Identification', help: 'Regions and lists of values use it by this name, e.g. COUNTRIES.' },
90
91
  { name: 'description', label: 'Description', kind: 'text', wide: true, group: 'Identification' },
92
+ { name: 'server', label: 'Remote server', kind: 'upper', group: 'Request',
93
+ help: 'Optional: the static id of a remote server of the workspace (Workspace utilities → Remote servers), e.g. CRM. The URL is then the path after the server\'s base URL, and changing the server moves every source on it.' },
91
94
  { name: 'url', label: 'URL', kind: 'text', wide: true, group: 'Request',
92
- help: 'e.g. https://api.example.com/v1/cities/{city}/weather — {name} is a path parameter. The host must be on the server\'s allow-list (PGKILN_REST_ALLOWED_HOSTS).' },
95
+ help: 'e.g. https://api.example.com/v1/cities/{city}/weather — {name} is a path parameter. With a remote server: the path, e.g. /v1/cities/{city}/weather. The host must be on the server\'s allow-list (PGKILN_REST_ALLOWED_HOSTS).' },
93
96
  { name: 'method', label: 'Method', kind: 'select', options: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'], group: 'Request' },
94
- { name: 'credential', label: 'Web credential', kind: 'upper', group: 'Request', help: 'The name of a web credential, or empty.' },
97
+ { name: 'credential', label: 'Web credential', kind: 'upper', group: 'Request', help: 'The name of a web credential (the application\'s, else one of the workspace\'s), or empty: on a remote server, the server\'s default credential.' },
95
98
  { name: 'params', label: 'Parameters (JSON)', kind: 'json', wide: true, group: 'Request',
96
99
  help: '[{"name": "city", "in": "path", "default": "&P1_CITY.", "required": true}, {"name": "units", "in": "query", "default": "metric"}] · in: path, query, header, body. Defaults may use &ITEM. substitutions; regions override them with {"rest_params": {…}}.' },
97
100
  { name: 'headers', label: 'Headers (JSON)', kind: 'json', group: 'Request', help: '{"Accept-Language": "en"} — not for secrets: use a web credential.' },
@@ -267,11 +270,14 @@ export async function webSourceRoutes(app: FastifyInstance) {
267
270
  app.post(`${BASE}/apps/:id(^\\d+$)/rest-sources/:sid(^\\d+$)/test`, async (req: Req, reply) => {
268
271
  const s = await developer(req, reply);
269
272
  if (!s) return;
270
- const src = await owner.one<RestSource>('select * from meta.rest_source where id = $1 and app_id = $2', [req.params.sid, req.params.id]);
271
- if (!src) return reply.code(404).send('Not found');
272
- const result: TestResult = { id: src.id, ok: false, message: '' };
273
+ const stored = await owner.one<RestSource>('select * from meta.rest_source where id = $1 and app_id = $2', [req.params.sid, req.params.id]);
274
+ if (!stored) return reply.code(404).send('Not found');
275
+ const result: TestResult = { id: stored.id, ok: false, message: '' };
273
276
  const started = Date.now();
277
+ let src = stored;
274
278
  try {
279
+ // (141) on a remote server: its URL now
280
+ src = await resolveSource(stored);
275
281
  const values: Record<string, string> = {};
276
282
  for (const p of src.params) values[p.name] = String(req.body?.[`p_${p.name}`] ?? '');
277
283
  const r = buildRequest(src, values);
@@ -73,7 +73,8 @@ export async function workflowExtras(appId: number, row: Definition, s: Session,
73
73
  const shown = dev ? (row.dev_steps ?? []) : row.steps;
74
74
  const refs = {
75
75
  sources: new Map((await owner.query('select name, params, columns from meta.rest_source where app_id = $1', [appId])).rows.map((r) => [r.name as string, r])),
76
- credentials: new Set((await owner.query('select name from meta.web_credential where app_id = $1', [appId])).rows.map((r) => r.name as string)),
76
+ // (141) the application's credentials and its workspace's
77
+ credentials: new Set((await owner.query('select name from meta.web_credential where app_id = $1 union select name from meta.workspace_credential where workspace_id = meta.app_workspace($1)', [appId])).rows.map((r) => r.name as string)),
77
78
  startVars: [...String((row as { title?: string }).title ?? '').matchAll(/&([A-Za-z][A-Za-z0-9_]*)\./g)].map((m) => m[1]),
78
79
  };
79
80
  const invokeRefs = invokeStepReferences(shown, refs);