pgkiln 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SECURITY.md +3 -2
- package/db/migrations/116_dump_jobs.sql +41 -0
- package/db/migrations/117_rest_handlers.sql +63 -0
- package/db/migrations/118_change_log.sql +54 -0
- package/docs/apex-feature-parity.md +9 -9
- package/docs/guide/01-installation.md +7 -0
- package/docs/guide/03-builder.md +227 -10
- package/docs/guide/08-security.md +11 -0
- package/docs/guide/12-development.md +64 -13
- package/docs/guide/13-rest-api.md +59 -1
- package/docs/guide/15-extensions.md +1 -1
- package/examples/hr/hr_62_rest_module.sql +39 -0
- package/package.json +1 -1
- package/public/app.css +1 -1
- package/public/builder-icons.svg +2 -0
- package/public/builder.css +478 -173
- package/public/builder.js +48 -0
- package/public/code-editor.css +32 -79
- package/public/code-editor.js +58 -3
- package/public/data-modeler.css +17 -14
- package/public/sql-monitor.js +73 -0
- package/public/worksheet.css +14 -7
- package/public/worksheet.js +119 -3
- package/src/app.ts +6 -0
- package/src/automations.ts +6 -0
- package/src/binds.ts +2 -1
- package/src/builder/advisor.ts +24 -2
- package/src/builder/autorest.ts +3 -2
- package/src/builder/changelog.ts +240 -0
- package/src/builder/components.ts +1 -1
- package/src/builder/data-modeler.ts +1 -1
- package/src/builder/datapump.ts +364 -0
- package/src/builder/db-users.ts +2 -2
- package/src/builder/ddl-tools.ts +13 -4
- package/src/builder/find-workspace.ts +2 -2
- package/src/builder/object-browser.ts +4 -4
- package/src/builder/plan-view.ts +25 -0
- package/src/builder/querybuilder.ts +1 -1
- package/src/builder/quicksql.ts +1 -1
- package/src/builder/rest-modules.ts +629 -0
- package/src/builder/rest.ts +1 -0
- package/src/builder/routes.ts +4 -2
- package/src/builder/scheduling.ts +495 -0
- package/src/builder/scripts.ts +1 -1
- package/src/builder/sql-charts.ts +1 -1
- package/src/builder/sql-monitor.ts +397 -0
- package/src/builder/sql-tools.ts +6 -2
- package/src/builder/sql-utilities.ts +7 -39
- package/src/builder/sql.ts +6 -0
- package/src/builder/table-editor.ts +6 -3
- package/src/builder/top-sql.ts +24 -6
- package/src/builder/ui.ts +6 -4
- package/src/builder/worksheet.ts +173 -41
- package/src/changelog.ts +109 -0
- package/src/cron-schedule.ts +282 -0
- package/src/datapump.ts +546 -0
- package/src/dumpscript.ts +407 -0
- package/src/pgcron.ts +225 -0
- package/src/runtime/rest-handlers.ts +75 -0
- package/src/runtime/rest.ts +116 -22
- package/src/session.ts +5 -0
- package/src/sqlworksheet.ts +9 -6
- package/src/worksheet-txn.ts +270 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto';
|
|
2
|
+
import pg from 'pg';
|
|
3
|
+
import { onClosePools, owner, type Db } from './db.ts';
|
|
4
|
+
import { onSessionEnd, sessionLimits } from './session.ts';
|
|
5
|
+
import { isSchemaDdl } from './changelog.ts';
|
|
6
|
+
|
|
7
|
+
// (sprint 107, DA-2) Transaction mode of the SQL Worksheet. Database Actions autocommits every statement;
|
|
8
|
+
// in transaction mode a worksheet tab keeps one connection of its own with an open transaction between
|
|
9
|
+
// runs, until Commit, Rollback, a COMMIT / ROLLBACK typed in the editor, or an idle timeout.
|
|
10
|
+
//
|
|
11
|
+
// The registry lives in this process only (no table): a transaction is a live connection, so it can't
|
|
12
|
+
// outlive the server anyway. Each open transaction is bound to the builder session that started it, its
|
|
13
|
+
// developer, workspace and worksheet tab, and has a random id; a lookup always starts from the caller's own
|
|
14
|
+
// session id, so one developer can never reach another one's connection. Limits: a few per developer and
|
|
15
|
+
// per instance (each holds a database connection outside the pools), an idle timeout (rolled back by a
|
|
16
|
+
// sweeper) and the connection's idle_in_transaction_session_timeout a little later as the backstop. The
|
|
17
|
+
// connection is a plain pg.Client made from the workshop's (or the "Run as" application's) pool settings,
|
|
18
|
+
// never a pooled one: SET ROLE and settings typed by the developer die with it.
|
|
19
|
+
|
|
20
|
+
const intEnv = (name: string, dflt: number, min: number, max: number) => {
|
|
21
|
+
const n = Number(process.env[name]);
|
|
22
|
+
return Number.isFinite(n) && n >= min && n <= max ? Math.floor(n) : dflt;
|
|
23
|
+
};
|
|
24
|
+
/** Seconds without a run before an open transaction is rolled back (WORKSHEET_TXN_IDLE_SECONDS, default 5 minutes). */
|
|
25
|
+
export const txnIdleSeconds = () => intEnv('WORKSHEET_TXN_IDLE_SECONDS', 300, 10, 86_400);
|
|
26
|
+
/** Open transactions per developer (WORKSHEET_TXN_MAX_PER_DEVELOPER, default 3) and per server (WORKSHEET_TXN_MAX, default 20). */
|
|
27
|
+
export const txnMaxPerDeveloper = () => intEnv('WORKSHEET_TXN_MAX_PER_DEVELOPER', 3, 1, 100);
|
|
28
|
+
export const txnMax = () => intEnv('WORKSHEET_TXN_MAX', 20, 1, 1000);
|
|
29
|
+
/** The database's own backstop: this much after the idle timeout it ends the connection itself. */
|
|
30
|
+
const BACKSTOP_SECONDS = 60;
|
|
31
|
+
/** How long the note of a transaction the server ended (timeout, workspace switch) stays for the worksheet. */
|
|
32
|
+
const NOTE_MS = 60 * 60 * 1000;
|
|
33
|
+
|
|
34
|
+
export interface WorksheetTxn {
|
|
35
|
+
/** random handle (shown to the page, never trusted for a lookup on its own) */
|
|
36
|
+
id: string;
|
|
37
|
+
sessionId: string;
|
|
38
|
+
username: string;
|
|
39
|
+
workspaceId: number;
|
|
40
|
+
tab: string;
|
|
41
|
+
client: pg.Client;
|
|
42
|
+
/** the application whose role the transaction runs as (Run as), or null */
|
|
43
|
+
asAppId: number | null;
|
|
44
|
+
/** who it runs as, for the badge */
|
|
45
|
+
runsAs: string;
|
|
46
|
+
startedMs: number;
|
|
47
|
+
lastMs: number;
|
|
48
|
+
/** statements run in the current transaction */
|
|
49
|
+
statements: number;
|
|
50
|
+
/** a run (or commit / rollback) is going on */
|
|
51
|
+
busy: boolean;
|
|
52
|
+
/** DDL that succeeded in the transaction: it goes into the Change Log at COMMIT, not before */
|
|
53
|
+
pendingDdl: string[];
|
|
54
|
+
defaultSchema: string;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export type TxnState = 'open' | 'aborted' | 'none';
|
|
58
|
+
|
|
59
|
+
const open = new Map<string, WorksheetTxn>();
|
|
60
|
+
const notes = new Map<string, { message: string; at: number }>();
|
|
61
|
+
const keyOf = (sessionId: string, tab: string) => `${sessionId}\u0000${tab}`;
|
|
62
|
+
|
|
63
|
+
/** A worksheet tab's key as the page sends it (src/builder/worksheet.ts uses the same rule). */
|
|
64
|
+
export const validTab = (tab: unknown) => typeof tab === 'string' && /^[a-z0-9]{1,20}$/.test(tab);
|
|
65
|
+
|
|
66
|
+
/** The open transaction of this session's worksheet tab. */
|
|
67
|
+
export function findTxn(sessionId: string, tab: string): WorksheetTxn | undefined {
|
|
68
|
+
return validTab(tab) ? open.get(keyOf(sessionId, tab)) : undefined;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Open, aborted (a statement failed: only ROLLBACK helps) or none (committed or rolled back by a typed statement). */
|
|
72
|
+
export function txnState(t: WorksheetTxn | undefined): TxnState {
|
|
73
|
+
const s = t?.client.getTransactionStatus();
|
|
74
|
+
return s === 'T' ? 'open' : s === 'E' ? 'aborted' : 'none';
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Why the server ended this tab's last transaction (idle timeout, …), if it did. */
|
|
78
|
+
export function txnNote(sessionId: string, tab: string): string | null {
|
|
79
|
+
const n = notes.get(keyOf(sessionId, tab));
|
|
80
|
+
if (!n) return null;
|
|
81
|
+
if (Date.now() - n.at > NOTE_MS) {
|
|
82
|
+
notes.delete(keyOf(sessionId, tab));
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
return n.message;
|
|
86
|
+
}
|
|
87
|
+
export const clearTxnNote = (sessionId: string, tab: string) => notes.delete(keyOf(sessionId, tab));
|
|
88
|
+
|
|
89
|
+
/** Open transactions (tests and the instance's monitor). */
|
|
90
|
+
export const openTxnCount = (username?: string) => [...open.values()].filter((t) => username === undefined || t.username === username).length;
|
|
91
|
+
|
|
92
|
+
/** Statements PostgreSQL refuses inside a transaction block (refused before they abort it): the command, or ''. */
|
|
93
|
+
export const outsideTxnOnly = (sql: string): string => {
|
|
94
|
+
const s = sql.replace(/^(\s|--[^\n]*\n|\/\*[\s\S]*?\*\/)+/, '').toLowerCase();
|
|
95
|
+
return /^(vacuum|alter\s+system|(create|drop)\s+(database|tablespace)|(create|drop)\s+(unique\s+)?index\s+concurrently|reindex\b[^;]*\bconcurrently|alter\s+table\b[^;]*\bdetach\s+partition\b[^;]*\bconcurrently|prepare\s+transaction|commit\s+prepared|rollback\s+prepared)\b/.exec(s)?.[0].replace(/\s+/g, ' ').toUpperCase() ?? '';
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
export interface BeginOptions {
|
|
99
|
+
sessionId: string;
|
|
100
|
+
username: string;
|
|
101
|
+
workspaceId: number;
|
|
102
|
+
tab: string;
|
|
103
|
+
/** the pool whose login the connection uses (the workshop's, or the Run as application's) */
|
|
104
|
+
db: Db;
|
|
105
|
+
asApp?: { id: number; db_role: string } | null;
|
|
106
|
+
runsAs: string;
|
|
107
|
+
searchSchema: string | null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Start a transaction for a worksheet tab: a connection of its own, BEGIN. A string is a refusal. */
|
|
111
|
+
export async function beginTxn(o: BeginOptions): Promise<WorksheetTxn | string> {
|
|
112
|
+
if (!validTab(o.tab)) return 'Transaction mode needs the worksheet tab (JavaScript).';
|
|
113
|
+
const key = keyOf(o.sessionId, o.tab);
|
|
114
|
+
if (open.has(key)) return 'This worksheet already has an open transaction.';
|
|
115
|
+
const mine = openTxnCount(o.username);
|
|
116
|
+
if (mine >= txnMaxPerDeveloper())
|
|
117
|
+
return `You have ${mine} worksheets with an open transaction, the most allowed: commit or roll back one of them first.`;
|
|
118
|
+
if (open.size >= txnMax()) return 'The server has as many open worksheet transactions as it allows: try again later, or use Autocommit.';
|
|
119
|
+
const idle = txnIdleSeconds();
|
|
120
|
+
const opts = o.db.pool.options;
|
|
121
|
+
const client = new pg.Client({
|
|
122
|
+
connectionString: opts.connectionString,
|
|
123
|
+
password: opts.password,
|
|
124
|
+
ssl: opts.ssl,
|
|
125
|
+
application_name: 'pgkiln-worksheet-txn',
|
|
126
|
+
connectionTimeoutMillis: 10_000,
|
|
127
|
+
idle_in_transaction_session_timeout: (idle + BACKSTOP_SECONDS) * 1000,
|
|
128
|
+
});
|
|
129
|
+
const now = Date.now();
|
|
130
|
+
const t: WorksheetTxn = {
|
|
131
|
+
id: randomBytes(16).toString('hex'), sessionId: o.sessionId, username: o.username, workspaceId: o.workspaceId, tab: o.tab, client,
|
|
132
|
+
asAppId: o.asApp?.id ?? null, runsAs: o.runsAs, startedMs: now, lastMs: now, statements: 0, busy: true, pendingDdl: [], defaultSchema: o.searchSchema ?? 'public',
|
|
133
|
+
};
|
|
134
|
+
open.set(key, t); // before the first await: the limits count it at once
|
|
135
|
+
notes.delete(key);
|
|
136
|
+
// the database ended the connection (its backstop timeout, a restart, pg_terminate_backend)
|
|
137
|
+
const gone = () => {
|
|
138
|
+
if (open.get(key) !== t) return;
|
|
139
|
+
open.delete(key);
|
|
140
|
+
notes.set(key, { message: 'The database ended the transaction\'s connection: its changes were rolled back.', at: Date.now() });
|
|
141
|
+
};
|
|
142
|
+
client.on('error', gone);
|
|
143
|
+
client.on('end', gone);
|
|
144
|
+
try {
|
|
145
|
+
await client.connect();
|
|
146
|
+
if (o.searchSchema) await client.query(`select set_config('search_path', $1, false)`, [`${pg.escapeIdentifier(o.searchSchema)}, public`]);
|
|
147
|
+
if (o.asApp) {
|
|
148
|
+
await client.query(`select set_config('pgkiln.app_id', $1, false)`, [String(o.asApp.id)]);
|
|
149
|
+
await client.query(`set role ${pg.escapeIdentifier(o.asApp.db_role)}`);
|
|
150
|
+
}
|
|
151
|
+
await client.query('begin');
|
|
152
|
+
} catch (e) {
|
|
153
|
+
open.delete(key);
|
|
154
|
+
await client.end().catch(() => {});
|
|
155
|
+
return `The transaction could not start: ${(e as Error).message}`;
|
|
156
|
+
} finally {
|
|
157
|
+
t.busy = false;
|
|
158
|
+
}
|
|
159
|
+
startSweeper();
|
|
160
|
+
return t;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** BEGIN again on the transaction's connection after a COMMIT / ROLLBACK typed in the editor. */
|
|
164
|
+
export async function beginAgain(t: WorksheetTxn) {
|
|
165
|
+
await t.client.query('begin');
|
|
166
|
+
t.startedMs = Date.now();
|
|
167
|
+
t.statements = 0;
|
|
168
|
+
t.pendingDdl = [];
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* End a transaction: COMMIT or ROLLBACK, then close its connection. The tag is what the database answered
|
|
173
|
+
* (COMMIT of a failed transaction answers ROLLBACK). `note` is kept for the worksheet (the server ended it).
|
|
174
|
+
*/
|
|
175
|
+
export async function endTxn(t: WorksheetTxn, action: 'commit' | 'rollback', note?: string): Promise<{ tag: string; error?: string; ddl: string[] }> {
|
|
176
|
+
const key = keyOf(t.sessionId, t.tab);
|
|
177
|
+
if (open.get(key) === t) open.delete(key);
|
|
178
|
+
if (note) {
|
|
179
|
+
notes.set(key, { message: note, at: Date.now() });
|
|
180
|
+
if (notes.size > 1000) notes.delete(notes.keys().next().value!); // the oldest
|
|
181
|
+
}
|
|
182
|
+
let tag = action.toUpperCase();
|
|
183
|
+
let error: string | undefined;
|
|
184
|
+
try {
|
|
185
|
+
if (t.client.getTransactionStatus() !== 'I') {
|
|
186
|
+
const r = await t.client.query(action);
|
|
187
|
+
tag = (r.command ?? tag).toUpperCase();
|
|
188
|
+
}
|
|
189
|
+
} catch (e) {
|
|
190
|
+
error = (e as Error).message;
|
|
191
|
+
tag = 'ROLLBACK';
|
|
192
|
+
} finally {
|
|
193
|
+
await t.client.end().catch(() => {});
|
|
194
|
+
}
|
|
195
|
+
return { tag, error, ddl: action === 'commit' && tag === 'COMMIT' ? t.pendingDdl : [] };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Count a run's statements in the transaction and keep its DDL for the Change Log until COMMIT. A COMMIT
|
|
200
|
+
* (or END) typed in the editor hands back the DDL it committed; a ROLLBACK (or ABORT, or COMMIT of a failed
|
|
201
|
+
* transaction, which answers ROLLBACK) drops it.
|
|
202
|
+
*/
|
|
203
|
+
export function accountRun(t: WorksheetTxn, results: { status: string; sql: string; tag?: string }[]): string[] {
|
|
204
|
+
const committed: string[] = [];
|
|
205
|
+
for (const r of results) {
|
|
206
|
+
if (r.status !== 'ok') continue;
|
|
207
|
+
const tag = (r.tag ?? '').toUpperCase();
|
|
208
|
+
if (tag === 'COMMIT') {
|
|
209
|
+
committed.push(...t.pendingDdl);
|
|
210
|
+
t.pendingDdl = [];
|
|
211
|
+
t.statements = 0;
|
|
212
|
+
} else if (tag === 'ROLLBACK' && !/^\s*rollback\s+(work\s+|transaction\s+)?to\b/i.test(r.sql)) {
|
|
213
|
+
t.pendingDdl = [];
|
|
214
|
+
t.statements = 0;
|
|
215
|
+
} else if (tag !== 'BEGIN' && tag !== 'START') {
|
|
216
|
+
t.statements++;
|
|
217
|
+
if (isSchemaDdl(r.sql)) t.pendingDdl.push(r.sql);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return committed;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** Roll back and close every transaction of a session (it ended: sign-out, a new sign-in). */
|
|
224
|
+
export async function endSessionTxns(sessionId: string, note: string) {
|
|
225
|
+
await Promise.all([...open.values()].filter((t) => t.sessionId === sessionId).map((t) => endTxn(t, 'rollback', note)));
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** The idle timeout and ended sessions: roll back. `now` lets tests move the clock. */
|
|
229
|
+
export async function sweepTxns(now = Date.now()) {
|
|
230
|
+
const idleMs = txnIdleSeconds() * 1000;
|
|
231
|
+
const minutes = Math.round(txnIdleSeconds() / 6) / 10;
|
|
232
|
+
const ended: Promise<unknown>[] = [];
|
|
233
|
+
for (const t of open.values())
|
|
234
|
+
if (!t.busy && now - t.lastMs > idleMs)
|
|
235
|
+
ended.push(endTxn(t, 'rollback', `The transaction was rolled back after ${minutes} minute${minutes === 1 ? '' : 's'} without a run (${t.statements} statement${t.statements === 1 ? '' : 's'} undone).`));
|
|
236
|
+
await Promise.all(ended);
|
|
237
|
+
const sessions = [...new Set([...open.values()].map((t) => t.sessionId))];
|
|
238
|
+
if (sessions.length) {
|
|
239
|
+
// a session that ended or expired (its row may stay a while after it expires)
|
|
240
|
+
const { idleMinutes, maxHours } = sessionLimits();
|
|
241
|
+
const alive = new Set((await owner.query<{ id: string }>(
|
|
242
|
+
`select id::text as id from meta.session where id::text = any ($1)
|
|
243
|
+
and last_seen > now() - make_interval(mins => $2) and created_at > now() - make_interval(hours => $3)`, [sessions, idleMinutes, maxHours])
|
|
244
|
+
.catch(() => ({ rows: sessions.map((id) => ({ id })) }))).rows.map((r) => r.id));
|
|
245
|
+
await Promise.all(sessions.filter((id) => !alive.has(id)).map((id) => endSessionTxns(id, 'The builder session ended: the transaction was rolled back.')));
|
|
246
|
+
}
|
|
247
|
+
for (const [k, n] of notes) if (now - n.at > NOTE_MS) notes.delete(k);
|
|
248
|
+
if (!open.size) stopSweeper();
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
let sweeper: NodeJS.Timeout | null = null;
|
|
252
|
+
function startSweeper() {
|
|
253
|
+
if (sweeper) return;
|
|
254
|
+
const every = Math.max(1000, Math.min(15_000, (txnIdleSeconds() * 1000) / 4));
|
|
255
|
+
sweeper = setInterval(() => void sweepTxns().catch((e) => console.error('worksheet transactions:', (e as Error).message)), every);
|
|
256
|
+
sweeper.unref();
|
|
257
|
+
}
|
|
258
|
+
function stopSweeper() {
|
|
259
|
+
if (sweeper) clearInterval(sweeper);
|
|
260
|
+
sweeper = null;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** Everything rolled back (the server stops). */
|
|
264
|
+
export async function endAllTxns(note = 'The server stopped: the transaction was rolled back.') {
|
|
265
|
+
stopSweeper();
|
|
266
|
+
await Promise.all([...open.values()].map((t) => endTxn(t, 'rollback', note)));
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
onClosePools(() => endAllTxns());
|
|
270
|
+
onSessionEnd((sessionId) => endSessionTxns(sessionId, 'You signed out: the transaction was rolled back.'));
|