turbine-orm 0.79.1 → 0.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/checkout.d.ts +53 -0
- package/dist/checkout.js +78 -0
- package/dist/cjs/checkout.d.ts +53 -0
- package/dist/cjs/checkout.js +82 -0
- package/dist/cjs/cli/index.js +4 -0
- package/dist/cjs/cli/mcp.js +4 -0
- package/dist/cjs/cli/migrate.js +6 -0
- package/dist/cjs/cli/observe.js +8 -0
- package/dist/cjs/cli/studio.js +13 -1
- package/dist/cjs/client.d.ts +12 -2
- package/dist/cjs/client.js +61 -75
- package/dist/cjs/connection-guard.d.ts +120 -0
- package/dist/cjs/connection-guard.js +191 -0
- package/dist/cjs/errors.d.ts +26 -0
- package/dist/cjs/errors.js +85 -1
- package/dist/cjs/index.d.ts +1 -1
- package/dist/cjs/nested-write.d.ts +12 -2
- package/dist/cjs/nested-write.js +4 -10
- package/dist/cjs/pipeline.js +12 -7
- package/dist/cjs/plan-flip-probe.js +4 -0
- package/dist/cjs/powdb-shared.d.ts +22 -2
- package/dist/cjs/powdb-shared.js +27 -2
- package/dist/cjs/powdb.js +36 -37
- package/dist/cjs/powql.d.ts +51 -6
- package/dist/cjs/powql.js +199 -45
- package/dist/cjs/prisma-compat.js +28 -4
- package/dist/cjs/query/builder.d.ts +44 -24
- package/dist/cjs/query/builder.js +125 -66
- package/dist/cjs/query/deferred.d.ts +9 -0
- package/dist/cjs/query/option-surface.js +12 -0
- package/dist/cjs/query/types.d.ts +68 -4
- package/dist/cjs/query/writes.d.ts +39 -9
- package/dist/cjs/query/writes.js +72 -34
- package/dist/cjs/realtime.d.ts +46 -2
- package/dist/cjs/realtime.js +125 -20
- package/dist/cjs/schema-sql.js +6 -0
- package/dist/cli/index.js +4 -0
- package/dist/cli/mcp.js +4 -0
- package/dist/cli/migrate.js +6 -0
- package/dist/cli/observe.js +8 -0
- package/dist/cli/studio.js +13 -1
- package/dist/client.d.ts +12 -2
- package/dist/client.js +62 -76
- package/dist/connection-guard.d.ts +120 -0
- package/dist/connection-guard.js +183 -0
- package/dist/errors.d.ts +26 -0
- package/dist/errors.js +83 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/nested-write.d.ts +12 -2
- package/dist/nested-write.js +4 -10
- package/dist/pipeline.js +13 -8
- package/dist/plan-flip-probe.js +4 -0
- package/dist/powdb-shared.d.ts +22 -2
- package/dist/powdb-shared.js +25 -2
- package/dist/powdb.js +23 -24
- package/dist/powql.d.ts +51 -6
- package/dist/powql.js +200 -46
- package/dist/prisma-compat.js +28 -4
- package/dist/query/builder.d.ts +44 -24
- package/dist/query/builder.js +126 -67
- package/dist/query/deferred.d.ts +9 -0
- package/dist/query/option-surface.js +12 -0
- package/dist/query/types.d.ts +68 -4
- package/dist/query/writes.d.ts +39 -9
- package/dist/query/writes.js +71 -34
- package/dist/realtime.d.ts +46 -2
- package/dist/realtime.js +125 -20
- package/dist/schema-sql.js +6 -0
- package/package.json +5 -3
package/dist/client.js
CHANGED
|
@@ -22,9 +22,11 @@
|
|
|
22
22
|
* ```
|
|
23
23
|
*/
|
|
24
24
|
import pg from '#pg';
|
|
25
|
+
import { openCheckout } from './checkout.js';
|
|
26
|
+
import { absorbCheckedOutErrors, settleLongIdleCheckouts } from './connection-guard.js';
|
|
25
27
|
import { mergeConnectionStringOptions } from './connection-url.js';
|
|
26
28
|
import { postgresDialect } from './dialect.js';
|
|
27
|
-
import { ConnectionError, errorMessageModesDiverged, PipelineError, registerClientErrorMessageMode, runWithErrorMessageMode, setErrorMessageMode, TimeoutError, UnsupportedFeatureError, ValidationError, wrapPgError, } from './errors.js';
|
|
29
|
+
import { ConnectionError, errorMessageModesDiverged, explainConnectionLoss, PipelineError, registerClientErrorMessageMode, runWithErrorMessageMode, setErrorMessageMode, TimeoutError, UnsupportedFeatureError, ValidationError, wrapPgError, } from './errors.js';
|
|
28
30
|
import { ObserveEngine } from './observe.js';
|
|
29
31
|
import { executePipeline, pipelineSupported } from './pipeline.js';
|
|
30
32
|
import { QueryInterface, } from './query/index.js';
|
|
@@ -356,29 +358,6 @@ const ISOLATION_LEVELS = Object.assign(Object.create(null), {
|
|
|
356
358
|
* loops, and a loop keyed on it would never fire for the commit-time conflicts
|
|
357
359
|
* that are the main reason to run SERIALIZABLE at all.
|
|
358
360
|
*/
|
|
359
|
-
/**
|
|
360
|
-
* Check out a pooled connection, translating a driver failure into a typed
|
|
361
|
-
* Turbine error.
|
|
362
|
-
*
|
|
363
|
-
* `pool.connect()` is where the first-run failures actually land: wrong
|
|
364
|
-
* password (SQLSTATE 28P01), no such database (3D000), nothing listening
|
|
365
|
-
* (ECONNREFUSED), an unverifiable TLS certificate. Unwrapped, every one of
|
|
366
|
-
* those left `$transaction`, `transaction()` and `connect()` as a raw pg
|
|
367
|
-
* `DatabaseError` carrying a SQLSTATE in `.code`, the same property Turbine
|
|
368
|
-
* puts `TURBINE_E0NN` in, so the single error a new user is most likely to see
|
|
369
|
-
* was the one error the typed-error contract did not cover.
|
|
370
|
-
*
|
|
371
|
-
* Query paths need no equivalent: `pool.query()` opens the connection itself
|
|
372
|
-
* and rejects with the connect error, which the query boundary already wraps.
|
|
373
|
-
*/
|
|
374
|
-
async function acquireConnection(pool) {
|
|
375
|
-
try {
|
|
376
|
-
return await pool.connect();
|
|
377
|
-
}
|
|
378
|
-
catch (err) {
|
|
379
|
-
throw wrapPgError(err);
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
361
|
async function runTxControl(client, sql) {
|
|
383
362
|
try {
|
|
384
363
|
await client.query(sql);
|
|
@@ -974,6 +953,11 @@ export class TurbineClient {
|
|
|
974
953
|
ownPool.on('error', (err) => {
|
|
975
954
|
console.error('[turbine] Unexpected pool error:', err.message);
|
|
976
955
|
});
|
|
956
|
+
absorbCheckedOutErrors(ownPool);
|
|
957
|
+
// After a freeze (a serverless function between invocations), wait one
|
|
958
|
+
// loop turn before reusing a long-idle connection, so the pool has read
|
|
959
|
+
// any close the server sent meanwhile. See connection-guard.ts.
|
|
960
|
+
settleLongIdleCheckouts(ownPool);
|
|
977
961
|
this.pool = ownPool;
|
|
978
962
|
this.ownsPool = true;
|
|
979
963
|
if (this.logging) {
|
|
@@ -997,6 +981,8 @@ export class TurbineClient {
|
|
|
997
981
|
replicaPool.on('error', (err) => {
|
|
998
982
|
console.error('[turbine] Unexpected replica pool error:', err.message);
|
|
999
983
|
});
|
|
984
|
+
absorbCheckedOutErrors(replicaPool);
|
|
985
|
+
settleLongIdleCheckouts(replicaPool);
|
|
1000
986
|
this.replicaPools.push(replicaPool);
|
|
1001
987
|
this.ownedReplicaPools.push(replicaPool);
|
|
1002
988
|
}
|
|
@@ -1700,19 +1686,15 @@ export class TurbineClient {
|
|
|
1700
1686
|
* ```
|
|
1701
1687
|
*/
|
|
1702
1688
|
async transaction(fn) {
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
*/
|
|
1712
|
-
let began = false;
|
|
1689
|
+
// BEGIN runs inside openCheckout, which sends it once more on a fresh
|
|
1690
|
+
// connection when the first one turns out to be dead. A BEGIN that fails
|
|
1691
|
+
// for good throws from there with the connection already released, so the
|
|
1692
|
+
// catch below only ever sees a transaction that began: its ROLLBACK can
|
|
1693
|
+
// never be a stray statement from a context that opened none, which on a
|
|
1694
|
+
// driver with one shared engine handle (PowDB embedded) would roll back a
|
|
1695
|
+
// DIFFERENT caller's open transaction.
|
|
1696
|
+
const { client, checkout } = await openCheckout(this.pool, (c) => runTxControl(c, this.dialect.beginStatement()));
|
|
1713
1697
|
try {
|
|
1714
|
-
await runTxControl(client, this.dialect.beginStatement());
|
|
1715
|
-
began = true;
|
|
1716
1698
|
// Engine seam: single-writer engines scope their transaction re-entrancy
|
|
1717
1699
|
// marker to the callback's async subtree (see
|
|
1718
1700
|
// PgCompatPoolClient.wrapTransactionCallback). Absent everywhere else.
|
|
@@ -1723,18 +1705,16 @@ export class TurbineClient {
|
|
|
1723
1705
|
return result;
|
|
1724
1706
|
}
|
|
1725
1707
|
catch (err) {
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
await client.query(this.dialect.rollbackStatement());
|
|
1729
|
-
}
|
|
1730
|
-
catch {
|
|
1731
|
-
// Best-effort rollback, the connection may have died mid-query.
|
|
1732
|
-
}
|
|
1708
|
+
try {
|
|
1709
|
+
await client.query(this.dialect.rollbackStatement());
|
|
1733
1710
|
}
|
|
1734
|
-
|
|
1711
|
+
catch {
|
|
1712
|
+
// Best-effort rollback, the connection may have died mid-query.
|
|
1713
|
+
}
|
|
1714
|
+
throw explainConnectionLoss(err, checkout.lostWith);
|
|
1735
1715
|
}
|
|
1736
1716
|
finally {
|
|
1737
|
-
|
|
1717
|
+
checkout.release();
|
|
1738
1718
|
}
|
|
1739
1719
|
}
|
|
1740
1720
|
async $transaction(fnOrQueries, options) {
|
|
@@ -1753,7 +1733,13 @@ export class TurbineClient {
|
|
|
1753
1733
|
// Resolve the isolation level BEFORE taking a pool slot: a bad argument is
|
|
1754
1734
|
// the caller's bug and should not cost a connection to discover.
|
|
1755
1735
|
const isolationSql = resolveIsolationLevel(options?.isolationLevel);
|
|
1756
|
-
|
|
1736
|
+
// BEGIN with optional isolation level, the dialect owns the keyword and
|
|
1737
|
+
// BEGIN+isolation composition (Postgres appends ` ISOLATION LEVEL …`). It
|
|
1738
|
+
// runs inside openCheckout, which sends it once more on a fresh connection
|
|
1739
|
+
// when the first one turns out to be dead, and which releases the
|
|
1740
|
+
// connection itself when BEGIN fails for good. So everything below runs in
|
|
1741
|
+
// a transaction that began.
|
|
1742
|
+
const { client, checkout } = await openCheckout(this.pool, (c) => runTxControl(c, this.dialect.beginStatement(isolationSql)));
|
|
1757
1743
|
const timeout = options?.timeout;
|
|
1758
1744
|
/**
|
|
1759
1745
|
* Track whether the connection has already been released so the finally
|
|
@@ -1766,27 +1752,14 @@ export class TurbineClient {
|
|
|
1766
1752
|
return;
|
|
1767
1753
|
released = true;
|
|
1768
1754
|
try {
|
|
1769
|
-
|
|
1755
|
+
checkout.release(err);
|
|
1770
1756
|
}
|
|
1771
1757
|
catch {
|
|
1772
1758
|
// pg may throw if the client is already released, swallow.
|
|
1773
1759
|
}
|
|
1774
1760
|
};
|
|
1775
1761
|
let timedOut = false;
|
|
1776
|
-
/**
|
|
1777
|
-
* Only true once BEGIN has actually succeeded. If BEGIN itself throws -
|
|
1778
|
-
* e.g. a single-writer engine's transaction gate times out in its FIFO
|
|
1779
|
-
* queue or rejects a re-entrant begin (PowDB, E002/E017), this context
|
|
1780
|
-
* never opened a transaction, so the catch below must NOT issue its
|
|
1781
|
-
* best-effort ROLLBACK: on a driver with one shared engine handle that
|
|
1782
|
-
* stray ROLLBACK would tear down a DIFFERENT caller's open transaction.
|
|
1783
|
-
*/
|
|
1784
|
-
let began = false;
|
|
1785
1762
|
try {
|
|
1786
|
-
// BEGIN with optional isolation level, the dialect owns the keyword and
|
|
1787
|
-
// BEGIN+isolation composition (Postgres appends ` ISOLATION LEVEL …`).
|
|
1788
|
-
await runTxControl(client, this.dialect.beginStatement(isolationSql));
|
|
1789
|
-
began = true;
|
|
1790
1763
|
// Apply transaction-local session context (RLS / multi-tenant GUCs).
|
|
1791
1764
|
// Order matters: BEGIN -> isolation level (above) -> set_config loop ->
|
|
1792
1765
|
// user fn. Any error here propagates to the catch below and rolls back
|
|
@@ -1867,11 +1840,9 @@ export class TurbineClient {
|
|
|
1867
1840
|
// If the timeout fired we already destroyed the connection, issuing a
|
|
1868
1841
|
// ROLLBACK on a released client would throw "Client has already been
|
|
1869
1842
|
// released". Skip the rollback in that case (the backend rolled back
|
|
1870
|
-
// when its socket was closed).
|
|
1871
|
-
//
|
|
1872
|
-
|
|
1873
|
-
// shared-handle engine.
|
|
1874
|
-
if (began && !timedOut && !released) {
|
|
1843
|
+
// when its socket was closed). A BEGIN that failed never gets here, see
|
|
1844
|
+
// openCheckout above, so there is always a transaction to roll back.
|
|
1845
|
+
if (!timedOut && !released) {
|
|
1875
1846
|
try {
|
|
1876
1847
|
await client.query(this.dialect.rollbackStatement());
|
|
1877
1848
|
}
|
|
@@ -1882,7 +1853,7 @@ export class TurbineClient {
|
|
|
1882
1853
|
if (this.logging) {
|
|
1883
1854
|
console.log('[turbine] Transaction rolled back');
|
|
1884
1855
|
}
|
|
1885
|
-
throw err;
|
|
1856
|
+
throw explainConnectionLoss(err, checkout.lostWith);
|
|
1886
1857
|
}
|
|
1887
1858
|
finally {
|
|
1888
1859
|
releaseOnce();
|
|
@@ -1993,17 +1964,27 @@ export class TurbineClient {
|
|
|
1993
1964
|
* cannot do this, `$listen` throws a `ConnectionError` rather than hang.
|
|
1994
1965
|
* `$notify` works on every driver.
|
|
1995
1966
|
*
|
|
1967
|
+
* **Connection loss:** if the subscription's connection dies (a restart,
|
|
1968
|
+
* failover, compute suspend, `pg_terminate_backend`), it reconnects with
|
|
1969
|
+
* exponential backoff and re-issues `LISTEN`; `options.reconnect: false`
|
|
1970
|
+
* ends it instead. Postgres does not hold notifications for a disconnected
|
|
1971
|
+
* listener, so anything sent during the gap is lost: use
|
|
1972
|
+
* `options.onReconnect` to resynchronise. `options.onError` receives the loss
|
|
1973
|
+
* and each failed attempt (default: one `console.error` line each).
|
|
1974
|
+
*
|
|
1996
1975
|
* @example
|
|
1997
1976
|
* ```ts
|
|
1998
1977
|
* const sub = await db.$listen('order_created', (payload) => {
|
|
1999
1978
|
* const order = JSON.parse(payload);
|
|
2000
1979
|
* console.log('new order', order.id);
|
|
1980
|
+
* }, {
|
|
1981
|
+
* onReconnect: () => refreshOrdersFromDatabase(),
|
|
2001
1982
|
* });
|
|
2002
1983
|
* // ...later
|
|
2003
1984
|
* await sub.unsubscribe();
|
|
2004
1985
|
* ```
|
|
2005
1986
|
*/
|
|
2006
|
-
async $listen(channel, handler) {
|
|
1987
|
+
async $listen(channel, handler, options) {
|
|
2007
1988
|
if (!this.dialect.supportsListenNotify) {
|
|
2008
1989
|
throw new UnsupportedFeatureError('$listen (LISTEN/NOTIFY realtime)', this.dialect.name, 'Realtime pub/sub requires PostgreSQL.');
|
|
2009
1990
|
}
|
|
@@ -2014,7 +1995,7 @@ export class TurbineClient {
|
|
|
2014
1995
|
}
|
|
2015
1996
|
const sub = await createSubscription(this.pool, channel, quoted, handler, (closed) => {
|
|
2016
1997
|
this.activeSubscriptions.delete(closed);
|
|
2017
|
-
});
|
|
1998
|
+
}, options);
|
|
2018
1999
|
this.activeSubscriptions.add(sub);
|
|
2019
2000
|
return sub;
|
|
2020
2001
|
}
|
|
@@ -2075,15 +2056,20 @@ export class TurbineClient {
|
|
|
2075
2056
|
* Throws if the connection fails.
|
|
2076
2057
|
*/
|
|
2077
2058
|
async connect() {
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2059
|
+
// A connection the server closed while idle says nothing about whether
|
|
2060
|
+
// the database is reachable, so the check runs on a fresh one when the
|
|
2061
|
+
// first turns out to be dead (see openCheckout).
|
|
2062
|
+
const { checkout } = await openCheckout(this.pool, async (client) => {
|
|
2063
|
+
try {
|
|
2064
|
+
await client.query('SELECT 1');
|
|
2083
2065
|
}
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2066
|
+
catch (err) {
|
|
2067
|
+
throw wrapPgError(err);
|
|
2068
|
+
}
|
|
2069
|
+
});
|
|
2070
|
+
checkout.release();
|
|
2071
|
+
if (this.logging) {
|
|
2072
|
+
console.log('[turbine] Connection verified');
|
|
2087
2073
|
}
|
|
2088
2074
|
}
|
|
2089
2075
|
/**
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* turbine-orm, connection error guard
|
|
3
|
+
*
|
|
4
|
+
* A pg client emits `'error'` when its socket dies: a database restart, a
|
|
5
|
+
* failover, a serverless compute suspend, `pg_terminate_backend`. pg-pool keeps
|
|
6
|
+
* a listener on every IDLE client and removes it at checkout, so for as long as
|
|
7
|
+
* a client is checked out nobody is listening unless the borrower is, and an
|
|
8
|
+
* `'error'` event with no listener is thrown by Node's EventEmitter. The
|
|
9
|
+
* process exits. Every place Turbine holds a connection across an await
|
|
10
|
+
* (`$transaction`, `transaction()`, nested writes, cursor streams, pipelines,
|
|
11
|
+
* `$listen`, Studio and MCP requests) was one database restart away from taking
|
|
12
|
+
* the application down with it.
|
|
13
|
+
*
|
|
14
|
+
* The fix is a listener for exactly the checkout window: attached right after
|
|
15
|
+
* `pool.connect()` resolves (a promise continuation, so no socket event can
|
|
16
|
+
* slip in before it), removed right after `release()`, which is where pg-pool
|
|
17
|
+
* re-attaches its own. The listener does not need to reject anything itself:
|
|
18
|
+
* pg already fails the in-flight query and every queued one, and any later
|
|
19
|
+
* query on the dead client rejects with "not queryable", so the pending call
|
|
20
|
+
* always rejects on its own. What was missing was only the listener that stops
|
|
21
|
+
* the event from being fatal. It also records the first error, for two reasons:
|
|
22
|
+
* `release()` passes it on so the pool destroys the connection instead of
|
|
23
|
+
* lending it out again, and callers can report the ORIGINAL cause (say, 57P01
|
|
24
|
+
* `terminating connection due to administrator command`) rather than the
|
|
25
|
+
* follow-on "Client has encountered a connection error and is not queryable".
|
|
26
|
+
*
|
|
27
|
+
* ZERO imports, same reason as connection-url.ts: `query/`, `cli/` and
|
|
28
|
+
* client.ts all check connections out, and a leaf is the only place all three
|
|
29
|
+
* can share without a new edge in the import graph.
|
|
30
|
+
*/
|
|
31
|
+
/** Anything checked out of a pool: a `release()` plus, on pg, the emitter surface. */
|
|
32
|
+
interface Releasable {
|
|
33
|
+
release(err?: Error | boolean): void;
|
|
34
|
+
}
|
|
35
|
+
export interface ConnectionGuard {
|
|
36
|
+
/** The first error the connection emitted while guarded, if it emitted one. */
|
|
37
|
+
readonly lostWith: Error | undefined;
|
|
38
|
+
/** Remove the listener. Idempotent. For clients Turbine owns outright (a bare `pg.Client`). */
|
|
39
|
+
detach(): void;
|
|
40
|
+
}
|
|
41
|
+
export interface CheckoutGuard extends ConnectionGuard {
|
|
42
|
+
/**
|
|
43
|
+
* Hand the client back to its pool and remove the listener. Idempotent. A
|
|
44
|
+
* connection that errored while checked out is released WITH that error, so
|
|
45
|
+
* pg-pool destroys it rather than returning it to the idle set; an explicit
|
|
46
|
+
* `err` argument takes precedence.
|
|
47
|
+
*/
|
|
48
|
+
release(err?: Error | boolean): void;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Keep every connection a pool opens from ever emitting an unheard `'error'`.
|
|
52
|
+
* For pools Turbine creates itself, where it can: pg-pool listens on IDLE
|
|
53
|
+
* clients only, so a checkout made outside Turbine's own guarded paths
|
|
54
|
+
* (`db.pool.connect()` in application code, a CLI server's request handler) is
|
|
55
|
+
* otherwise one database restart from exiting the process. The listener
|
|
56
|
+
* absorbs nothing that matters: the borrower's pending query still rejects,
|
|
57
|
+
* and pg-pool still evicts a client that failed. A pool with no `connect`
|
|
58
|
+
* event (an engine shim) is left alone.
|
|
59
|
+
*/
|
|
60
|
+
export declare function absorbCheckedOutErrors(pool: unknown): void;
|
|
61
|
+
/**
|
|
62
|
+
* Guard a connection for as long as the caller holds it. Use {@link guardCheckout}
|
|
63
|
+
* for a pooled checkout; this form is for a client whose whole lifetime the
|
|
64
|
+
* caller owns (`new pg.Client()` in a CLI command or `schemaPush`).
|
|
65
|
+
*
|
|
66
|
+
* `onLost` runs once, on the first error, for a holder that has no query in
|
|
67
|
+
* flight to learn about the loss from (a LISTEN connection waiting for
|
|
68
|
+
* notifications).
|
|
69
|
+
*/
|
|
70
|
+
export declare function guardConnection(client: unknown, onLost?: (err: Error) => void): ConnectionGuard;
|
|
71
|
+
/** Guard a pooled checkout from `pool.connect()` until its `release()`. */
|
|
72
|
+
export declare function guardCheckout(client: Releasable, onLost?: (err: Error) => void): CheckoutGuard;
|
|
73
|
+
/**
|
|
74
|
+
* Resolve after the event loop has run at least one full poll phase.
|
|
75
|
+
*
|
|
76
|
+
* Two `setImmediate` hops, not one: a continuation that starts in the poll
|
|
77
|
+
* phase would reach its first hop in the SAME iteration's check phase, before
|
|
78
|
+
* any socket event that arrived meanwhile has been read. The second hop is
|
|
79
|
+
* queued for the next iteration, which polls first. `setTimeout` stands in on
|
|
80
|
+
* runtimes without `setImmediate` (edge), where one timer turn also polls.
|
|
81
|
+
*/
|
|
82
|
+
export declare function settleEventLoop(): Promise<void>;
|
|
83
|
+
/**
|
|
84
|
+
* How long a pool must have gone without a connection coming back before the
|
|
85
|
+
* next checkout waits for {@link settleEventLoop}. Long enough that a pool in
|
|
86
|
+
* steady use never pays it; short enough to cover a serverless freeze between
|
|
87
|
+
* invocations, which is where dead idle connections come from.
|
|
88
|
+
*/
|
|
89
|
+
export declare const LONG_IDLE_SETTLE_MS = 1000;
|
|
90
|
+
/**
|
|
91
|
+
* Stop a pool from lending out a connection the server has already closed.
|
|
92
|
+
*
|
|
93
|
+
* pg-pool evicts an idle connection when its socket reports the close, which
|
|
94
|
+
* needs the event loop to read the socket. When the loop has NOT run (a
|
|
95
|
+
* serverless function frozen between invocations, a long synchronous block),
|
|
96
|
+
* a restart, failover, pooler recycle or compute suspend in that window leaves
|
|
97
|
+
* every idle connection dead but still in the idle list. The first checkout
|
|
98
|
+
* after the thaw runs before the loop polls, so it gets one, and so does every
|
|
99
|
+
* concurrent caller: measured on PostgreSQL 17 with the loop blocked across a
|
|
100
|
+
* `pg_terminate_backend` of all five idle connections, five concurrent queries
|
|
101
|
+
* all failed with 57P01. Awake, the same terminate costs nothing, because the
|
|
102
|
+
* closes are read as they arrive.
|
|
103
|
+
*
|
|
104
|
+
* So a checkout that would reuse a connection idle for at least
|
|
105
|
+
* {@link LONG_IDLE_SETTLE_MS} first waits for one poll phase. Every close that
|
|
106
|
+
* arrived during the freeze is read then, pg-pool evicts those connections,
|
|
107
|
+
* and the checkout gets a live one or opens a fresh one. No round trip: a
|
|
108
|
+
* `SELECT 1` validation would cost one on every such checkout, and the loop
|
|
109
|
+
* turn is what actually carries the information. A pool in steady use returns
|
|
110
|
+
* a connection far more often than once a second and never pays it.
|
|
111
|
+
*
|
|
112
|
+
* Idle age is taken from the last clean `'release'`: pg-pool reuses the most
|
|
113
|
+
* recently returned connection first, so that is the age of the one the next
|
|
114
|
+
* checkout gets. Wrapping `connect` covers `pool.query` too, which checks out
|
|
115
|
+
* through `this.connect`. Pools without pg-pool's `'release'` event (engine
|
|
116
|
+
* shims, HTTP pools) are left alone. For pools Turbine creates only: a
|
|
117
|
+
* caller's own pool is not Turbine's to patch.
|
|
118
|
+
*/
|
|
119
|
+
export declare function settleLongIdleCheckouts(pool: unknown, idleMs?: number): void;
|
|
120
|
+
export {};
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* turbine-orm, connection error guard
|
|
3
|
+
*
|
|
4
|
+
* A pg client emits `'error'` when its socket dies: a database restart, a
|
|
5
|
+
* failover, a serverless compute suspend, `pg_terminate_backend`. pg-pool keeps
|
|
6
|
+
* a listener on every IDLE client and removes it at checkout, so for as long as
|
|
7
|
+
* a client is checked out nobody is listening unless the borrower is, and an
|
|
8
|
+
* `'error'` event with no listener is thrown by Node's EventEmitter. The
|
|
9
|
+
* process exits. Every place Turbine holds a connection across an await
|
|
10
|
+
* (`$transaction`, `transaction()`, nested writes, cursor streams, pipelines,
|
|
11
|
+
* `$listen`, Studio and MCP requests) was one database restart away from taking
|
|
12
|
+
* the application down with it.
|
|
13
|
+
*
|
|
14
|
+
* The fix is a listener for exactly the checkout window: attached right after
|
|
15
|
+
* `pool.connect()` resolves (a promise continuation, so no socket event can
|
|
16
|
+
* slip in before it), removed right after `release()`, which is where pg-pool
|
|
17
|
+
* re-attaches its own. The listener does not need to reject anything itself:
|
|
18
|
+
* pg already fails the in-flight query and every queued one, and any later
|
|
19
|
+
* query on the dead client rejects with "not queryable", so the pending call
|
|
20
|
+
* always rejects on its own. What was missing was only the listener that stops
|
|
21
|
+
* the event from being fatal. It also records the first error, for two reasons:
|
|
22
|
+
* `release()` passes it on so the pool destroys the connection instead of
|
|
23
|
+
* lending it out again, and callers can report the ORIGINAL cause (say, 57P01
|
|
24
|
+
* `terminating connection due to administrator command`) rather than the
|
|
25
|
+
* follow-on "Client has encountered a connection error and is not queryable".
|
|
26
|
+
*
|
|
27
|
+
* ZERO imports, same reason as connection-url.ts: `query/`, `cli/` and
|
|
28
|
+
* client.ts all check connections out, and a leaf is the only place all three
|
|
29
|
+
* can share without a new edge in the import graph.
|
|
30
|
+
*/
|
|
31
|
+
/**
|
|
32
|
+
* Keep every connection a pool opens from ever emitting an unheard `'error'`.
|
|
33
|
+
* For pools Turbine creates itself, where it can: pg-pool listens on IDLE
|
|
34
|
+
* clients only, so a checkout made outside Turbine's own guarded paths
|
|
35
|
+
* (`db.pool.connect()` in application code, a CLI server's request handler) is
|
|
36
|
+
* otherwise one database restart from exiting the process. The listener
|
|
37
|
+
* absorbs nothing that matters: the borrower's pending query still rejects,
|
|
38
|
+
* and pg-pool still evicts a client that failed. A pool with no `connect`
|
|
39
|
+
* event (an engine shim) is left alone.
|
|
40
|
+
*/
|
|
41
|
+
export function absorbCheckedOutErrors(pool) {
|
|
42
|
+
const emitter = pool;
|
|
43
|
+
if (typeof emitter.on !== 'function')
|
|
44
|
+
return;
|
|
45
|
+
emitter.on('connect', (client) => {
|
|
46
|
+
client.on?.('error', () => { });
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Guard a connection for as long as the caller holds it. Use {@link guardCheckout}
|
|
51
|
+
* for a pooled checkout; this form is for a client whose whole lifetime the
|
|
52
|
+
* caller owns (`new pg.Client()` in a CLI command or `schemaPush`).
|
|
53
|
+
*
|
|
54
|
+
* `onLost` runs once, on the first error, for a holder that has no query in
|
|
55
|
+
* flight to learn about the loss from (a LISTEN connection waiting for
|
|
56
|
+
* notifications).
|
|
57
|
+
*/
|
|
58
|
+
export function guardConnection(client, onLost) {
|
|
59
|
+
const emitter = client;
|
|
60
|
+
let lostWith;
|
|
61
|
+
let attached = false;
|
|
62
|
+
const onError = (err) => {
|
|
63
|
+
if (lostWith)
|
|
64
|
+
return;
|
|
65
|
+
lostWith = err;
|
|
66
|
+
onLost?.(err);
|
|
67
|
+
};
|
|
68
|
+
if (typeof emitter.on === 'function') {
|
|
69
|
+
emitter.on('error', onError);
|
|
70
|
+
attached = true;
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
get lostWith() {
|
|
74
|
+
return lostWith;
|
|
75
|
+
},
|
|
76
|
+
detach() {
|
|
77
|
+
if (!attached)
|
|
78
|
+
return;
|
|
79
|
+
attached = false;
|
|
80
|
+
emitter.removeListener?.('error', onError);
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
/** Guard a pooled checkout from `pool.connect()` until its `release()`. */
|
|
85
|
+
export function guardCheckout(client, onLost) {
|
|
86
|
+
const guard = guardConnection(client, onLost);
|
|
87
|
+
let released = false;
|
|
88
|
+
return {
|
|
89
|
+
get lostWith() {
|
|
90
|
+
return guard.lostWith;
|
|
91
|
+
},
|
|
92
|
+
detach: guard.detach,
|
|
93
|
+
release(err) {
|
|
94
|
+
if (released)
|
|
95
|
+
return;
|
|
96
|
+
released = true;
|
|
97
|
+
try {
|
|
98
|
+
client.release(err ?? guard.lostWith);
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
// After release, not before: pg-pool re-attaches its idle listener
|
|
102
|
+
// inside release(), so detaching second leaves no window with none.
|
|
103
|
+
// A connection that already failed keeps the listener: it is being
|
|
104
|
+
// destroyed, pg emits 'error' a second time when the socket finally
|
|
105
|
+
// ends, and a pool that does not re-listen on release would let that
|
|
106
|
+
// one through. A live connection must shed it, or listeners pile up
|
|
107
|
+
// across checkouts.
|
|
108
|
+
if (!guard.lostWith)
|
|
109
|
+
guard.detach();
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Resolve after the event loop has run at least one full poll phase.
|
|
116
|
+
*
|
|
117
|
+
* Two `setImmediate` hops, not one: a continuation that starts in the poll
|
|
118
|
+
* phase would reach its first hop in the SAME iteration's check phase, before
|
|
119
|
+
* any socket event that arrived meanwhile has been read. The second hop is
|
|
120
|
+
* queued for the next iteration, which polls first. `setTimeout` stands in on
|
|
121
|
+
* runtimes without `setImmediate` (edge), where one timer turn also polls.
|
|
122
|
+
*/
|
|
123
|
+
export function settleEventLoop() {
|
|
124
|
+
const hop = typeof setImmediate === 'function' ? setImmediate : (fn) => setTimeout(fn, 0);
|
|
125
|
+
return new Promise((resolve) => {
|
|
126
|
+
hop(() => hop(resolve));
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* How long a pool must have gone without a connection coming back before the
|
|
131
|
+
* next checkout waits for {@link settleEventLoop}. Long enough that a pool in
|
|
132
|
+
* steady use never pays it; short enough to cover a serverless freeze between
|
|
133
|
+
* invocations, which is where dead idle connections come from.
|
|
134
|
+
*/
|
|
135
|
+
export const LONG_IDLE_SETTLE_MS = 1000;
|
|
136
|
+
/**
|
|
137
|
+
* Stop a pool from lending out a connection the server has already closed.
|
|
138
|
+
*
|
|
139
|
+
* pg-pool evicts an idle connection when its socket reports the close, which
|
|
140
|
+
* needs the event loop to read the socket. When the loop has NOT run (a
|
|
141
|
+
* serverless function frozen between invocations, a long synchronous block),
|
|
142
|
+
* a restart, failover, pooler recycle or compute suspend in that window leaves
|
|
143
|
+
* every idle connection dead but still in the idle list. The first checkout
|
|
144
|
+
* after the thaw runs before the loop polls, so it gets one, and so does every
|
|
145
|
+
* concurrent caller: measured on PostgreSQL 17 with the loop blocked across a
|
|
146
|
+
* `pg_terminate_backend` of all five idle connections, five concurrent queries
|
|
147
|
+
* all failed with 57P01. Awake, the same terminate costs nothing, because the
|
|
148
|
+
* closes are read as they arrive.
|
|
149
|
+
*
|
|
150
|
+
* So a checkout that would reuse a connection idle for at least
|
|
151
|
+
* {@link LONG_IDLE_SETTLE_MS} first waits for one poll phase. Every close that
|
|
152
|
+
* arrived during the freeze is read then, pg-pool evicts those connections,
|
|
153
|
+
* and the checkout gets a live one or opens a fresh one. No round trip: a
|
|
154
|
+
* `SELECT 1` validation would cost one on every such checkout, and the loop
|
|
155
|
+
* turn is what actually carries the information. A pool in steady use returns
|
|
156
|
+
* a connection far more often than once a second and never pays it.
|
|
157
|
+
*
|
|
158
|
+
* Idle age is taken from the last clean `'release'`: pg-pool reuses the most
|
|
159
|
+
* recently returned connection first, so that is the age of the one the next
|
|
160
|
+
* checkout gets. Wrapping `connect` covers `pool.query` too, which checks out
|
|
161
|
+
* through `this.connect`. Pools without pg-pool's `'release'` event (engine
|
|
162
|
+
* shims, HTTP pools) are left alone. For pools Turbine creates only: a
|
|
163
|
+
* caller's own pool is not Turbine's to patch.
|
|
164
|
+
*/
|
|
165
|
+
export function settleLongIdleCheckouts(pool, idleMs = LONG_IDLE_SETTLE_MS) {
|
|
166
|
+
const p = pool;
|
|
167
|
+
const connect = p.connect;
|
|
168
|
+
if (typeof p.on !== 'function' || typeof connect !== 'function')
|
|
169
|
+
return;
|
|
170
|
+
let lastReturnedAt = performance.now();
|
|
171
|
+
p.on('release', (err) => {
|
|
172
|
+
if (!err)
|
|
173
|
+
lastReturnedAt = performance.now();
|
|
174
|
+
});
|
|
175
|
+
p.connect = (...args) => {
|
|
176
|
+
if (!((p.idleCount ?? 0) > 0) || performance.now() - lastReturnedAt < idleMs)
|
|
177
|
+
return connect.apply(p, args);
|
|
178
|
+
const settled = settleEventLoop().then(() => connect.apply(p, args));
|
|
179
|
+
// Callback form (pool.query's internal checkout): the callback carries the
|
|
180
|
+
// result, and pg-pool returns nothing in that form either.
|
|
181
|
+
return typeof args[0] === 'function' ? undefined : settled;
|
|
182
|
+
};
|
|
183
|
+
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -556,6 +556,32 @@ export declare class ReadOnlyError extends TurbineError {
|
|
|
556
556
|
reason?: 'snapshot' | 'rbac';
|
|
557
557
|
});
|
|
558
558
|
}
|
|
559
|
+
/**
|
|
560
|
+
* Report the error that KILLED a held connection instead of its follow-on.
|
|
561
|
+
*
|
|
562
|
+
* When a connection dies while a transaction callback is between queries, the
|
|
563
|
+
* next query fails with "Client has encountered a connection error and is not
|
|
564
|
+
* queryable", which says nothing about why. The connection guard recorded the
|
|
565
|
+
* real cause (`lostWith`, e.g. 57P01 `terminating connection due to
|
|
566
|
+
* administrator command`), so a not-queryable ConnectionError is swapped for
|
|
567
|
+
* that one. Anything else passes through: an error the caller threw, or a
|
|
568
|
+
* query that was IN FLIGHT when the connection died, which already carries the
|
|
569
|
+
* server's own message.
|
|
570
|
+
*/
|
|
571
|
+
export declare function explainConnectionLoss(err: unknown, lostWith: Error | undefined): unknown;
|
|
572
|
+
/**
|
|
573
|
+
* Whether `err` says a connection that was already open is gone, so the same
|
|
574
|
+
* statement on a fresh connection can succeed. Accepts the raw driver error or
|
|
575
|
+
* the {@link ConnectionError} wrapPgError made of it.
|
|
576
|
+
*
|
|
577
|
+
* Narrower than "is a ConnectionError" on purpose. A connection that could not
|
|
578
|
+
* be OPENED (refused, DNS, auth, a connect timeout, 57P03 while the server
|
|
579
|
+
* starts) will not open on an immediate second try either, and retrying it
|
|
580
|
+
* only doubles the wait before the caller hears about it. Nor does it cover
|
|
581
|
+
* `Connection terminated` without "unexpectedly", which is the pool itself
|
|
582
|
+
* shutting down.
|
|
583
|
+
*/
|
|
584
|
+
export declare function isStaleConnectionError(err: unknown): boolean;
|
|
559
585
|
/**
|
|
560
586
|
* Translate a pg driver error into a typed Turbine error.
|
|
561
587
|
*
|