vouchington-tooling 0.62.0 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -0
- package/dist/postgres-test-faults/fault-support.d.mts +11 -0
- package/dist/postgres-test-faults/fault-support.mjs +61 -0
- package/dist/postgres-test-faults/fault-types.d.mts +43 -0
- package/dist/postgres-test-faults/fault-types.mjs +1 -0
- package/dist/postgres-test-faults/index.d.mts +5 -0
- package/dist/postgres-test-faults/index.mjs +4 -0
- package/dist/postgres-test-faults/lock-observers.d.mts +20 -0
- package/dist/postgres-test-faults/lock-observers.mjs +14 -0
- package/dist/postgres-test-faults/lock-wait.d.mts +13 -0
- package/dist/postgres-test-faults/lock-wait.mjs +62 -0
- package/dist/postgres-test-faults/lock-waiter.d.mts +19 -0
- package/dist/postgres-test-faults/lock-waiter.mjs +95 -0
- package/dist/postgres-test-faults/observation-queue.d.mts +2 -0
- package/dist/postgres-test-faults/observation-queue.mjs +22 -0
- package/dist/postgres-test-faults/polling.d.mts +6 -0
- package/dist/postgres-test-faults/polling.mjs +33 -0
- package/dist/postgres-test-faults/pool-query-failure.d.mts +8 -0
- package/dist/postgres-test-faults/pool-query-failure.mjs +136 -0
- package/dist/postgres-test-faults/query-failure.d.mts +8 -0
- package/dist/postgres-test-faults/query-failure.mjs +108 -0
- package/dist/postgres-test-faults/test-cluster.test-helpers.d.mts +13 -0
- package/dist/postgres-test-faults/test-cluster.test-helpers.mjs +84 -0
- package/dist/provider-replay/abort-listeners.d.mts +4 -0
- package/dist/provider-replay/abort-listeners.mjs +19 -0
- package/dist/provider-replay/aws-replay-abort-signal.d.mts +5 -0
- package/dist/provider-replay/aws-replay-abort-signal.mjs +63 -0
- package/dist/provider-replay/aws-replay-request-handler.d.mts +38 -0
- package/dist/provider-replay/aws-replay-request-handler.mjs +66 -0
- package/dist/provider-replay/force-client-option.d.mts +6 -0
- package/dist/provider-replay/force-client-option.mjs +13 -0
- package/dist/provider-replay/index.d.mts +6 -0
- package/dist/provider-replay/index.mjs +6 -0
- package/dist/provider-replay/provider-replay.d.mts +32 -0
- package/dist/provider-replay/provider-replay.mjs +54 -0
- package/dist/provider-replay/recorded-response.d.mts +17 -0
- package/dist/provider-replay/recorded-response.mjs +42 -0
- package/dist/provider-replay/replay-response.d.mts +8 -0
- package/dist/provider-replay/replay-response.mjs +38 -0
- package/package.json +17 -1
- package/scripts/__pycache__/run_bounded_process.cpython-312.pyc +0 -0
package/README.md
CHANGED
|
@@ -385,6 +385,15 @@ import {
|
|
|
385
385
|
import { createArtifactClassifier, runCleanup } from 'vouchington-tooling/gha-artifacts-cleanup'
|
|
386
386
|
import { ensureLabels, labelsFromPaths } from 'vouchington-tooling/agent-issue-labels'
|
|
387
387
|
import { validateOptionalHttpOrigin } from 'vouchington-tooling/http-origin'
|
|
388
|
+
// Replay recorded raw `.http` responses through an SDK `fetch` or AWS SDK v3 `requestHandler`:
|
|
389
|
+
// const replay = createProviderReplay({ chunkBytes: 3 })
|
|
390
|
+
// replay.respondWith(loadRecordedResponse('ok.http', { fixtureRoot: new URL('./fixtures/', import.meta.url) }))
|
|
391
|
+
// new Sdk({ fetch: replay.fetch }) or new AwsClient({ requestHandler: createAwsReplayRequestHandler(replay) })
|
|
392
|
+
import {
|
|
393
|
+
createAwsReplayRequestHandler,
|
|
394
|
+
createProviderReplay,
|
|
395
|
+
loadRecordedResponse,
|
|
396
|
+
} from 'vouchington-tooling/provider-replay'
|
|
388
397
|
import { boundPendingLine, splitCompleteLines } from 'vouchington-tooling/process-line-buffer'
|
|
389
398
|
import {
|
|
390
399
|
isProcessGroupAlive,
|
|
@@ -1075,3 +1084,53 @@ from `TMUX_TARGET_BIN`, the worktree from `TMUX_TARGET_ROOT` (default: current d
|
|
|
1075
1084
|
touching it. `tmux post-tool` reads an agent post-tool hook payload on stdin and prints a rename
|
|
1076
1085
|
reminder at plan, PR, push and reset checkpoints; reminders name `TMUX_NAME_COMMAND` (default
|
|
1077
1086
|
`vouchington tmux name`).
|
|
1087
|
+
|
|
1088
|
+
## PostgreSQL test faults and lock waits
|
|
1089
|
+
|
|
1090
|
+
`vouchington-tooling/postgres-test-faults` injects real PostgreSQL failures and observes real lock
|
|
1091
|
+
waits for any `pg.Pool` application. It needs the optional peer `pg` (>=8) and `@types/pg`; no
|
|
1092
|
+
product code is imported, so pools and request ownership are passed in.
|
|
1093
|
+
|
|
1094
|
+
```ts
|
|
1095
|
+
import {
|
|
1096
|
+
createPostgresLockObservers,
|
|
1097
|
+
withPostgresPoolQueryFailureForTest,
|
|
1098
|
+
withPostgresQueryFailureForTest,
|
|
1099
|
+
} from 'vouchington-tooling/postgres-test-faults'
|
|
1100
|
+
|
|
1101
|
+
const { result, error } = await withPostgresQueryFailureForTest(
|
|
1102
|
+
'/* orders:insert */', // exact leading block comment that marks the statement to fail
|
|
1103
|
+
() => placeOrder().catch((err: unknown) => err), // absorb the injected error
|
|
1104
|
+
{ pools: [readPool, writePool], command: 'INSERT' },
|
|
1105
|
+
)
|
|
1106
|
+
// error.code === '25P02': PostgreSQL rejected the marked statement on an aborted transaction.
|
|
1107
|
+
```
|
|
1108
|
+
|
|
1109
|
+
- `withPostgresQueryFailureForTest(marker, operation, options)` fails one marked statement on a
|
|
1110
|
+
client leased from `options.pools` with the promise form of `pool.connect()` (the callback
|
|
1111
|
+
overload is not instrumented), inside a transaction the operation began with `BEGIN`. A
|
|
1112
|
+
division by zero aborts that transaction, so PostgreSQL itself answers `25P02`.
|
|
1113
|
+
- `withPostgresPoolQueryFailureForTest(marker, operation, options)` does the same for a marked
|
|
1114
|
+
`pool.query({ text })` call: it leases a real client, aborts its transaction, and hands the client
|
|
1115
|
+
to the unchanged `pg.Pool#query` core.
|
|
1116
|
+
- Both resolve to `{ result, error }` only when the operation absorbs the injected `25P02` (catch
|
|
1117
|
+
or translate it, and return the outcome as `result`). If the operation lets the error
|
|
1118
|
+
propagate, the helper rejects with that error; it also rejects when the marked SQL never ran.
|
|
1119
|
+
Observers give up after `timeoutMs` even when a query never settles; `signal` is checked between
|
|
1120
|
+
steps, not inside a pending query. `options` is
|
|
1121
|
+
`{ pools, command?, requestId?, isOwnedRequest?, endTimeoutMs? }`. A statement is owned by the
|
|
1122
|
+
operation's async context, or by `requestId` when `isOwnedRequest(requestId)` returns true
|
|
1123
|
+
(use it for work that runs outside that context, such as a request handler). Scopes are
|
|
1124
|
+
serialized and cannot be nested.
|
|
1125
|
+
- `createPostgresLockObservers({ write, advisoryLockPool?, timeoutMs?, intervalMs? })` binds the
|
|
1126
|
+
observers to your executors (any `({ text, values }) => Promise<{ rows }>`, for example
|
|
1127
|
+
`query => pool.query(query)`), with defaults of 5000 ms and 10 ms. It returns
|
|
1128
|
+
`getTestPostgresBackendProcessId(executor)`, `getTestPostgresAdvisoryLockHolderProcessId({ key,
|
|
1129
|
+
mode?, namespace? })`, `waitForTestPostgresLockWaiter(holderPid, marker, observer?, onDiagnostic?,
|
|
1130
|
+
signal?, waiterPid?)` and `waitForTestPostgresBlockedLockChain(holderPid, directMarker,
|
|
1131
|
+
blockedMarkers, observer?)`. The waiters poll `pg_stat_activity` and `pg_blocking_pids`; the
|
|
1132
|
+
observer is `'write'`, `'advisoryLock'` or a reserved executor.
|
|
1133
|
+
|
|
1134
|
+
The package's own tests start a throwaway cluster with `initdb`/`pg_ctl` found through
|
|
1135
|
+
`POSTGRES_BIN_DIR`, `PATH` or a standard install location. They skip where no server can run and
|
|
1136
|
+
fail under `CI`.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { PostgresFaultOptions } from './fault-types.mts';
|
|
2
|
+
export declare function assertFaultOptions(queryMarker: string, options: PostgresFaultOptions): void;
|
|
3
|
+
export declare function queryText(input: unknown): string;
|
|
4
|
+
/** True when `text` starts with the marker and, if given, the command that follows it. */
|
|
5
|
+
export declare function targetsStatement(text: string, queryMarker: string, command?: string): boolean;
|
|
6
|
+
/** The faulted call must be the adapter's plain `{ text }` config, never a callback or submittable. */
|
|
7
|
+
export declare function isPlainTextConfig(args: unknown[]): boolean;
|
|
8
|
+
export declare function ownsRequest(options: PostgresFaultOptions): boolean;
|
|
9
|
+
export declare function installProperty(object: object, key: string, value: unknown): void;
|
|
10
|
+
export declare function restoreProperty(object: object, key: string, descriptor: PropertyDescriptor | undefined): void;
|
|
11
|
+
export declare function hasErrorCode(err: unknown, code: string): err is Error;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// A marker is exactly one leading SQL block comment that holds a name.
|
|
2
|
+
const MARKER_PATTERN = /^\/\* [^*]+ \*\/$/;
|
|
3
|
+
export function assertFaultOptions(queryMarker, options) {
|
|
4
|
+
if (!MARKER_PATTERN.test(queryMarker))
|
|
5
|
+
throw new Error('PostgreSQL fault injection requires an exact leading query annotation');
|
|
6
|
+
if (options.pools.length === 0)
|
|
7
|
+
throw new Error('PostgreSQL fault injection requires at least one pool');
|
|
8
|
+
if (options.endTimeoutMs !== undefined &&
|
|
9
|
+
!(Number.isInteger(options.endTimeoutMs) && options.endTimeoutMs >= 0))
|
|
10
|
+
throw new Error('endTimeoutMs must be a non-negative integer');
|
|
11
|
+
if (options.requestId === undefined)
|
|
12
|
+
return;
|
|
13
|
+
if (options.requestId === '' || options.isOwnedRequest === undefined)
|
|
14
|
+
throw new Error('Request-scoped PostgreSQL faults require a request id and isOwnedRequest');
|
|
15
|
+
}
|
|
16
|
+
export function queryText(input) {
|
|
17
|
+
return typeof input === 'string'
|
|
18
|
+
? input
|
|
19
|
+
: typeof input === 'object' && input !== null && 'text' in input
|
|
20
|
+
? String(input.text)
|
|
21
|
+
: '';
|
|
22
|
+
}
|
|
23
|
+
/** True when `text` starts with the marker and, if given, the command that follows it. */
|
|
24
|
+
export function targetsStatement(text, queryMarker, command) {
|
|
25
|
+
const target = text.trimStart();
|
|
26
|
+
if (!target.startsWith(queryMarker))
|
|
27
|
+
return false;
|
|
28
|
+
return !command || target.slice(queryMarker.length).trimStart().toUpperCase().startsWith(command);
|
|
29
|
+
}
|
|
30
|
+
/** The faulted call must be the adapter's plain `{ text }` config, never a callback or submittable. */
|
|
31
|
+
export function isPlainTextConfig(args) {
|
|
32
|
+
const input = args[0];
|
|
33
|
+
return (args.length === 1 &&
|
|
34
|
+
typeof input === 'object' &&
|
|
35
|
+
input !== null &&
|
|
36
|
+
'text' in input &&
|
|
37
|
+
typeof input.text === 'string' &&
|
|
38
|
+
!('callback' in input) &&
|
|
39
|
+
!('submit' in input) &&
|
|
40
|
+
Object.getPrototypeOf(input) === Object.prototype);
|
|
41
|
+
}
|
|
42
|
+
export function ownsRequest(options) {
|
|
43
|
+
return options.requestId !== undefined && options.isOwnedRequest?.(options.requestId) === true;
|
|
44
|
+
}
|
|
45
|
+
export function installProperty(object, key, value) {
|
|
46
|
+
Object.defineProperty(object, key, {
|
|
47
|
+
configurable: true,
|
|
48
|
+
enumerable: true,
|
|
49
|
+
writable: true,
|
|
50
|
+
value,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
export function restoreProperty(object, key, descriptor) {
|
|
54
|
+
if (descriptor)
|
|
55
|
+
Object.defineProperty(object, key, descriptor);
|
|
56
|
+
else
|
|
57
|
+
Reflect.deleteProperty(object, key);
|
|
58
|
+
}
|
|
59
|
+
export function hasErrorCode(err, code) {
|
|
60
|
+
return err instanceof Error && 'code' in err && err.code === code;
|
|
61
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type pg from 'pg';
|
|
2
|
+
/** Minimal structural query executor: a `pg.Pool#query`, a pool client, or a wrapper around one. */
|
|
3
|
+
export type QueryExecutor = <Row extends object = Record<string, unknown>>(query: {
|
|
4
|
+
text: string;
|
|
5
|
+
values?: unknown[];
|
|
6
|
+
}) => Promise<{
|
|
7
|
+
rows: Row[];
|
|
8
|
+
}>;
|
|
9
|
+
/** Statement kinds a fault can target after the leading marker annotation. */
|
|
10
|
+
export type FaultCommand = 'SELECT' | 'INSERT' | 'UPDATE' | 'DELETE';
|
|
11
|
+
export type PostgresFaultOptions = {
|
|
12
|
+
/** Pools whose connections are instrumented while the operation runs. */
|
|
13
|
+
pools: readonly pg.Pool[];
|
|
14
|
+
/** Only fail a statement whose SQL, after the marker, starts with this command. */
|
|
15
|
+
command?: FaultCommand;
|
|
16
|
+
/** Own faults raised outside the async context of the operation, for example in a request handler. */
|
|
17
|
+
requestId?: string;
|
|
18
|
+
/** Decides whether the current execution context belongs to `requestId`. Required with `requestId`. */
|
|
19
|
+
isOwnedRequest?: (requestId: string) => boolean;
|
|
20
|
+
/** Milliseconds to wait for a failed pool client to finish closing. Defaults to 5000. */
|
|
21
|
+
endTimeoutMs?: number;
|
|
22
|
+
};
|
|
23
|
+
export type PostgresFaultResult<Result> = {
|
|
24
|
+
result: Result;
|
|
25
|
+
error: Error;
|
|
26
|
+
};
|
|
27
|
+
export type PollOptions = {
|
|
28
|
+
/** Milliseconds before an observer gives up. Defaults to 5000. */
|
|
29
|
+
timeoutMs?: number;
|
|
30
|
+
/** Milliseconds between observations. Defaults to 10. */
|
|
31
|
+
intervalMs?: number;
|
|
32
|
+
};
|
|
33
|
+
export type ObserverPool = 'write' | 'advisoryLock' | QueryExecutor;
|
|
34
|
+
export type LockObserverConfig = PollOptions & {
|
|
35
|
+
/** Executor for the default `'write'` observer and for advisory-lock holder lookups. */
|
|
36
|
+
write: QueryExecutor;
|
|
37
|
+
/** Executor for the `'advisoryLock'` observer, usually a pool reserved for observation. */
|
|
38
|
+
advisoryLockPool?: QueryExecutor;
|
|
39
|
+
};
|
|
40
|
+
export type LockWaiterDiagnostic = {
|
|
41
|
+
phase: 'snapshot-clear-start' | 'snapshot-cleared' | 'waiter-query-start' | 'waiter-query';
|
|
42
|
+
waiting: boolean;
|
|
43
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { withPostgresPoolQueryFailureForTest } from './pool-query-failure.mts';
|
|
2
|
+
export { withPostgresQueryFailureForTest } from './query-failure.mts';
|
|
3
|
+
export { createPostgresLockObservers, type PostgresLockObservers } from './lock-observers.mts';
|
|
4
|
+
export { getTestPostgresBackendProcessId } from './lock-wait.mts';
|
|
5
|
+
export type { FaultCommand, LockObserverConfig, LockWaiterDiagnostic, ObserverPool, PollOptions, PostgresFaultOptions, PostgresFaultResult, QueryExecutor, } from './fault-types.mts';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { withPostgresPoolQueryFailureForTest } from "./pool-query-failure.mjs";
|
|
2
|
+
export { withPostgresQueryFailureForTest } from "./query-failure.mjs";
|
|
3
|
+
export { createPostgresLockObservers } from "./lock-observers.mjs";
|
|
4
|
+
export { getTestPostgresBackendProcessId } from "./lock-wait.mjs";
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { LockObserverConfig, QueryExecutor } from './fault-types.mts';
|
|
2
|
+
import { waitForBlockedLockChain, waitForLockWaiter } from './lock-waiter.mts';
|
|
3
|
+
type ConfiguredWaiter = (...args: Parameters<typeof waitForLockWaiter> extends [unknown, ...infer Rest] ? Rest : never) => Promise<void>;
|
|
4
|
+
type ConfiguredChain = (...args: Parameters<typeof waitForBlockedLockChain> extends [unknown, ...infer Rest] ? Rest : never) => Promise<void>;
|
|
5
|
+
export type PostgresLockObservers = {
|
|
6
|
+
getTestPostgresBackendProcessId: (query: QueryExecutor) => Promise<number>;
|
|
7
|
+
getTestPostgresAdvisoryLockHolderProcessId: (input: {
|
|
8
|
+
key: string;
|
|
9
|
+
mode?: 'exclusive' | 'shared';
|
|
10
|
+
namespace?: number;
|
|
11
|
+
}) => Promise<number>;
|
|
12
|
+
waitForTestPostgresLockWaiter: ConfiguredWaiter;
|
|
13
|
+
waitForTestPostgresBlockedLockChain: ConfiguredChain;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Binds the lock-wait observers to the executors and timings of one application. Call once in a
|
|
17
|
+
* test setup module and re-export the functions.
|
|
18
|
+
*/
|
|
19
|
+
export declare function createPostgresLockObservers(config: LockObserverConfig): PostgresLockObservers;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { getAdvisoryLockHolderProcessId, getTestPostgresBackendProcessId } from "./lock-wait.mjs";
|
|
2
|
+
import { waitForBlockedLockChain, waitForLockWaiter } from "./lock-waiter.mjs";
|
|
3
|
+
/**
|
|
4
|
+
* Binds the lock-wait observers to the executors and timings of one application. Call once in a
|
|
5
|
+
* test setup module and re-export the functions.
|
|
6
|
+
*/
|
|
7
|
+
export function createPostgresLockObservers(config) {
|
|
8
|
+
return {
|
|
9
|
+
getTestPostgresBackendProcessId,
|
|
10
|
+
getTestPostgresAdvisoryLockHolderProcessId: (input) => getAdvisoryLockHolderProcessId(config, input),
|
|
11
|
+
waitForTestPostgresLockWaiter: (...args) => waitForLockWaiter(config, ...args),
|
|
12
|
+
waitForTestPostgresBlockedLockChain: (...args) => waitForBlockedLockChain(config, ...args),
|
|
13
|
+
};
|
|
14
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { LockObserverConfig, ObserverPool, QueryExecutor } from './fault-types.mts';
|
|
2
|
+
/** Returns the PostgreSQL backend PID serving `query`'s current connection. */
|
|
3
|
+
export declare function getTestPostgresBackendProcessId(query: QueryExecutor): Promise<number>;
|
|
4
|
+
/** Runs `text` on the executor chosen by `observerPool`. */
|
|
5
|
+
export declare function runObservation<Row extends object>(config: LockObserverConfig, observerPool: ObserverPool, text: string, values: unknown[]): Promise<{
|
|
6
|
+
rows: Row[];
|
|
7
|
+
}>;
|
|
8
|
+
/** Returns the backend PID holding an exact session advisory lock owned by this test. */
|
|
9
|
+
export declare function getAdvisoryLockHolderProcessId(config: LockObserverConfig, input: {
|
|
10
|
+
key: string;
|
|
11
|
+
mode?: 'exclusive' | 'shared';
|
|
12
|
+
namespace?: number;
|
|
13
|
+
}): Promise<number>;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/** Returns the PostgreSQL backend PID serving `query`'s current connection. */
|
|
2
|
+
export async function getTestPostgresBackendProcessId(query) {
|
|
3
|
+
const { rows } = await query({
|
|
4
|
+
text: `
|
|
5
|
+
/* getTestPostgresBackendProcessId */
|
|
6
|
+
SELECT pg_backend_pid()::integer AS process_id
|
|
7
|
+
`,
|
|
8
|
+
});
|
|
9
|
+
const processId = rows[0]?.process_id;
|
|
10
|
+
if (processId === undefined)
|
|
11
|
+
throw new Error('PostgreSQL did not return a backend process ID');
|
|
12
|
+
return processId;
|
|
13
|
+
}
|
|
14
|
+
/** Runs `text` on the executor chosen by `observerPool`. */
|
|
15
|
+
export function runObservation(config, observerPool, text, values) {
|
|
16
|
+
if (typeof observerPool === 'function')
|
|
17
|
+
return observerPool({ text, values });
|
|
18
|
+
if (observerPool === 'write')
|
|
19
|
+
return config.write({ text, values });
|
|
20
|
+
if (!config.advisoryLockPool)
|
|
21
|
+
throw new Error("The 'advisoryLock' observer requires an advisoryLockPool executor");
|
|
22
|
+
return config.advisoryLockPool({ text, values });
|
|
23
|
+
}
|
|
24
|
+
/** Returns the backend PID holding an exact session advisory lock owned by this test. */
|
|
25
|
+
export async function getAdvisoryLockHolderProcessId(config, input) {
|
|
26
|
+
const lockMode = input.mode === 'shared' ? 'ShareLock' : 'ExclusiveLock';
|
|
27
|
+
const result = input.namespace === undefined
|
|
28
|
+
? await config.write({
|
|
29
|
+
text: `
|
|
30
|
+
/* getTestPostgresAdvisoryLockHolderProcessId:oneInt */
|
|
31
|
+
SELECT pid::integer AS process_id
|
|
32
|
+
FROM pg_locks
|
|
33
|
+
WHERE locktype = 'advisory'
|
|
34
|
+
AND database = (SELECT oid FROM pg_database WHERE datname = current_database())
|
|
35
|
+
AND mode = $1
|
|
36
|
+
AND granted
|
|
37
|
+
AND objsubid = 1
|
|
38
|
+
AND classid::bigint = ((hashtextextended($2::text, 0) >> 32) & 4294967295)
|
|
39
|
+
AND objid::bigint = (hashtextextended($2::text, 0) & 4294967295)
|
|
40
|
+
`,
|
|
41
|
+
values: [lockMode, input.key],
|
|
42
|
+
})
|
|
43
|
+
: await config.write({
|
|
44
|
+
text: `
|
|
45
|
+
/* getTestPostgresAdvisoryLockHolderProcessId:twoInt */
|
|
46
|
+
SELECT pid::integer AS process_id
|
|
47
|
+
FROM pg_locks
|
|
48
|
+
WHERE locktype = 'advisory'
|
|
49
|
+
AND database = (SELECT oid FROM pg_database WHERE datname = current_database())
|
|
50
|
+
AND mode = $1
|
|
51
|
+
AND granted
|
|
52
|
+
AND objsubid = 2
|
|
53
|
+
AND classid = $2::integer::oid
|
|
54
|
+
AND objid = hashtext($3::text)::oid
|
|
55
|
+
`,
|
|
56
|
+
values: [lockMode, input.namespace, input.key],
|
|
57
|
+
});
|
|
58
|
+
const processId = result.rows[0]?.process_id;
|
|
59
|
+
if (processId === undefined)
|
|
60
|
+
throw new Error('PostgreSQL did not report the test advisory-lock holder');
|
|
61
|
+
return processId;
|
|
62
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { LockObserverConfig, LockWaiterDiagnostic, ObserverPool } from './fault-types.mts';
|
|
2
|
+
/**
|
|
3
|
+
* Waits until a query bearing `queryMarker` is blocked by this test's lock-holding backend.
|
|
4
|
+
* Use a dedicated advisory executor when the held lock and blocked operations can occupy every
|
|
5
|
+
* write connection; the observer must run before the holder is released. Passing a previously
|
|
6
|
+
* acquired transaction executor reserves the observer connection before the holder or blocked
|
|
7
|
+
* operation starts, and its transaction-local activity snapshot is discarded before each
|
|
8
|
+
* observation. Supply `waiterProcessId` to distinguish this operation from other matching waiters.
|
|
9
|
+
*/
|
|
10
|
+
export declare function waitForLockWaiter(config: LockObserverConfig, holderProcessId: number, queryMarker: string, observerPool?: ObserverPool, onDiagnostic?: (diagnostic: LockWaiterDiagnostic) => void, signal?: AbortSignal, waiterProcessId?: number): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Waits until one backend is blocked by `holderProcessId` on `directQueryMarker` and a second
|
|
13
|
+
* backend is blocked by that first backend on one of `blockedByDirectQueryMarkers`.
|
|
14
|
+
*
|
|
15
|
+
* Use this when the second operation cannot reach the holder's lock because the first operation
|
|
16
|
+
* already holds a conflicting row lock. `pg_blocking_pids` reports only the direct blocker, so
|
|
17
|
+
* the second wait is the transitive link.
|
|
18
|
+
*/
|
|
19
|
+
export declare function waitForBlockedLockChain(config: LockObserverConfig, holderProcessId: number, directQueryMarker: string, blockedByDirectQueryMarkers: readonly string[], observerPool?: ObserverPool): Promise<void>;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { runObservation } from "./lock-wait.mjs";
|
|
2
|
+
import { pollUntilNotNull } from "./polling.mjs";
|
|
3
|
+
/** A LIKE pattern matching `text` literally anywhere, so `%`, `_` and `\` in markers are not wildcards. */
|
|
4
|
+
function likeContains(text) {
|
|
5
|
+
return `%${text.replace(/[\\%_]/g, '\\$&')}%`;
|
|
6
|
+
}
|
|
7
|
+
const CLEAR_SNAPSHOT = 'SELECT pg_stat_clear_snapshot()';
|
|
8
|
+
const WAITER_SQL = `
|
|
9
|
+
/* waitForTestPostgresLockWaiter */
|
|
10
|
+
SELECT EXISTS (
|
|
11
|
+
SELECT 1
|
|
12
|
+
FROM pg_stat_activity activity
|
|
13
|
+
WHERE activity.pid <> pg_backend_pid()
|
|
14
|
+
AND ($1::integer IS NULL OR activity.pid = $1::integer)
|
|
15
|
+
AND activity.state = 'active'
|
|
16
|
+
AND activity.wait_event_type = 'Lock'
|
|
17
|
+
AND activity.query LIKE $2::text
|
|
18
|
+
AND $3::integer = ANY(pg_blocking_pids(activity.pid))
|
|
19
|
+
) AS waiting
|
|
20
|
+
`;
|
|
21
|
+
const CHAIN_SQL = `
|
|
22
|
+
/* waitForTestPostgresBlockedLockChain */
|
|
23
|
+
SELECT EXISTS (
|
|
24
|
+
SELECT 1
|
|
25
|
+
FROM pg_stat_activity direct_waiter
|
|
26
|
+
JOIN pg_stat_activity indirect_waiter
|
|
27
|
+
ON direct_waiter.pid = ANY(pg_blocking_pids(indirect_waiter.pid))
|
|
28
|
+
WHERE direct_waiter.pid <> pg_backend_pid()
|
|
29
|
+
AND indirect_waiter.pid <> pg_backend_pid()
|
|
30
|
+
AND direct_waiter.state = 'active'
|
|
31
|
+
AND indirect_waiter.state = 'active'
|
|
32
|
+
AND direct_waiter.wait_event_type = 'Lock'
|
|
33
|
+
AND indirect_waiter.wait_event_type = 'Lock'
|
|
34
|
+
AND direct_waiter.query LIKE $1::text
|
|
35
|
+
AND $2::integer = ANY(pg_blocking_pids(direct_waiter.pid))
|
|
36
|
+
AND indirect_waiter.query LIKE ANY($3::text[])
|
|
37
|
+
) AS waiting
|
|
38
|
+
`;
|
|
39
|
+
/**
|
|
40
|
+
* Waits until a query bearing `queryMarker` is blocked by this test's lock-holding backend.
|
|
41
|
+
* Use a dedicated advisory executor when the held lock and blocked operations can occupy every
|
|
42
|
+
* write connection; the observer must run before the holder is released. Passing a previously
|
|
43
|
+
* acquired transaction executor reserves the observer connection before the holder or blocked
|
|
44
|
+
* operation starts, and its transaction-local activity snapshot is discarded before each
|
|
45
|
+
* observation. Supply `waiterProcessId` to distinguish this operation from other matching waiters.
|
|
46
|
+
*/
|
|
47
|
+
export async function waitForLockWaiter(config, holderProcessId, queryMarker, observerPool = 'write', onDiagnostic, signal, waiterProcessId) {
|
|
48
|
+
await pollUntilNotNull(async () => {
|
|
49
|
+
signal?.throwIfAborted();
|
|
50
|
+
if (typeof observerPool === 'function') {
|
|
51
|
+
onDiagnostic?.({ phase: 'snapshot-clear-start', waiting: false });
|
|
52
|
+
signal?.throwIfAborted();
|
|
53
|
+
await observerPool({
|
|
54
|
+
text: `/* waitForTestPostgresLockWaiter.refresh */ ${CLEAR_SNAPSHOT}`,
|
|
55
|
+
});
|
|
56
|
+
signal?.throwIfAborted();
|
|
57
|
+
onDiagnostic?.({ phase: 'snapshot-cleared', waiting: false });
|
|
58
|
+
}
|
|
59
|
+
onDiagnostic?.({ phase: 'waiter-query-start', waiting: false });
|
|
60
|
+
signal?.throwIfAborted();
|
|
61
|
+
const { rows } = await runObservation(config, observerPool, WAITER_SQL, [waiterProcessId ?? null, likeContains(queryMarker), holderProcessId]);
|
|
62
|
+
signal?.throwIfAborted();
|
|
63
|
+
onDiagnostic?.({ phase: 'waiter-query', waiting: rows[0]?.waiting ?? false });
|
|
64
|
+
return rows[0]?.waiting ? true : null;
|
|
65
|
+
}, config.timeoutMs ?? 5_000, config.intervalMs ?? 10).catch((err) => {
|
|
66
|
+
const cause = signal?.aborted && err !== signal.reason
|
|
67
|
+
? new SuppressedError(err, signal.reason, 'Lock observation failed during cancellation')
|
|
68
|
+
: err;
|
|
69
|
+
throw new Error(`Operation did not wait for the test lock: ${queryMarker}`, { cause });
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Waits until one backend is blocked by `holderProcessId` on `directQueryMarker` and a second
|
|
74
|
+
* backend is blocked by that first backend on one of `blockedByDirectQueryMarkers`.
|
|
75
|
+
*
|
|
76
|
+
* Use this when the second operation cannot reach the holder's lock because the first operation
|
|
77
|
+
* already holds a conflicting row lock. `pg_blocking_pids` reports only the direct blocker, so
|
|
78
|
+
* the second wait is the transitive link.
|
|
79
|
+
*/
|
|
80
|
+
export async function waitForBlockedLockChain(config, holderProcessId, directQueryMarker, blockedByDirectQueryMarkers, observerPool = 'advisoryLock') {
|
|
81
|
+
await pollUntilNotNull(async () => {
|
|
82
|
+
if (typeof observerPool === 'function')
|
|
83
|
+
await observerPool({
|
|
84
|
+
text: `/* waitForTestPostgresBlockedLockChain.refresh */ ${CLEAR_SNAPSHOT}`,
|
|
85
|
+
});
|
|
86
|
+
const { rows } = await runObservation(config, observerPool, CHAIN_SQL, [
|
|
87
|
+
likeContains(directQueryMarker),
|
|
88
|
+
holderProcessId,
|
|
89
|
+
blockedByDirectQueryMarkers.map(likeContains),
|
|
90
|
+
]);
|
|
91
|
+
return rows[0]?.waiting ? true : null;
|
|
92
|
+
}, config.timeoutMs ?? 5_000, config.intervalMs ?? 10).catch((err) => {
|
|
93
|
+
throw new Error(`Operations did not form a lock chain behind the test holder: ${directQueryMarker}`, { cause: err });
|
|
94
|
+
});
|
|
95
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
const observationScope = new AsyncLocalStorage();
|
|
3
|
+
const poolObservationQueue = { tail: Promise.resolve() };
|
|
4
|
+
/** Serializes pool instrumentation so two fault scopes never wrap the same pool at once. */
|
|
5
|
+
export async function runPoolObservationExclusively(operation) {
|
|
6
|
+
if (observationScope.getStore()?.active)
|
|
7
|
+
throw new Error('PostgreSQL pool instrumentation cannot be nested');
|
|
8
|
+
const previousObservation = poolObservationQueue.tail;
|
|
9
|
+
let releaseObservation;
|
|
10
|
+
poolObservationQueue.tail = new Promise((resolve) => {
|
|
11
|
+
releaseObservation = resolve;
|
|
12
|
+
});
|
|
13
|
+
await previousObservation;
|
|
14
|
+
const context = { active: true };
|
|
15
|
+
try {
|
|
16
|
+
return await observationScope.run(context, operation);
|
|
17
|
+
}
|
|
18
|
+
finally {
|
|
19
|
+
context.active = false;
|
|
20
|
+
releaseObservation();
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Polls `fn` until it returns a non-null value, then returns it. Throws once `maxMs` elapses, even
|
|
3
|
+
* when a reading never settles, so a value that never appears fails at the wait instead of letting
|
|
4
|
+
* a later assertion pass vacuously.
|
|
5
|
+
*/
|
|
6
|
+
export declare function pollUntilNotNull<T>(fn: () => Promise<T | null | undefined>, maxMs?: number, intervalMs?: number, description?: string): Promise<T>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** How long the final reading after the deadline may take before the wait gives up. */
|
|
2
|
+
const FINAL_READING_GRACE_MS = 250;
|
|
3
|
+
async function within(reading, ms, timeoutError) {
|
|
4
|
+
let timer;
|
|
5
|
+
const timeout = new Promise((_, reject) => {
|
|
6
|
+
timer = setTimeout(() => reject(timeoutError), ms);
|
|
7
|
+
});
|
|
8
|
+
try {
|
|
9
|
+
return await Promise.race([reading, timeout]);
|
|
10
|
+
}
|
|
11
|
+
finally {
|
|
12
|
+
clearTimeout(timer);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Polls `fn` until it returns a non-null value, then returns it. Throws once `maxMs` elapses, even
|
|
17
|
+
* when a reading never settles, so a value that never appears fails at the wait instead of letting
|
|
18
|
+
* a later assertion pass vacuously.
|
|
19
|
+
*/
|
|
20
|
+
export async function pollUntilNotNull(fn, maxMs = 2000, intervalMs = 25, description = 'a non-null value') {
|
|
21
|
+
const timeoutError = new Error(`Timed out waiting for ${description} (after ${maxMs}ms)`);
|
|
22
|
+
const deadline = Date.now() + maxMs;
|
|
23
|
+
while (Date.now() < deadline) {
|
|
24
|
+
const result = await within(fn(), Math.max(deadline - Date.now(), 1), timeoutError);
|
|
25
|
+
if (result != null)
|
|
26
|
+
return result;
|
|
27
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
28
|
+
}
|
|
29
|
+
const last = await within(fn(), FINAL_READING_GRACE_MS, timeoutError);
|
|
30
|
+
if (last != null)
|
|
31
|
+
return last;
|
|
32
|
+
throw timeoutError;
|
|
33
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PostgresFaultOptions, PostgresFaultResult } from './fault-types.mts';
|
|
2
|
+
/**
|
|
3
|
+
* Lets the unchanged `pg.Pool#query` core execute its original SQL on an actual aborted client.
|
|
4
|
+
* The statement whose text starts with `queryMarker` (for example a leading block comment naming
|
|
5
|
+
* the statement) is run after a real division by zero aborts the leased transaction, so
|
|
6
|
+
* PostgreSQL answers it with `25P02`.
|
|
7
|
+
*/
|
|
8
|
+
export declare function withPostgresPoolQueryFailureForTest<Result>(queryMarker: string, operation: () => Promise<Result>, options: PostgresFaultOptions): Promise<PostgresFaultResult<Result>>;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import { once } from 'node:events';
|
|
3
|
+
import { assertFaultOptions, hasErrorCode, installProperty, isPlainTextConfig, ownsRequest, queryText, restoreProperty, targetsStatement, } from "./fault-support.mjs";
|
|
4
|
+
import { runPoolObservationExclusively } from "./observation-queue.mjs";
|
|
5
|
+
const leaseContext = new AsyncLocalStorage();
|
|
6
|
+
const failureContext = new AsyncLocalStorage();
|
|
7
|
+
/**
|
|
8
|
+
* Lets the unchanged `pg.Pool#query` core execute its original SQL on an actual aborted client.
|
|
9
|
+
* The statement whose text starts with `queryMarker` (for example a leading block comment naming
|
|
10
|
+
* the statement) is run after a real division by zero aborts the leased transaction, so
|
|
11
|
+
* PostgreSQL answers it with `25P02`.
|
|
12
|
+
*/
|
|
13
|
+
export async function withPostgresPoolQueryFailureForTest(queryMarker, operation, options) {
|
|
14
|
+
assertFaultOptions(queryMarker, options);
|
|
15
|
+
const endTimeoutMs = options.endTimeoutMs ?? 5_000;
|
|
16
|
+
return runPoolObservationExclusively(async () => {
|
|
17
|
+
const context = { active: true };
|
|
18
|
+
let fired = false;
|
|
19
|
+
let targetError;
|
|
20
|
+
let cleanupFailure;
|
|
21
|
+
const originals = options.pools.map((pool) => ({
|
|
22
|
+
pool,
|
|
23
|
+
query: pool.query,
|
|
24
|
+
connect: pool.connect,
|
|
25
|
+
queryDescriptor: Object.getOwnPropertyDescriptor(pool, 'query'),
|
|
26
|
+
connectDescriptor: Object.getOwnPropertyDescriptor(pool, 'connect'),
|
|
27
|
+
}));
|
|
28
|
+
function requireFailure(result) {
|
|
29
|
+
if (cleanupFailure)
|
|
30
|
+
throw cleanupFailure;
|
|
31
|
+
if (!targetError)
|
|
32
|
+
throw new Error('The operation did not execute the targeted failed SQL');
|
|
33
|
+
return { result, error: targetError };
|
|
34
|
+
}
|
|
35
|
+
async function executeFault(pool, query, connect, args) {
|
|
36
|
+
const client = await Reflect.apply(connect, pool, []);
|
|
37
|
+
const lease = { pool, client, active: true, handedOff: false };
|
|
38
|
+
// Register before release: pg-pool destroys errored clients without awaiting their end.
|
|
39
|
+
const ended = once(client, 'end', { signal: AbortSignal.timeout(endTimeoutMs) });
|
|
40
|
+
// Observe rejection immediately even if SQL fails before cleanup awaits this promise.
|
|
41
|
+
const settledEnd = ended.then(() => undefined, (err) => err);
|
|
42
|
+
let result;
|
|
43
|
+
let failure;
|
|
44
|
+
try {
|
|
45
|
+
// The lease transfers release to the unchanged pg-pool core; destroying that client owns rollback.
|
|
46
|
+
await client.query('BEGIN');
|
|
47
|
+
try {
|
|
48
|
+
await client.query('/* withPostgresPoolQueryFailureForTest */ SELECT 1 / 0');
|
|
49
|
+
throw new Error('PostgreSQL unexpectedly accepted division by zero');
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
if (!hasErrorCode(err, '22012'))
|
|
53
|
+
throw err;
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
result = await leaseContext.run(lease, () => Reflect.apply(query, pool, args));
|
|
57
|
+
}
|
|
58
|
+
catch (err) {
|
|
59
|
+
if (!hasErrorCode(err, '25P02'))
|
|
60
|
+
throw err;
|
|
61
|
+
if (!lease.handedOff)
|
|
62
|
+
throw new Error('The original pool core did not acquire the lease', { cause: err });
|
|
63
|
+
targetError = err;
|
|
64
|
+
throw err;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
failure = err;
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
lease.active = false;
|
|
72
|
+
if (!lease.handedOff)
|
|
73
|
+
client.release(true);
|
|
74
|
+
}
|
|
75
|
+
const endError = await settledEnd;
|
|
76
|
+
if (endError) {
|
|
77
|
+
cleanupFailure = new AggregateError([endError, failure], 'The failed pool client did not finish closing');
|
|
78
|
+
throw cleanupFailure;
|
|
79
|
+
}
|
|
80
|
+
if (failure instanceof Error)
|
|
81
|
+
throw failure;
|
|
82
|
+
if (failure)
|
|
83
|
+
throw new Error('Pool query failed with a non-Error rejection', { cause: failure });
|
|
84
|
+
return result;
|
|
85
|
+
}
|
|
86
|
+
function wrapPoolQuery(pool, query, connect) {
|
|
87
|
+
return ((...args) => {
|
|
88
|
+
const text = queryText(args[0]);
|
|
89
|
+
const ownsContext = failureContext.getStore() === context || ownsRequest(options);
|
|
90
|
+
if (!ownsContext ||
|
|
91
|
+
!context.active ||
|
|
92
|
+
fired ||
|
|
93
|
+
!targetsStatement(text, queryMarker, options.command))
|
|
94
|
+
return Reflect.apply(query, pool, args);
|
|
95
|
+
if (!isPlainTextConfig(args))
|
|
96
|
+
throw new Error('Target fault requires the adapter plain-config Promise pool query');
|
|
97
|
+
fired = true;
|
|
98
|
+
return executeFault(pool, query, connect, args);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
function wrapPoolConnect(pool, connect) {
|
|
102
|
+
return ((...args) => {
|
|
103
|
+
const lease = leaseContext.getStore();
|
|
104
|
+
if (lease?.active &&
|
|
105
|
+
lease.pool === pool &&
|
|
106
|
+
!lease.handedOff &&
|
|
107
|
+
args.length === 1 &&
|
|
108
|
+
typeof args[0] === 'function') {
|
|
109
|
+
const callback = args[0];
|
|
110
|
+
queueMicrotask(() => {
|
|
111
|
+
if (!lease.active)
|
|
112
|
+
return;
|
|
113
|
+
lease.handedOff = true;
|
|
114
|
+
callback(null, lease.client, lease.client.release);
|
|
115
|
+
});
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
return Reflect.apply(connect, pool, args);
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
try {
|
|
122
|
+
for (const { pool, query, connect } of originals) {
|
|
123
|
+
installProperty(pool, 'connect', wrapPoolConnect(pool, connect));
|
|
124
|
+
installProperty(pool, 'query', wrapPoolQuery(pool, query, connect));
|
|
125
|
+
}
|
|
126
|
+
return await failureContext.run(context, async () => requireFailure(await operation()));
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
context.active = false;
|
|
130
|
+
for (const { pool, queryDescriptor, connectDescriptor } of originals) {
|
|
131
|
+
restoreProperty(pool, 'query', queryDescriptor);
|
|
132
|
+
restoreProperty(pool, 'connect', connectDescriptor);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
}
|