slicetest 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -2
- package/dist/auth.d.ts +45 -0
- package/dist/auth.js +128 -0
- package/dist/cli.js +8 -3
- package/dist/config.d.ts +14 -0
- package/dist/config.js +16 -0
- package/dist/db.d.ts +26 -0
- package/dist/db.js +50 -0
- package/dist/drivers/driver.d.ts +27 -0
- package/dist/drivers/mysql.d.ts +2 -1
- package/dist/drivers/mysql.js +38 -0
- package/dist/drivers/postgres.d.ts +2 -1
- package/dist/drivers/postgres.js +28 -0
- package/dist/drivers/sqlite.d.ts +2 -1
- package/dist/drivers/sqlite.js +52 -0
- package/dist/factory.d.ts +21 -0
- package/dist/factory.js +129 -0
- package/dist/gen.d.ts +1 -0
- package/dist/gen.js +9 -4
- package/dist/http.d.ts +7 -0
- package/dist/http.js +10 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +2 -0
- package/dist/init.js +44 -1
- package/dist/openapi.d.ts +8 -0
- package/dist/openapi.js +18 -0
- package/dist/query-log.d.ts +49 -0
- package/dist/query-log.js +261 -0
- package/dist/runtime.d.ts +6 -0
- package/dist/runtime.js +40 -2
- package/dist/stub.d.ts +32 -0
- package/dist/stub.js +87 -0
- package/dist/webhook.d.ts +40 -0
- package/dist/webhook.js +52 -0
- package/dist/yaml-runtime.js +41 -3
- package/dist/yaml.d.ts +36 -1
- package/dist/yaml.js +56 -2
- package/package.json +8 -2
- package/schema/scenario.schema.json +180 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import net from "node:net";
|
|
2
|
+
const TRANSACTION = /^\s*(BEGIN|START\s+TRANSACTION|COMMIT|END|ROLLBACK|SAVEPOINT|RELEASE|SET|SHOW|DISCARD|DEALLOCATE)\b/i;
|
|
3
|
+
/** A list of queries, with helpers for spotting N+1 patterns. */
|
|
4
|
+
export class QueryList extends Array {
|
|
5
|
+
/** Statements grouped by shape (literals and parameters replaced by `?`), most frequent first. */
|
|
6
|
+
shapes() {
|
|
7
|
+
const counts = new Map();
|
|
8
|
+
for (const q of this) {
|
|
9
|
+
const shape = normalizeSql(q.sql);
|
|
10
|
+
counts.set(shape, (counts.get(shape) ?? 0) + 1);
|
|
11
|
+
}
|
|
12
|
+
return [...counts].map(([sql, count]) => ({ sql, count })).sort((a, b) => b.count - a.count);
|
|
13
|
+
}
|
|
14
|
+
/** Without transaction control and session statements (`BEGIN`, `COMMIT`, `SAVEPOINT`, `SET`, …), which some drivers add around every query. */
|
|
15
|
+
withoutTransactions() {
|
|
16
|
+
return queryList(this.filter((q) => !TRANSACTION.test(q.sql)));
|
|
17
|
+
}
|
|
18
|
+
/** Shapes run at least `min` times (default 3): the usual signature of an N+1. */
|
|
19
|
+
repeated(min = 3) {
|
|
20
|
+
return this.shapes().filter((s) => s.count >= min);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export function queryList(queries) {
|
|
24
|
+
const list = new QueryList();
|
|
25
|
+
list.push(...queries);
|
|
26
|
+
return list;
|
|
27
|
+
}
|
|
28
|
+
/** `SELECT * FROM t WHERE id = 7 AND name = 'x'` → `SELECT * FROM t WHERE id = ? AND name = ?`. */
|
|
29
|
+
export function normalizeSql(sql) {
|
|
30
|
+
return sql
|
|
31
|
+
.replace(/\s+/g, " ")
|
|
32
|
+
.trim()
|
|
33
|
+
.replace(/'(?:[^']|'')*'/g, "?")
|
|
34
|
+
.replace(/\$\d+/g, "?")
|
|
35
|
+
.replace(/(?<![\w"`.])-?\d+(\.\d+)?\b/g, "?")
|
|
36
|
+
.replace(/\(\s*\?(\s*,\s*\?)+\s*\)/g, "(?)");
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* A TCP proxy between the app and its database that reads the wire protocol
|
|
40
|
+
* and records every statement the app runs, whatever its language or driver.
|
|
41
|
+
* Postgres: simple queries and extended-protocol executions. MySQL: COM_QUERY
|
|
42
|
+
* and prepared-statement executions. Connections that switch to TLS are
|
|
43
|
+
* forwarded but not read.
|
|
44
|
+
*/
|
|
45
|
+
export class QueryLog {
|
|
46
|
+
server;
|
|
47
|
+
port;
|
|
48
|
+
#queries = [];
|
|
49
|
+
#started = performance.now();
|
|
50
|
+
#sockets = new Set();
|
|
51
|
+
constructor(server, port) {
|
|
52
|
+
this.server = server;
|
|
53
|
+
this.port = port;
|
|
54
|
+
}
|
|
55
|
+
static async start(engine, upstream) {
|
|
56
|
+
let log;
|
|
57
|
+
const server = net.createServer((client) => log.#connect(engine, client, upstream));
|
|
58
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
59
|
+
log = new QueryLog(server, server.address().port);
|
|
60
|
+
return log;
|
|
61
|
+
}
|
|
62
|
+
/** `url` with its host and port pointing at the proxy. */
|
|
63
|
+
proxyUrl(url) {
|
|
64
|
+
const u = new URL(url);
|
|
65
|
+
u.hostname = "127.0.0.1";
|
|
66
|
+
u.port = String(this.port);
|
|
67
|
+
return u.toString();
|
|
68
|
+
}
|
|
69
|
+
/** Queries since the last `reset()`, or since `mark` (an index from `mark()`). */
|
|
70
|
+
queries(since = 0) {
|
|
71
|
+
return queryList(this.#queries.slice(since));
|
|
72
|
+
}
|
|
73
|
+
mark() {
|
|
74
|
+
return this.#queries.length;
|
|
75
|
+
}
|
|
76
|
+
reset() {
|
|
77
|
+
this.#queries = [];
|
|
78
|
+
}
|
|
79
|
+
#record(sql) {
|
|
80
|
+
this.#queries.push({ sql, at: Math.round(performance.now() - this.#started) });
|
|
81
|
+
}
|
|
82
|
+
#connect(engine, client, upstream) {
|
|
83
|
+
const server = net.connect(upstream.port, upstream.host);
|
|
84
|
+
this.#sockets.add(client).add(server);
|
|
85
|
+
const close = () => {
|
|
86
|
+
client.destroy();
|
|
87
|
+
server.destroy();
|
|
88
|
+
this.#sockets.delete(client);
|
|
89
|
+
this.#sockets.delete(server);
|
|
90
|
+
};
|
|
91
|
+
client.on("error", close).on("close", close);
|
|
92
|
+
server.on("error", close).on("close", close);
|
|
93
|
+
const reader = engine === "postgres" ? new PostgresReader((sql) => this.#record(sql)) : new MysqlReader((sql) => this.#record(sql));
|
|
94
|
+
client.on("data", (chunk) => {
|
|
95
|
+
try {
|
|
96
|
+
reader.fromClient(chunk);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
reader.opaque = true;
|
|
100
|
+
}
|
|
101
|
+
server.write(chunk);
|
|
102
|
+
});
|
|
103
|
+
server.on("data", (chunk) => {
|
|
104
|
+
try {
|
|
105
|
+
reader.fromServer(chunk);
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
reader.opaque = true;
|
|
109
|
+
}
|
|
110
|
+
client.write(chunk);
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
async close() {
|
|
114
|
+
for (const s of this.#sockets)
|
|
115
|
+
s.destroy();
|
|
116
|
+
await new Promise((resolve) => this.server.close(resolve));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const cstring = (buf, start) => {
|
|
120
|
+
const end = buf.indexOf(0, start);
|
|
121
|
+
return [buf.toString("utf8", start, end), end + 1];
|
|
122
|
+
};
|
|
123
|
+
class PostgresReader {
|
|
124
|
+
record;
|
|
125
|
+
opaque = false;
|
|
126
|
+
#buf = Buffer.alloc(0);
|
|
127
|
+
#startup = true;
|
|
128
|
+
#awaitingSsl = false;
|
|
129
|
+
#statements = new Map();
|
|
130
|
+
#portals = new Map();
|
|
131
|
+
constructor(record) {
|
|
132
|
+
this.record = record;
|
|
133
|
+
}
|
|
134
|
+
fromServer(chunk) {
|
|
135
|
+
if (!this.#awaitingSsl)
|
|
136
|
+
return;
|
|
137
|
+
this.#awaitingSsl = false;
|
|
138
|
+
// 'S' (TLS) or 'G' (GSSAPI encryption): the rest is encrypted.
|
|
139
|
+
if (chunk[0] === 0x53 || chunk[0] === 0x47)
|
|
140
|
+
this.opaque = true;
|
|
141
|
+
}
|
|
142
|
+
fromClient(chunk) {
|
|
143
|
+
if (this.opaque)
|
|
144
|
+
return;
|
|
145
|
+
this.#buf = Buffer.concat([this.#buf, chunk]);
|
|
146
|
+
for (;;) {
|
|
147
|
+
if (this.#startup) {
|
|
148
|
+
if (this.#buf.length < 8)
|
|
149
|
+
return;
|
|
150
|
+
const len = this.#buf.readInt32BE(0);
|
|
151
|
+
if (this.#buf.length < len)
|
|
152
|
+
return;
|
|
153
|
+
const code = this.#buf.readInt32BE(4);
|
|
154
|
+
this.#buf = this.#buf.subarray(len);
|
|
155
|
+
if (code === 80877103 || code === 80877104)
|
|
156
|
+
this.#awaitingSsl = true;
|
|
157
|
+
else if (code !== 80877102)
|
|
158
|
+
this.#startup = false;
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
if (this.#buf.length < 5)
|
|
162
|
+
return;
|
|
163
|
+
const type = String.fromCharCode(this.#buf[0]);
|
|
164
|
+
const len = this.#buf.readInt32BE(1);
|
|
165
|
+
if (this.#buf.length < len + 1)
|
|
166
|
+
return;
|
|
167
|
+
const msg = this.#buf.subarray(5, len + 1);
|
|
168
|
+
this.#buf = this.#buf.subarray(len + 1);
|
|
169
|
+
this.#message(type, msg);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
#message(type, msg) {
|
|
173
|
+
if (type === "Q") {
|
|
174
|
+
this.record(cstring(msg, 0)[0]);
|
|
175
|
+
}
|
|
176
|
+
else if (type === "P") {
|
|
177
|
+
const [name, next] = cstring(msg, 0);
|
|
178
|
+
this.#statements.set(name, cstring(msg, next)[0]);
|
|
179
|
+
}
|
|
180
|
+
else if (type === "B") {
|
|
181
|
+
const [portal, next] = cstring(msg, 0);
|
|
182
|
+
this.#portals.set(portal, this.#statements.get(cstring(msg, next)[0]) ?? "");
|
|
183
|
+
}
|
|
184
|
+
else if (type === "E") {
|
|
185
|
+
const sql = this.#portals.get(cstring(msg, 0)[0]);
|
|
186
|
+
if (sql)
|
|
187
|
+
this.record(sql);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
const CLIENT_QUERY_ATTRIBUTES = 1 << 27;
|
|
192
|
+
class MysqlReader {
|
|
193
|
+
record;
|
|
194
|
+
opaque = false;
|
|
195
|
+
#client = Buffer.alloc(0);
|
|
196
|
+
#server = Buffer.alloc(0);
|
|
197
|
+
#prepares = [];
|
|
198
|
+
#attributes = false;
|
|
199
|
+
#statements = new Map();
|
|
200
|
+
constructor(record) {
|
|
201
|
+
this.record = record;
|
|
202
|
+
}
|
|
203
|
+
#read(buf, each) {
|
|
204
|
+
while (buf.length >= 4) {
|
|
205
|
+
const len = buf.readUIntLE(0, 3);
|
|
206
|
+
if (buf.length < len + 4)
|
|
207
|
+
break;
|
|
208
|
+
each(buf[3], buf.subarray(4, len + 4));
|
|
209
|
+
buf = buf.subarray(len + 4);
|
|
210
|
+
}
|
|
211
|
+
return Buffer.from(buf);
|
|
212
|
+
}
|
|
213
|
+
fromClient(chunk) {
|
|
214
|
+
if (this.opaque)
|
|
215
|
+
return;
|
|
216
|
+
this.#client = this.#read(Buffer.concat([this.#client, chunk]), (seq, p) => {
|
|
217
|
+
// The handshake response asking for TLS is a short packet with CLIENT_SSL set.
|
|
218
|
+
if (seq === 1 && p.length === 32 && (p.readUInt32LE(0) & 0x800) !== 0)
|
|
219
|
+
this.opaque = true;
|
|
220
|
+
if (seq === 1 && p.length >= 4)
|
|
221
|
+
this.#attributes = (p.readUInt32LE(0) & CLIENT_QUERY_ATTRIBUTES) !== 0;
|
|
222
|
+
// Commands start a new sequence; handshake and auth packets don't.
|
|
223
|
+
if (seq !== 0 || p.length === 0)
|
|
224
|
+
return;
|
|
225
|
+
if (p[0] === 0x03)
|
|
226
|
+
this.record(this.#queryText(p));
|
|
227
|
+
else if (p[0] === 0x16)
|
|
228
|
+
this.#prepares.push(p.toString("utf8", 1));
|
|
229
|
+
else if (p[0] === 0x17 && p.length >= 5) {
|
|
230
|
+
const sql = this.#statements.get(p.readUInt32LE(1));
|
|
231
|
+
if (sql)
|
|
232
|
+
this.record(sql);
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
/** COM_QUERY text; with query attributes it follows a parameter count, a set count and the attributes. */
|
|
237
|
+
#queryText(p) {
|
|
238
|
+
if (!this.#attributes)
|
|
239
|
+
return p.toString("utf8", 1);
|
|
240
|
+
const count = p[1];
|
|
241
|
+
// Drivers send no attributes (count 0, one set); anything else is read past heuristically.
|
|
242
|
+
if (count === 0)
|
|
243
|
+
return p.toString("utf8", 3);
|
|
244
|
+
const text = p.toString("utf8", 3);
|
|
245
|
+
const start = text.search(/\b(SELECT|INSERT|UPDATE|DELETE|WITH|REPLACE|CALL|SET|BEGIN|COMMIT|ROLLBACK|START)\b/i);
|
|
246
|
+
return start >= 0 ? text.slice(start) : text;
|
|
247
|
+
}
|
|
248
|
+
fromServer(chunk) {
|
|
249
|
+
if (this.opaque || this.#prepares.length === 0) {
|
|
250
|
+
this.#server = Buffer.alloc(0);
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
this.#server = this.#read(Buffer.concat([this.#server, chunk]), (seq, p) => {
|
|
254
|
+
if (seq !== 1 || this.#prepares.length === 0)
|
|
255
|
+
return;
|
|
256
|
+
const sql = this.#prepares.shift();
|
|
257
|
+
if (p[0] === 0x00 && p.length >= 5)
|
|
258
|
+
this.#statements.set(p.readUInt32LE(1), sql);
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
}
|
package/dist/runtime.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { App } from "./app.js";
|
|
2
|
+
import { Issuer } from "./auth.js";
|
|
2
3
|
import type { ResolvedOptions } from "./config.js";
|
|
3
4
|
import { Dependency } from "./containers.js";
|
|
4
5
|
import { Db } from "./db.js";
|
|
5
6
|
import { HttpClient } from "./http.js";
|
|
6
7
|
import { Mailbox } from "./mail.js";
|
|
8
|
+
import { QueryLog } from "./query-log.js";
|
|
7
9
|
import { Stub } from "./stub.js";
|
|
8
10
|
import { type MaskOptions, type Trace } from "./trace.js";
|
|
9
11
|
export interface ScenarioContext {
|
|
@@ -22,6 +24,8 @@ export interface ScenarioContext {
|
|
|
22
24
|
trace: (opts?: MaskOptions) => Promise<Trace>;
|
|
23
25
|
/** Mail the app sent during the scenario. Needs `mail: true` in the config. */
|
|
24
26
|
mail: Mailbox;
|
|
27
|
+
/** The OpenID Connect issuer the app trusts: `auth.token(claims)`. Needs `auth` in the config. */
|
|
28
|
+
auth: Issuer;
|
|
25
29
|
}
|
|
26
30
|
/** Everything one test file needs: its own database, stub servers and app process. */
|
|
27
31
|
export declare class Runtime {
|
|
@@ -38,6 +42,8 @@ export declare class Runtime {
|
|
|
38
42
|
private readonly recordDir?;
|
|
39
43
|
readonly containers: Map<string, Dependency>;
|
|
40
44
|
readonly mailbox?: Mailbox | undefined;
|
|
45
|
+
readonly issuer?: Issuer | undefined;
|
|
46
|
+
readonly queryLog?: QueryLog | undefined;
|
|
41
47
|
private constructor();
|
|
42
48
|
get http(): HttpClient;
|
|
43
49
|
static start(opts: ResolvedOptions, shared: {
|
package/dist/runtime.js
CHANGED
|
@@ -2,12 +2,14 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { writeFile } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { App } from "./app.js";
|
|
5
|
+
import { Issuer } from "./auth.js";
|
|
5
6
|
import { Dependency } from "./containers.js";
|
|
6
7
|
import { Db, formatChanges } from "./db.js";
|
|
7
8
|
import { engineFor } from "./drivers/index.js";
|
|
8
9
|
import { formatHistory, HttpClient } from "./http.js";
|
|
9
10
|
import { Mailbox } from "./mail.js";
|
|
10
11
|
import { OpenApiSpec } from "./openapi.js";
|
|
12
|
+
import { QueryLog } from "./query-log.js";
|
|
11
13
|
import { Recorder } from "./recording.js";
|
|
12
14
|
import { Stub } from "./stub.js";
|
|
13
15
|
import { buildTrace, mask } from "./trace.js";
|
|
@@ -25,12 +27,14 @@ export class Runtime {
|
|
|
25
27
|
recordDir;
|
|
26
28
|
containers;
|
|
27
29
|
mailbox;
|
|
30
|
+
issuer;
|
|
31
|
+
queryLog;
|
|
28
32
|
#http;
|
|
29
33
|
/** Responses from the app that don't match its OpenAPI spec, this scenario. */
|
|
30
34
|
#contract = [];
|
|
31
35
|
/** Documented responses seen in this file, for the run's coverage report. */
|
|
32
36
|
#covered = new Set();
|
|
33
|
-
constructor(app, services, db, stubs, opts, vars, specs, coverageDir, recorders = new Map(), recordDir, containers = new Map(), mailbox) {
|
|
37
|
+
constructor(app, services, db, stubs, opts, vars, specs, coverageDir, recorders = new Map(), recordDir, containers = new Map(), mailbox, issuer, queryLog) {
|
|
34
38
|
this.app = app;
|
|
35
39
|
this.services = services;
|
|
36
40
|
this.db = db;
|
|
@@ -43,6 +47,8 @@ export class Runtime {
|
|
|
43
47
|
this.recordDir = recordDir;
|
|
44
48
|
this.containers = containers;
|
|
45
49
|
this.mailbox = mailbox;
|
|
50
|
+
this.issuer = issuer;
|
|
51
|
+
this.queryLog = queryLog;
|
|
46
52
|
this.#http = this.#client();
|
|
47
53
|
}
|
|
48
54
|
#client() {
|
|
@@ -76,6 +82,8 @@ export class Runtime {
|
|
|
76
82
|
const services = new Map();
|
|
77
83
|
let db;
|
|
78
84
|
let mailbox;
|
|
85
|
+
let issuer;
|
|
86
|
+
let queryLog;
|
|
79
87
|
try {
|
|
80
88
|
// Loaded first: a broken spec should fail before anything is started.
|
|
81
89
|
const specs = {
|
|
@@ -106,12 +114,22 @@ export class Runtime {
|
|
|
106
114
|
if (failed)
|
|
107
115
|
throw failed.reason;
|
|
108
116
|
const vars = { "db.url": url };
|
|
117
|
+
if (opts.db.queries && engine.name !== "sqlite") {
|
|
118
|
+
const u = new URL(url);
|
|
119
|
+
queryLog = await QueryLog.start(engine.name, { host: u.hostname, port: Number(u.port || (engine.name === "mysql" ? 3306 : 5432)) });
|
|
120
|
+
vars["db.url"] = queryLog.proxyUrl(url);
|
|
121
|
+
db.attachQueryLog(queryLog);
|
|
122
|
+
}
|
|
109
123
|
if (engine.name === "sqlite")
|
|
110
124
|
vars["db.path"] = (await import("./drivers/sqlite.js")).sqlitePath(url);
|
|
111
125
|
if (opts.mail) {
|
|
112
126
|
mailbox = await Mailbox.start();
|
|
113
127
|
Object.assign(vars, { "mail.host": mailbox.host, "mail.port": String(mailbox.port), "mail.url": mailbox.url });
|
|
114
128
|
}
|
|
129
|
+
if (opts.auth) {
|
|
130
|
+
issuer = await Issuer.start(opts.auth);
|
|
131
|
+
Object.assign(vars, { "auth.issuer": issuer.url, "auth.jwks": issuer.jwksUrl, "auth.audience": issuer.audience });
|
|
132
|
+
}
|
|
115
133
|
for (const [name, c] of containers) {
|
|
116
134
|
vars[`container.${name}`] = c.address;
|
|
117
135
|
vars[`container.${name}.host`] = c.host;
|
|
@@ -126,7 +144,7 @@ export class Runtime {
|
|
|
126
144
|
vars[`service.${name}.port`] = String(started.port);
|
|
127
145
|
}
|
|
128
146
|
const app = await App.start(opts.app, opts.root, vars);
|
|
129
|
-
return new Runtime(app, services, db, stubs, opts, vars, specs, shared.coverageDir, recorders, shared.recordDir, containers, mailbox);
|
|
147
|
+
return new Runtime(app, services, db, stubs, opts, vars, specs, shared.coverageDir, recorders, shared.recordDir, containers, mailbox, issuer, queryLog);
|
|
130
148
|
}
|
|
131
149
|
catch (e) {
|
|
132
150
|
await Promise.all([...services.values()].map((s) => s.stop()));
|
|
@@ -134,11 +152,14 @@ export class Runtime {
|
|
|
134
152
|
await Promise.all([...stubs.values()].map((s) => s.close()));
|
|
135
153
|
await Promise.allSettled([...containers.values()].map((c) => c.stop()));
|
|
136
154
|
await mailbox?.close();
|
|
155
|
+
await issuer?.close();
|
|
156
|
+
await queryLog?.close();
|
|
137
157
|
throw e;
|
|
138
158
|
}
|
|
139
159
|
}
|
|
140
160
|
context() {
|
|
141
161
|
const mailbox = this.mailbox;
|
|
162
|
+
const issuer = this.issuer;
|
|
142
163
|
return {
|
|
143
164
|
http: this.http,
|
|
144
165
|
db: this.db,
|
|
@@ -169,6 +190,11 @@ export class Runtime {
|
|
|
169
190
|
throw new Error("slicetest: mail is off. Add `mail: true` to the config and point the app's SMTP settings at {{mail.host}} / {{mail.port}}.");
|
|
170
191
|
return mailbox;
|
|
171
192
|
},
|
|
193
|
+
get auth() {
|
|
194
|
+
if (!issuer)
|
|
195
|
+
throw new Error("slicetest: auth is off. Add `auth: true` to the config and point the app's JWT settings at {{auth.issuer}} / {{auth.jwks}}.");
|
|
196
|
+
return issuer;
|
|
197
|
+
},
|
|
172
198
|
};
|
|
173
199
|
}
|
|
174
200
|
/** The app and every service, for checks that apply to all of them. */
|
|
@@ -195,6 +221,8 @@ export class Runtime {
|
|
|
195
221
|
for (const recorder of this.recorders.values())
|
|
196
222
|
recorder.reset();
|
|
197
223
|
this.mailbox?.reset();
|
|
224
|
+
this.issuer?.reset();
|
|
225
|
+
this.queryLog?.reset();
|
|
198
226
|
await Promise.all([...this.containers.values()].map((c) => c.reset()));
|
|
199
227
|
this.http.reset();
|
|
200
228
|
for (const p of this.#processes())
|
|
@@ -265,6 +293,9 @@ export class Runtime {
|
|
|
265
293
|
const unmatched = this.#unmatched();
|
|
266
294
|
if (unmatched.length > 0)
|
|
267
295
|
sections.push(`stub calls with no matching route:\n${unmatched.join("\n")}`);
|
|
296
|
+
const chaos = [...this.stubs.values()].map((s) => s.describeChaos()).filter((c) => c !== undefined);
|
|
297
|
+
if (chaos.length > 0)
|
|
298
|
+
sections.push(chaos.join("\n"));
|
|
268
299
|
const contract = this.#contractViolations();
|
|
269
300
|
if (contract.length > 0)
|
|
270
301
|
sections.push(`OpenAPI mismatches:\n${contract.map((c) => ` ${c}`).join("\n")}`);
|
|
@@ -279,6 +310,11 @@ export class Runtime {
|
|
|
279
310
|
}
|
|
280
311
|
if (this.mailbox)
|
|
281
312
|
sections.push(this.mailbox.describe());
|
|
313
|
+
if (this.queryLog) {
|
|
314
|
+
const q = this.queryLog.queries();
|
|
315
|
+
const top = q.shapes().slice(0, 5).map((s) => ` ${s.count > 1 ? `×${s.count} ` : ""}${s.sql.length > 160 ? `${s.sql.slice(0, 157)}...` : s.sql}`);
|
|
316
|
+
sections.push(q.length ? `SQL the app ran during this scenario (${q.length} statements, most frequent first):\n${top.join("\n")}` : "SQL the app ran during this scenario: (none)");
|
|
317
|
+
}
|
|
282
318
|
const logs = this.app.scenarioLogs();
|
|
283
319
|
sections.push(logs ? `app output during this scenario:\n${logs}` : "app output during this scenario: (none)");
|
|
284
320
|
for (const [name, service] of this.services) {
|
|
@@ -303,6 +339,8 @@ export class Runtime {
|
|
|
303
339
|
...[...this.stubs.values()].map((s) => s.close()),
|
|
304
340
|
...[...this.containers.values()].map((c) => c.stop()),
|
|
305
341
|
this.mailbox?.close(),
|
|
342
|
+
this.issuer?.close(),
|
|
343
|
+
this.queryLog?.close(),
|
|
306
344
|
]);
|
|
307
345
|
const failed = results.find((r) => r.status === "rejected");
|
|
308
346
|
if (failed)
|
package/dist/stub.d.ts
CHANGED
|
@@ -19,6 +19,27 @@ export interface RecordedCall {
|
|
|
19
19
|
};
|
|
20
20
|
/** Answered by the fallback (e.g. an example from the provider's OpenAPI spec), not a registered route. */
|
|
21
21
|
fallback?: boolean;
|
|
22
|
+
/** The fault `chaos()` injected instead of the normal answer: `503`, `reset`. */
|
|
23
|
+
fault?: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Faults injected into a stub's answers, to test the app's retries, timeouts and
|
|
27
|
+
* fallbacks. Random faults come from a seeded generator, so a failing run can be
|
|
28
|
+
* replayed with the seed printed in the failure output.
|
|
29
|
+
*/
|
|
30
|
+
export interface ChaosOptions {
|
|
31
|
+
/** Fail the first `n` calls, then answer normally: the shape of a retry test. */
|
|
32
|
+
failFirst?: number;
|
|
33
|
+
/** Share of calls (0–1) answered with one of `statuses`. */
|
|
34
|
+
errorRate?: number;
|
|
35
|
+
/** Error statuses to pick from. Default [500, 502, 503]; 429 and 503 come with `Retry-After: 1`. */
|
|
36
|
+
statuses?: number[];
|
|
37
|
+
/** Share of calls (0–1) whose connection is dropped without an answer. */
|
|
38
|
+
networkErrorRate?: number;
|
|
39
|
+
/** Extra delay for every call, in ms: a fixed value or a [min, max] range. */
|
|
40
|
+
latency?: number | [number, number];
|
|
41
|
+
/** Seed for the random choices. Default: `SLICETEST_CHAOS_SEED`, else random. */
|
|
42
|
+
seed?: number;
|
|
22
43
|
}
|
|
23
44
|
export interface StubResponse {
|
|
24
45
|
status?: number;
|
|
@@ -70,6 +91,17 @@ export declare class Stub {
|
|
|
70
91
|
* `call.params`) or be a RegExp; `method` may be `*`. Later routes win.
|
|
71
92
|
*/
|
|
72
93
|
on(method: string, path: string | RegExp, match?: MatchOptions): RouteBuilder;
|
|
94
|
+
/**
|
|
95
|
+
* Inject faults into this stub's answers for the rest of the scenario:
|
|
96
|
+
* `stub("payments").chaos({ failFirst: 2 })` to test a retry,
|
|
97
|
+
* `chaos({ errorRate: 0.3, latency: [50, 200] })` to test resilience.
|
|
98
|
+
* Calls that fault don't use up `once()` / `times()` routes.
|
|
99
|
+
*/
|
|
100
|
+
chaos(opts: ChaosOptions): this;
|
|
101
|
+
/** Calls `chaos()` answered with a fault. */
|
|
102
|
+
faults(): RecordedCall[];
|
|
103
|
+
/** The active `chaos()` settings and what they did, for failure output; undefined when off. */
|
|
104
|
+
describeChaos(): string | undefined;
|
|
73
105
|
/** Calls received so far, optionally filtered by method, path and conditions (same syntax as `on()`). */
|
|
74
106
|
calls(method?: string, path?: string | RegExp, match?: MatchOptions): RecordedCall[];
|
|
75
107
|
unmatched(): RecordedCall[];
|
package/dist/stub.js
CHANGED
|
@@ -12,6 +12,7 @@ export class Stub {
|
|
|
12
12
|
#fallback;
|
|
13
13
|
/** Appended to the 501 answer for a call nothing could answer. */
|
|
14
14
|
#hint;
|
|
15
|
+
#chaos;
|
|
15
16
|
url = "";
|
|
16
17
|
constructor(name) {
|
|
17
18
|
this.name = name;
|
|
@@ -61,6 +62,74 @@ export class Stub {
|
|
|
61
62
|
};
|
|
62
63
|
return builder;
|
|
63
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* Inject faults into this stub's answers for the rest of the scenario:
|
|
67
|
+
* `stub("payments").chaos({ failFirst: 2 })` to test a retry,
|
|
68
|
+
* `chaos({ errorRate: 0.3, latency: [50, 200] })` to test resilience.
|
|
69
|
+
* Calls that fault don't use up `once()` / `times()` routes.
|
|
70
|
+
*/
|
|
71
|
+
chaos(opts) {
|
|
72
|
+
for (const key of ["errorRate", "networkErrorRate"]) {
|
|
73
|
+
const v = opts[key];
|
|
74
|
+
if (v !== undefined && !(v >= 0 && v <= 1))
|
|
75
|
+
throw new Error(`slicetest: chaos ${key} must be between 0 and 1, got ${v}`);
|
|
76
|
+
}
|
|
77
|
+
if (opts.statuses && (opts.statuses.length === 0 || opts.statuses.some((s) => !Number.isInteger(s) || s < 400 || s > 599))) {
|
|
78
|
+
throw new Error(`slicetest: chaos statuses must be 4xx/5xx codes, got ${JSON.stringify(opts.statuses)}`);
|
|
79
|
+
}
|
|
80
|
+
const envSeed = Number(process.env.SLICETEST_CHAOS_SEED);
|
|
81
|
+
const seed = opts.seed ?? (Number.isInteger(envSeed) ? envSeed : Math.floor(Math.random() * 2 ** 31));
|
|
82
|
+
this.#chaos = { opts, seed, random: mulberry32(seed), calls: 0 };
|
|
83
|
+
return this;
|
|
84
|
+
}
|
|
85
|
+
/** Calls `chaos()` answered with a fault. */
|
|
86
|
+
faults() {
|
|
87
|
+
return this.#calls.filter((c) => c.fault !== undefined);
|
|
88
|
+
}
|
|
89
|
+
/** The active `chaos()` settings and what they did, for failure output; undefined when off. */
|
|
90
|
+
describeChaos() {
|
|
91
|
+
const c = this.#chaos;
|
|
92
|
+
if (!c)
|
|
93
|
+
return undefined;
|
|
94
|
+
const settings = Object.entries(c.opts)
|
|
95
|
+
.filter(([k]) => k !== "seed")
|
|
96
|
+
.map(([k, v]) => `${k} ${JSON.stringify(v)}`)
|
|
97
|
+
.join(", ");
|
|
98
|
+
return `chaos on ${this.name}: ${settings}; ${this.faults().length} of ${c.calls} calls faulted. Replay with SLICETEST_CHAOS_SEED=${c.seed}`;
|
|
99
|
+
}
|
|
100
|
+
/** Delay and fault for the next call, consuming the random sequence in a fixed order. */
|
|
101
|
+
async #injectFault() {
|
|
102
|
+
const c = this.#chaos;
|
|
103
|
+
if (!c)
|
|
104
|
+
return undefined;
|
|
105
|
+
const { opts, random } = c;
|
|
106
|
+
const n = ++c.calls;
|
|
107
|
+
const lat = opts.latency;
|
|
108
|
+
const ms = lat === undefined ? 0 : Array.isArray(lat) ? lat[0] + Math.floor(random() * (lat[1] - lat[0] + 1)) : lat;
|
|
109
|
+
if (ms > 0)
|
|
110
|
+
await new Promise((r) => setTimeout(r, ms));
|
|
111
|
+
const statuses = opts.statuses ?? [500, 502, 503];
|
|
112
|
+
const pick = () => statuses[Math.floor(random() * statuses.length)];
|
|
113
|
+
if (opts.failFirst !== undefined && n <= opts.failFirst)
|
|
114
|
+
return { status: pick() };
|
|
115
|
+
const roll = random();
|
|
116
|
+
if (roll < (opts.networkErrorRate ?? 0))
|
|
117
|
+
return "reset";
|
|
118
|
+
if (roll < (opts.networkErrorRate ?? 0) + (opts.errorRate ?? 0))
|
|
119
|
+
return { status: pick() };
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
#fail(call, req, res, fault) {
|
|
123
|
+
call.matched = true;
|
|
124
|
+
if (fault === "reset") {
|
|
125
|
+
call.fault = "reset";
|
|
126
|
+
req.socket.destroy();
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
call.fault = String(fault.status);
|
|
130
|
+
const retry = fault.status === 429 || fault.status === 503 ? { "retry-after": "1" } : undefined;
|
|
131
|
+
this.#send(call, res, { status: fault.status, headers: retry, body: { error: "slicetest chaos", status: fault.status } });
|
|
132
|
+
}
|
|
64
133
|
/** Calls received so far, optionally filtered by method, path and conditions (same syntax as `on()`). */
|
|
65
134
|
calls(method, path, match = {}) {
|
|
66
135
|
const compiled = path === undefined ? undefined : compilePath(path);
|
|
@@ -82,6 +151,7 @@ export class Stub {
|
|
|
82
151
|
reset() {
|
|
83
152
|
this.#routes = [];
|
|
84
153
|
this.#calls = [];
|
|
154
|
+
this.#chaos = undefined;
|
|
85
155
|
}
|
|
86
156
|
/**
|
|
87
157
|
* Answer calls that no registered route matches, instead of failing with 501.
|
|
@@ -127,6 +197,12 @@ export class Stub {
|
|
|
127
197
|
route = r;
|
|
128
198
|
break;
|
|
129
199
|
}
|
|
200
|
+
// Faults only replace answers the stub would give: an unknown route still fails as unmatched.
|
|
201
|
+
if (route || this.#fallback) {
|
|
202
|
+
const fault = await this.#injectFault();
|
|
203
|
+
if (fault)
|
|
204
|
+
return this.#fail(call, req, res, fault);
|
|
205
|
+
}
|
|
130
206
|
if (!route) {
|
|
131
207
|
let out;
|
|
132
208
|
try {
|
|
@@ -262,3 +338,14 @@ function parseJson(body) {
|
|
|
262
338
|
return undefined;
|
|
263
339
|
}
|
|
264
340
|
}
|
|
341
|
+
/** A small seeded PRNG: the same seed gives the same faults on every run. */
|
|
342
|
+
function mulberry32(seed) {
|
|
343
|
+
let a = seed >>> 0;
|
|
344
|
+
return () => {
|
|
345
|
+
a = (a + 0x6d2b79f5) >>> 0;
|
|
346
|
+
let t = a;
|
|
347
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
348
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
349
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
350
|
+
};
|
|
351
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/** A custom HMAC scheme: `header: prefix + hmac(secret, body)`. */
|
|
2
|
+
export interface HmacScheme {
|
|
3
|
+
header: string;
|
|
4
|
+
/** Default `sha256`. */
|
|
5
|
+
algorithm?: string;
|
|
6
|
+
/** Default `hex`. */
|
|
7
|
+
encoding?: "hex" | "base64";
|
|
8
|
+
/** Prepended to the digest, e.g. `sha256=`. */
|
|
9
|
+
prefix?: string;
|
|
10
|
+
}
|
|
11
|
+
export type WebhookProvider = "stripe" | "github" | "slack" | "shopify" | "standard" | HmacScheme;
|
|
12
|
+
export interface WebhookOptions {
|
|
13
|
+
provider: WebhookProvider;
|
|
14
|
+
/** The signing secret the app is configured with. For `standard` (Svix), `whsec_<base64>` or the raw key. */
|
|
15
|
+
secret: string;
|
|
16
|
+
/** Event type, sent where the provider puts it: `X-GitHub-Event`, `X-Shopify-Topic`. */
|
|
17
|
+
event?: string;
|
|
18
|
+
/** Unix seconds the signature is made for. Default: now. */
|
|
19
|
+
timestamp?: number;
|
|
20
|
+
/** Signed ten minutes ago: an app that rejects replays must refuse it. */
|
|
21
|
+
stale?: boolean;
|
|
22
|
+
/** Signed with a different secret: the app must refuse it. */
|
|
23
|
+
invalidSignature?: boolean;
|
|
24
|
+
/** Message id for `standard` (`webhook-id`). Default: a random `msg_…`. */
|
|
25
|
+
id?: string;
|
|
26
|
+
/** Extra request headers. */
|
|
27
|
+
headers?: Record<string, string>;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* The headers a provider sends with `body`, signed with `secret` the way its
|
|
31
|
+
* SDK verifies them: Stripe's `Stripe-Signature`, GitHub's
|
|
32
|
+
* `X-Hub-Signature-256`, Slack's `v0` signature, Shopify's base64 HMAC and
|
|
33
|
+
* Standard Webhooks (Svix, Resend, Clerk, …).
|
|
34
|
+
*/
|
|
35
|
+
export declare function signWebhook(body: string, opts: WebhookOptions): Record<string, string>;
|
|
36
|
+
/** The bytes to send and their content type: objects as JSON, URLSearchParams as a form (Slack commands), strings as-is. */
|
|
37
|
+
export declare function webhookBody(payload: unknown): {
|
|
38
|
+
body: string;
|
|
39
|
+
type: string;
|
|
40
|
+
};
|