socketduct 0.1.25 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/changelog.d/20261010130509-bound-log-frame-compression-memory.md +1 -0
- package/docs/runbooks/jsonl-integrity-rollout.md +8 -6
- package/package.json +2 -2
- package/scripts/test.js +9 -14
- package/src/logging/compressed-log-writer.js +1 -1
- package/src/logging/zstd-frame-codec.js +15 -1
- package/test/e2e/gateway-process.test.js +93 -92
- package/test/e2e/logger-shutdown-process.test.js +40 -37
- package/test/e2e/reverse-client-recovery-process.test.js +107 -103
- package/test/e2e/reverse-gateway-process.test.js +328 -318
- package/test/integration/initial-target-recovery.test.js +333 -334
- package/test/integration/logging-cli.test.js +121 -119
- package/test/integration/logging-compressed-recovery.test.js +96 -95
- package/test/integration/multiplexed-compatibility.test.js +178 -177
- package/test/integration/multiplexed-corrections.test.js +360 -359
- package/test/integration/multiplexed-failures.test.js +253 -251
- package/test/integration/multiplexed-transport-set.test.js +334 -322
- package/test/integration/multiplexed-transport.test.js +135 -134
- package/test/integration/reverse-gateway-stream-id-reset.test.js +90 -89
- package/test/integration/reverse-gateway.test.js +2309 -2218
- package/test/integration/reverse-resume-continuity.test.js +219 -218
- package/test/integration/reverse-route-class.test.js +358 -341
- package/test/integration/socketduct-http-agent.test.js +314 -312
- package/test/integration/socketduct-https-agent.test.js +516 -485
- package/test/integration/stream-compression.test.js +862 -824
- package/test/integration/tcp-forwarding.test.js +2550 -2447
- package/test/integration/tls-forwarding.test.js +593 -592
- package/test/unit/auth/auth.test.js +43 -44
- package/test/unit/auth/static-token-store.test.js +9 -8
- package/test/unit/client/connect.test.js +41 -40
- package/test/unit/client/deferred-virtual-socket.test.js +95 -94
- package/test/unit/client/multiplexed-spool-budget.test.js +155 -154
- package/test/unit/client/multiplexed-transport-set.test.js +830 -748
- package/test/unit/client/socketduct-socket.test.js +768 -771
- package/test/unit/client/virtual-socket.test.js +198 -168
- package/test/unit/config/docker-gateway-files.test.js +224 -224
- package/test/unit/config/environment.test.js +472 -492
- package/test/unit/index.test.js +53 -52
- package/test/unit/logging/compressed-log-worker.test.js +23 -22
- package/test/unit/logging/compressed-log-writer.test.js +155 -154
- package/test/unit/logging/log-retention.test.js +52 -47
- package/test/unit/logging/log-segment-index.test.js +84 -81
- package/test/unit/logging/zstd-frame-codec.test.js +28 -19
- package/test/unit/protocol/compression.test.js +254 -298
- package/test/unit/protocol/frame.test.js +280 -276
- package/test/unit/protocol/v3-compression.test.js +21 -23
- package/test/unit/protocol/v3-frame.test.js +42 -41
- package/test/unit/release-configuration.test.js +15 -12
- package/test/unit/reverse/disconnect-attribution.test.js +65 -64
- package/test/unit/reverse-terminal-outcomes.test.js +53 -50
- package/test/unit/server/control-handler.test.js +232 -231
- package/test/unit/server/primary.test.js +51 -49
- package/test/unit/server/relay-runtime.test.js +280 -294
- package/test/unit/sessions/session.test.js +48 -62
- package/test/unit/spool/spool-file.test.js +401 -385
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
|
|
2
|
-
import
|
|
2
|
+
import {describe, expect, it, withTestCleanup} from "@velocious/testing"
|
|
3
3
|
import {spawn} from "node:child_process"
|
|
4
4
|
import {mkdtemp, readFile, readdir, rm} from "node:fs/promises"
|
|
5
5
|
import {createServer} from "node:net"
|
|
6
6
|
import {tmpdir} from "node:os"
|
|
7
7
|
import {join} from "node:path"
|
|
8
|
-
import test from "node:test"
|
|
9
8
|
import {connect as connectTls} from "node:tls"
|
|
10
9
|
|
|
11
10
|
import {ReverseGatewayConnection} from "../../src/reverse/client.js"
|
|
@@ -46,7 +45,9 @@ async function listen(server) {
|
|
|
46
45
|
server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
|
|
47
46
|
})
|
|
48
47
|
const address = server.address()
|
|
49
|
-
|
|
48
|
+
if (address === null || typeof address === "string") {
|
|
49
|
+
throw new Error("Server did not bind to a TCP port")
|
|
50
|
+
}
|
|
50
51
|
return address.port
|
|
51
52
|
}
|
|
52
53
|
|
|
@@ -164,335 +165,344 @@ async function onceEvent(emitter, event) {
|
|
|
164
165
|
})
|
|
165
166
|
}
|
|
166
167
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
168
|
+
describe("reverse gateway process", () => {
|
|
169
|
+
it("reserved reverse-hub startup retries a pre-start listen EADDRINUSE without leaking attempts", async () => withTestCleanup(async (after) => {
|
|
170
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-port-retry-"))
|
|
171
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
172
|
+
const certificates = generateTestCertificates(root)
|
|
173
|
+
const occupiedListener = createServer()
|
|
174
|
+
const occupiedPort = await listen(occupiedListener)
|
|
175
|
+
after(() => new Promise((resolve) => occupiedListener.close(() => resolve(undefined))))
|
|
176
|
+
let reservations = 0
|
|
177
|
+
const {child, port, attempts} = await startProcessOnReservedPort({
|
|
178
|
+
SOCKETDUCT_MODE: "reverse-hub",
|
|
179
|
+
SOCKETDUCT_RELAY_HOST: "127.0.0.1",
|
|
180
|
+
SOCKETDUCT_TLS_CERT_FILE: certificates.serverCertFile,
|
|
181
|
+
SOCKETDUCT_TLS_KEY_FILE: certificates.serverKeyFile,
|
|
182
|
+
SOCKETDUCT_AUTH_TOKENS: "home-o2:process-gateway-token,tensorbuzz:process-client-token",
|
|
183
|
+
SOCKETDUCT_REVERSE_GATEWAY_BINDINGS: "o2:home-o2",
|
|
184
|
+
SOCKETDUCT_REVERSE_ROUTES: "tensorbuzz:o2:127.0.0.1:1",
|
|
185
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0"
|
|
186
|
+
}, "reverse-hub.started", {
|
|
187
|
+
reservePort: async () => {
|
|
188
|
+
reservations += 1
|
|
189
|
+
return reservations === 1 ? occupiedPort : reservePort()
|
|
190
|
+
}
|
|
191
|
+
})
|
|
192
|
+
after(() => stopProcess(child))
|
|
191
193
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
assert.equal(occupiedListener.listening, true)
|
|
194
|
+
expect(reservations).toBe(2)
|
|
195
|
+
expect(port).not.toBe(occupiedPort)
|
|
196
|
+
expect(attempts.length).toBe(2)
|
|
197
|
+
const [failedAttempt, successfulAttempt] = attempts
|
|
198
|
+
if (failedAttempt === undefined || successfulAttempt === undefined) throw new Error("Expected two startup attempts")
|
|
199
|
+
expect(failedAttempt.exitCode !== null || failedAttempt.signalCode !== null).toBeTruthy()
|
|
200
|
+
expect(successfulAttempt).toBe(child)
|
|
201
|
+
expect(child.exitCode).toBe(null)
|
|
202
|
+
expect(child.signalCode).toBe(null)
|
|
203
|
+
expect(occupiedListener.listening).toBe(true)
|
|
203
204
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
})
|
|
211
|
-
await new Promise((resolve, reject) => {
|
|
212
|
-
releasedPortProbe.once("error", reject)
|
|
213
|
-
releasedPortProbe.listen(port, "127.0.0.1", () => {
|
|
214
|
-
releasedPortProbe.off("error", reject)
|
|
215
|
-
resolve(undefined)
|
|
205
|
+
await stopProcess(child)
|
|
206
|
+
expect(child.exitCode !== null || child.signalCode !== null).toBeTruthy()
|
|
207
|
+
const releasedPortProbe = createServer()
|
|
208
|
+
after(() => {
|
|
209
|
+
if (!releasedPortProbe.listening) return undefined
|
|
210
|
+
return new Promise((resolve) => releasedPortProbe.close(() => resolve(undefined)))
|
|
216
211
|
})
|
|
217
|
-
|
|
218
|
-
|
|
212
|
+
await new Promise((resolve, reject) => {
|
|
213
|
+
releasedPortProbe.once("error", reject)
|
|
214
|
+
releasedPortProbe.listen(port, "127.0.0.1", () => {
|
|
215
|
+
releasedPortProbe.off("error", reject)
|
|
216
|
+
resolve(undefined)
|
|
217
|
+
})
|
|
218
|
+
})
|
|
219
|
+
}))
|
|
219
220
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
221
|
+
it("reserved process startup does not retry an unclassified EADDRINUSE failure", async () => {
|
|
222
|
+
const failure = new Error("untrusted EADDRINUSE text is not a listener-bind classification")
|
|
223
|
+
let reservations = 0
|
|
224
|
+
let starts = 0
|
|
225
|
+
let error
|
|
226
|
+
try {
|
|
227
|
+
await startProcessOnReservedPort({}, "reverse-hub.started", {
|
|
228
|
+
reservePort: async () => {
|
|
229
|
+
reservations += 1
|
|
230
|
+
return 31_337
|
|
231
|
+
},
|
|
232
|
+
startProcess: async () => {
|
|
233
|
+
starts += 1
|
|
234
|
+
throw failure
|
|
235
|
+
}
|
|
236
|
+
})
|
|
237
|
+
} catch (caught) {
|
|
238
|
+
error = caught
|
|
232
239
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
test("spawned reverse hub and outbound gateway expose an allowlisted Docker stream over verified TLS", async (t) => {
|
|
239
|
-
const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-process-"))
|
|
240
|
-
t.after(() => rm(root, {recursive: true, force: true}))
|
|
241
|
-
const certificates = generateTestCertificates(root)
|
|
242
|
-
const logDirectory = join(root, "logs")
|
|
243
|
-
const targetSockets = new Set()
|
|
244
|
-
const target = createServer((socket) => {
|
|
245
|
-
targetSockets.add(socket)
|
|
246
|
-
socket.once("close", () => targetSockets.delete(socket))
|
|
247
|
-
socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("docker:"), chunk])))
|
|
248
|
-
})
|
|
249
|
-
const targetPort = await listen(target)
|
|
250
|
-
t.after(() => {
|
|
251
|
-
for (const socket of targetSockets) socket.destroy()
|
|
252
|
-
return new Promise((resolve) => target.close(() => resolve(undefined)))
|
|
240
|
+
expect(error).toBeDefined()
|
|
241
|
+
expect(error === failure).toBeTruthy()
|
|
242
|
+
expect(reservations).toBe(1)
|
|
243
|
+
expect(starts).toBe(1)
|
|
253
244
|
})
|
|
254
|
-
const {child: hub, port: hubPort} = await startProcessOnReservedPort({
|
|
255
|
-
SOCKETDUCT_MODE: "reverse-hub",
|
|
256
|
-
SOCKETDUCT_RELAY_HOST: "127.0.0.1",
|
|
257
|
-
SOCKETDUCT_TLS_CERT_FILE: certificates.serverCertFile,
|
|
258
|
-
SOCKETDUCT_TLS_KEY_FILE: certificates.serverKeyFile,
|
|
259
|
-
SOCKETDUCT_AUTH_TOKENS: "home-o2:process-gateway-token,tensorbuzz:process-client-token",
|
|
260
|
-
SOCKETDUCT_REVERSE_GATEWAY_BINDINGS: "o2:home-o2",
|
|
261
|
-
SOCKETDUCT_REVERSE_ROUTES: `tensorbuzz:o2:127.0.0.1:${targetPort}`,
|
|
262
|
-
SOCKETDUCT_STATUS_INTERVAL_MS: "0",
|
|
263
|
-
SOCKETDUCT_LOG_DIRECTORY: logDirectory,
|
|
264
|
-
SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096",
|
|
265
|
-
SOCKETDUCT_CONNECTION_FINGERPRINT_KEY: TEST_FINGERPRINT_KEY,
|
|
266
|
-
SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH: "process-test"
|
|
267
|
-
}, "reverse-hub.started")
|
|
268
|
-
t.after(() => stopProcess(hub))
|
|
269
|
-
const gateway = await startProcess({
|
|
270
|
-
SOCKETDUCT_MODE: "reverse-gateway",
|
|
271
|
-
SOCKETDUCT_REVERSE_GATEWAY_ID: "o2",
|
|
272
|
-
SOCKETDUCT_REVERSE_CONNECTION_ID: "process_home_o2_gateway",
|
|
273
|
-
SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
|
|
274
|
-
SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
|
|
275
|
-
SOCKETDUCT_REVERSE_TOKEN: "process-gateway-token",
|
|
276
|
-
SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
|
|
277
|
-
SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
|
|
278
|
-
SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`,
|
|
279
|
-
SOCKETDUCT_STATUS_INTERVAL_MS: "0",
|
|
280
|
-
SOCKETDUCT_LOG_DIRECTORY: logDirectory,
|
|
281
|
-
SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096"
|
|
282
|
-
}, "reverse-gateway.started")
|
|
283
|
-
t.after(() => stopProcess(gateway))
|
|
284
245
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
246
|
+
it("spawned reverse hub and outbound gateway expose an allowlisted Docker stream over verified TLS", async () => withTestCleanup(async (after) => {
|
|
247
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-process-"))
|
|
248
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
249
|
+
const certificates = generateTestCertificates(root)
|
|
250
|
+
const logDirectory = join(root, "logs")
|
|
251
|
+
const targetSockets = new Set()
|
|
252
|
+
const target = createServer((socket) => {
|
|
253
|
+
targetSockets.add(socket)
|
|
254
|
+
socket.once("close", () => targetSockets.delete(socket))
|
|
255
|
+
socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("docker:"), chunk])))
|
|
256
|
+
})
|
|
257
|
+
const targetPort = await listen(target)
|
|
258
|
+
after(() => {
|
|
259
|
+
for (const socket of targetSockets) socket.destroy()
|
|
260
|
+
return new Promise((resolve) => target.close(() => resolve(undefined)))
|
|
261
|
+
})
|
|
262
|
+
const {child: hub, port: hubPort} = await startProcessOnReservedPort({
|
|
263
|
+
SOCKETDUCT_MODE: "reverse-hub",
|
|
264
|
+
SOCKETDUCT_RELAY_HOST: "127.0.0.1",
|
|
265
|
+
SOCKETDUCT_TLS_CERT_FILE: certificates.serverCertFile,
|
|
266
|
+
SOCKETDUCT_TLS_KEY_FILE: certificates.serverKeyFile,
|
|
267
|
+
SOCKETDUCT_AUTH_TOKENS: "home-o2:process-gateway-token,tensorbuzz:process-client-token",
|
|
268
|
+
SOCKETDUCT_REVERSE_GATEWAY_BINDINGS: "o2:home-o2",
|
|
269
|
+
SOCKETDUCT_REVERSE_ROUTES: `tensorbuzz:o2:127.0.0.1:${targetPort}`,
|
|
270
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0",
|
|
271
|
+
SOCKETDUCT_LOG_DIRECTORY: logDirectory,
|
|
272
|
+
SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096",
|
|
273
|
+
SOCKETDUCT_CONNECTION_FINGERPRINT_KEY: TEST_FINGERPRINT_KEY,
|
|
274
|
+
SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH: "process-test"
|
|
275
|
+
}, "reverse-hub.started")
|
|
276
|
+
after(() => stopProcess(hub))
|
|
277
|
+
const gateway = await startProcess({
|
|
278
|
+
SOCKETDUCT_MODE: "reverse-gateway",
|
|
279
|
+
SOCKETDUCT_REVERSE_GATEWAY_ID: "o2",
|
|
280
|
+
SOCKETDUCT_REVERSE_CONNECTION_ID: "process_home_o2_gateway",
|
|
281
|
+
SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
|
|
282
|
+
SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
|
|
283
|
+
SOCKETDUCT_REVERSE_TOKEN: "process-gateway-token",
|
|
284
|
+
SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
|
|
285
|
+
SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
|
|
286
|
+
SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`,
|
|
287
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0",
|
|
288
|
+
SOCKETDUCT_LOG_DIRECTORY: logDirectory,
|
|
289
|
+
SOCKETDUCT_LOG_SEGMENT_MAX_BYTES: "4096"
|
|
290
|
+
}, "reverse-gateway.started")
|
|
291
|
+
after(() => stopProcess(gateway))
|
|
303
292
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
})
|
|
293
|
+
const client = new ReverseGatewayConnection({
|
|
294
|
+
gatewayId: "o2",
|
|
295
|
+
connectionId: "process_tensorbuzz_client",
|
|
296
|
+
hub: {host: "127.0.0.1", port: hubPort, token: "process-client-token"},
|
|
297
|
+
tls: {caFile: certificates.caCertFile, servername: "localhost"}
|
|
298
|
+
})
|
|
299
|
+
after(() => client.close())
|
|
300
|
+
await withTimeout(client.connect(), "process client connect")
|
|
301
|
+
const socket = client.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "docker-version-process"})
|
|
302
|
+
await withTimeout(onceEvent(socket, "connect"), "process stream open")
|
|
303
|
+
const response = new Promise((resolve, reject) => { socket.once("error", reject); socket.once("data", resolve) })
|
|
304
|
+
socket.write("GET /version")
|
|
305
|
+
expect(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(response, "process response"))).toString()).toBe("docker:GET /version")
|
|
306
|
+
expect(client.status().compression).toBe("zstd")
|
|
307
|
+
socket.destroy()
|
|
308
|
+
client.close()
|
|
309
|
+
await stopProcess(gateway)
|
|
310
|
+
await stopProcess(hub)
|
|
313
311
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
})
|
|
324
|
-
const targetB = createServer((socket) => {
|
|
325
|
-
targetSockets.add(socket)
|
|
326
|
-
socket.once("close", () => targetSockets.delete(socket))
|
|
327
|
-
socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("b:"), chunk])))
|
|
328
|
-
})
|
|
329
|
-
const targetAPort = await listen(targetA)
|
|
330
|
-
const targetBPort = await listen(targetB)
|
|
331
|
-
t.after(() => {
|
|
332
|
-
for (const socket of targetSockets) socket.destroy()
|
|
333
|
-
return Promise.all([
|
|
334
|
-
new Promise((resolve) => targetA.close(() => resolve(undefined))),
|
|
335
|
-
new Promise((resolve) => targetB.close(() => resolve(undefined)))
|
|
336
|
-
])
|
|
337
|
-
})
|
|
312
|
+
const logs = await readCompressedLogDirectory(logDirectory)
|
|
313
|
+
expect(logs.incomplete).toBe(false)
|
|
314
|
+
const hubEntries = logs.records.filter(({role}) => role === "reverse-hub")
|
|
315
|
+
const gatewayEntries = logs.records.filter(({role}) => role === "reverse-gateway")
|
|
316
|
+
expect(hubEntries.some(({event}) => event === "process.started")).toBeTruthy()
|
|
317
|
+
expect(gatewayEntries.some(({event}) => event === "process.started")).toBeTruthy()
|
|
318
|
+
expect(gatewayEntries.some(({event}) => event === "transport.connected")).toBeTruthy()
|
|
319
|
+
expect(JSON.stringify(logs.records)).not.toMatch(/process-gateway-token|127\.0\.0\.1|docker-server|process_home_o2_gateway/)
|
|
320
|
+
}))
|
|
338
321
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
322
|
+
it("one route-class client physical connection interleaves streams through different gateway processes", async () => withTestCleanup(async (after) => {
|
|
323
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-process-"))
|
|
324
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
325
|
+
const certificates = generateTestCertificates(root)
|
|
326
|
+
const targetSockets = new Set()
|
|
327
|
+
const targetA = createServer((socket) => {
|
|
328
|
+
targetSockets.add(socket)
|
|
329
|
+
socket.once("close", () => targetSockets.delete(socket))
|
|
330
|
+
socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("a:"), chunk])))
|
|
331
|
+
})
|
|
332
|
+
const targetB = createServer((socket) => {
|
|
333
|
+
targetSockets.add(socket)
|
|
334
|
+
socket.once("close", () => targetSockets.delete(socket))
|
|
335
|
+
socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("b:"), chunk])))
|
|
336
|
+
})
|
|
337
|
+
const targetAPort = await listen(targetA)
|
|
338
|
+
const targetBPort = await listen(targetB)
|
|
339
|
+
after(() => {
|
|
340
|
+
for (const socket of targetSockets) socket.destroy()
|
|
341
|
+
return Promise.all([
|
|
342
|
+
new Promise((resolve) => targetA.close(() => resolve(undefined))),
|
|
343
|
+
new Promise((resolve) => targetB.close(() => resolve(undefined)))
|
|
344
|
+
]).then(() => undefined)
|
|
345
|
+
})
|
|
350
346
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
}, "reverse-gateway.started")
|
|
363
|
-
t.after(() => stopProcess(gatewayA))
|
|
364
|
-
const gatewayB = await startProcess({
|
|
365
|
-
SOCKETDUCT_MODE: "reverse-gateway",
|
|
366
|
-
SOCKETDUCT_REVERSE_GATEWAY_ID: "server-b-starlink",
|
|
367
|
-
SOCKETDUCT_REVERSE_CONNECTION_ID: "route_class_process_gateway_b",
|
|
368
|
-
SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
|
|
369
|
-
SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
|
|
370
|
-
SOCKETDUCT_REVERSE_TOKEN: "gateway-b-token",
|
|
371
|
-
SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
|
|
372
|
-
SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
|
|
373
|
-
SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetBPort}`,
|
|
374
|
-
SOCKETDUCT_STATUS_INTERVAL_MS: "0"
|
|
375
|
-
}, "reverse-gateway.started")
|
|
376
|
-
t.after(() => stopProcess(gatewayB))
|
|
347
|
+
const {child: hub, port: hubPort} = await startProcessOnReservedPort({
|
|
348
|
+
SOCKETDUCT_MODE: "reverse-hub",
|
|
349
|
+
SOCKETDUCT_RELAY_HOST: "127.0.0.1",
|
|
350
|
+
SOCKETDUCT_TLS_CERT_FILE: certificates.serverCertFile,
|
|
351
|
+
SOCKETDUCT_TLS_KEY_FILE: certificates.serverKeyFile,
|
|
352
|
+
SOCKETDUCT_AUTH_TOKENS: "home-a:gateway-a-token,home-b:gateway-b-token,tensorbuzz:client-token",
|
|
353
|
+
SOCKETDUCT_REVERSE_GATEWAY_BINDINGS: "server-a-starlink:home-a,server-b-starlink:home-b",
|
|
354
|
+
SOCKETDUCT_REVERSE_ROUTE_CLASS_ROUTES: `tensorbuzz:starlink:server-a-starlink:127.0.0.1:${targetAPort},tensorbuzz:starlink:server-b-starlink:127.0.0.1:${targetBPort}`,
|
|
355
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0"
|
|
356
|
+
}, "reverse-hub.started")
|
|
357
|
+
after(() => stopProcess(hub))
|
|
377
358
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
})
|
|
359
|
+
const gatewayA = await startProcess({
|
|
360
|
+
SOCKETDUCT_MODE: "reverse-gateway",
|
|
361
|
+
SOCKETDUCT_REVERSE_GATEWAY_ID: "server-a-starlink",
|
|
362
|
+
SOCKETDUCT_REVERSE_CONNECTION_ID: "route_class_process_gateway_a",
|
|
363
|
+
SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
|
|
364
|
+
SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
|
|
365
|
+
SOCKETDUCT_REVERSE_TOKEN: "gateway-a-token",
|
|
366
|
+
SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
|
|
367
|
+
SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
|
|
368
|
+
SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetAPort}`,
|
|
369
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0"
|
|
370
|
+
}, "reverse-gateway.started")
|
|
371
|
+
after(() => stopProcess(gatewayA))
|
|
372
|
+
const gatewayB = await startProcess({
|
|
373
|
+
SOCKETDUCT_MODE: "reverse-gateway",
|
|
374
|
+
SOCKETDUCT_REVERSE_GATEWAY_ID: "server-b-starlink",
|
|
375
|
+
SOCKETDUCT_REVERSE_CONNECTION_ID: "route_class_process_gateway_b",
|
|
376
|
+
SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
|
|
377
|
+
SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
|
|
378
|
+
SOCKETDUCT_REVERSE_TOKEN: "gateway-b-token",
|
|
379
|
+
SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
|
|
380
|
+
SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
|
|
381
|
+
SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetBPort}`,
|
|
382
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0"
|
|
383
|
+
}, "reverse-gateway.started")
|
|
384
|
+
after(() => stopProcess(gatewayB))
|
|
405
385
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
SOCKETDUCT_REVERSE_ROUTES: `tensorbuzz:o2:127.0.0.1:${targetPort}`,
|
|
434
|
-
SOCKETDUCT_STATUS_INTERVAL_MS: "0"
|
|
435
|
-
}, "reverse-hub.started")
|
|
436
|
-
const gateway = await startProcess({
|
|
437
|
-
SOCKETDUCT_MODE: "reverse-gateway",
|
|
438
|
-
SOCKETDUCT_REVERSE_GATEWAY_ID: "o2",
|
|
439
|
-
SOCKETDUCT_REVERSE_CONNECTION_ID: "spool_process_gateway",
|
|
440
|
-
SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
|
|
441
|
-
SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
|
|
442
|
-
SOCKETDUCT_REVERSE_TOKEN: "spool-gateway-token",
|
|
443
|
-
SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
|
|
444
|
-
SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
|
|
445
|
-
SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`,
|
|
446
|
-
SOCKETDUCT_STATUS_INTERVAL_MS: "0",
|
|
447
|
-
SOCKETDUCT_SPOOL_DIRECTORY: gatewaySpoolDirectory
|
|
448
|
-
}, "reverse-gateway.started")
|
|
449
|
-
t.after(() => stopProcess(gateway))
|
|
450
|
-
const client = new ReverseGatewayConnection({
|
|
451
|
-
gatewayId: "o2",
|
|
452
|
-
connectionId: "spool_process_client",
|
|
453
|
-
hub: {host: "127.0.0.1", port: hubPort, token: "spool-client-token"},
|
|
454
|
-
tls: {caFile: certificates.caCertFile, servername: "localhost"},
|
|
455
|
-
spoolDirectory: clientSpoolDirectory,
|
|
456
|
-
reconnectDelayMs: 50,
|
|
457
|
-
maxReconnectDelayMs: 100
|
|
458
|
-
})
|
|
459
|
-
t.after(() => client.close())
|
|
460
|
-
await withTimeout(client.connect(), "spool client connect")
|
|
461
|
-
const socket = client.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "docker-spool-process"})
|
|
462
|
-
await withTimeout(onceEvent(socket, "connect"), "spool stream open")
|
|
463
|
-
const initialResponse = new Promise((resolve, reject) => { socket.once("error", reject); socket.once("data", resolve) })
|
|
464
|
-
socket.write("GET /version")
|
|
465
|
-
assert.equal(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(initialResponse, "spool initial response"))).toString(), "docker:GET /version")
|
|
386
|
+
let clientPhysicalConnections = 0
|
|
387
|
+
const client = new ReverseGatewayConnection({
|
|
388
|
+
routeClass: "starlink",
|
|
389
|
+
connectionId: "route_class_process_client",
|
|
390
|
+
hub: {host: "127.0.0.1", port: hubPort, token: "client-token"},
|
|
391
|
+
tls: {caFile: certificates.caCertFile, servername: "localhost"},
|
|
392
|
+
createPhysicalSocket: ({host, port, tls}) => {
|
|
393
|
+
clientPhysicalConnections += 1
|
|
394
|
+
return connectTls({host, port, ...tls})
|
|
395
|
+
}
|
|
396
|
+
})
|
|
397
|
+
after(() => client.close())
|
|
398
|
+
await withTimeout(client.connect(), "route-class process client connect")
|
|
399
|
+
const streamA = client.createConnection({host: "127.0.0.1", port: targetAPort, sessionName: "route-class-a"})
|
|
400
|
+
const streamB = client.createConnection({host: "127.0.0.1", port: targetBPort, sessionName: "route-class-b"})
|
|
401
|
+
await withTimeout(Promise.all([onceEvent(streamA, "connect"), onceEvent(streamB, "connect")]), "route-class process streams")
|
|
402
|
+
const responseA = new Promise((resolve, reject) => { streamA.once("error", reject); streamA.once("data", resolve) })
|
|
403
|
+
const responseB = new Promise((resolve, reject) => { streamB.once("error", reject); streamB.once("data", resolve) })
|
|
404
|
+
streamA.write("one")
|
|
405
|
+
streamB.write("two")
|
|
406
|
+
expect(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(responseA, "route-class response A"))).toString()).toBe("a:one")
|
|
407
|
+
expect(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(responseB, "route-class response B"))).toString()).toBe("b:two")
|
|
408
|
+
expect(clientPhysicalConnections).toBe(1)
|
|
409
|
+
expect(client.status().physicalConnects).toBe(1)
|
|
410
|
+
streamA.destroy()
|
|
411
|
+
streamB.destroy()
|
|
412
|
+
}))
|
|
466
413
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
414
|
+
it("outbound gateway process spools durably under SOCKETDUCT_SPOOL_DIRECTORY when the hub link is lost", async () => withTestCleanup(async (after) => {
|
|
415
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-spool-process-"))
|
|
416
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
417
|
+
const certificates = generateTestCertificates(root)
|
|
418
|
+
const gatewaySpoolDirectory = join(root, "gateway-spool")
|
|
419
|
+
const clientSpoolDirectory = join(root, "client-spool")
|
|
420
|
+
/** @type {import('node:net').Socket | undefined} */
|
|
421
|
+
let targetSocket
|
|
422
|
+
const targetSockets = new Set()
|
|
423
|
+
const target = createServer((socket) => {
|
|
424
|
+
targetSocket = socket
|
|
425
|
+
targetSockets.add(socket)
|
|
426
|
+
socket.once("close", () => targetSockets.delete(socket))
|
|
427
|
+
socket.on("data", (chunk) => socket.write(Buffer.concat([Buffer.from("docker:"), chunk])))
|
|
428
|
+
})
|
|
429
|
+
const targetPort = await listen(target)
|
|
430
|
+
after(() => {
|
|
431
|
+
for (const socket of targetSockets) socket.destroy()
|
|
432
|
+
return new Promise((resolve) => target.close(() => resolve(undefined)))
|
|
433
|
+
})
|
|
434
|
+
const {child: hub, port: hubPort} = await startProcessOnReservedPort({
|
|
435
|
+
SOCKETDUCT_MODE: "reverse-hub",
|
|
436
|
+
SOCKETDUCT_RELAY_HOST: "127.0.0.1",
|
|
437
|
+
SOCKETDUCT_TLS_CERT_FILE: certificates.serverCertFile,
|
|
438
|
+
SOCKETDUCT_TLS_KEY_FILE: certificates.serverKeyFile,
|
|
439
|
+
SOCKETDUCT_AUTH_TOKENS: "home-o2:spool-gateway-token,tensorbuzz:spool-client-token",
|
|
440
|
+
SOCKETDUCT_REVERSE_GATEWAY_BINDINGS: "o2:home-o2",
|
|
441
|
+
SOCKETDUCT_REVERSE_ROUTES: `tensorbuzz:o2:127.0.0.1:${targetPort}`,
|
|
442
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0"
|
|
443
|
+
}, "reverse-hub.started")
|
|
444
|
+
const gateway = await startProcess({
|
|
445
|
+
SOCKETDUCT_MODE: "reverse-gateway",
|
|
446
|
+
SOCKETDUCT_REVERSE_GATEWAY_ID: "o2",
|
|
447
|
+
SOCKETDUCT_REVERSE_CONNECTION_ID: "spool_process_gateway",
|
|
448
|
+
SOCKETDUCT_REVERSE_HUB_HOST: "127.0.0.1",
|
|
449
|
+
SOCKETDUCT_REVERSE_HUB_PORT: String(hubPort),
|
|
450
|
+
SOCKETDUCT_REVERSE_TOKEN: "spool-gateway-token",
|
|
451
|
+
SOCKETDUCT_REVERSE_HUB_CA_FILE: certificates.caCertFile,
|
|
452
|
+
SOCKETDUCT_REVERSE_HUB_SERVERNAME: "localhost",
|
|
453
|
+
SOCKETDUCT_TARGET_ALLOWLIST: `127.0.0.1:${targetPort}`,
|
|
454
|
+
SOCKETDUCT_STATUS_INTERVAL_MS: "0",
|
|
455
|
+
SOCKETDUCT_SPOOL_DIRECTORY: gatewaySpoolDirectory
|
|
456
|
+
}, "reverse-gateway.started")
|
|
457
|
+
after(() => stopProcess(gateway))
|
|
458
|
+
const client = new ReverseGatewayConnection({
|
|
459
|
+
gatewayId: "o2",
|
|
460
|
+
connectionId: "spool_process_client",
|
|
461
|
+
hub: {host: "127.0.0.1", port: hubPort, token: "spool-client-token"},
|
|
462
|
+
tls: {caFile: certificates.caCertFile, servername: "localhost"},
|
|
463
|
+
spoolDirectory: clientSpoolDirectory,
|
|
464
|
+
reconnectDelayMs: 50,
|
|
465
|
+
maxReconnectDelayMs: 100
|
|
466
|
+
})
|
|
467
|
+
after(() => client.close())
|
|
468
|
+
await withTimeout(client.connect(), "spool client connect")
|
|
469
|
+
const socket = client.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "docker-spool-process"})
|
|
470
|
+
await withTimeout(onceEvent(socket, "connect"), "spool stream open")
|
|
471
|
+
const initialResponse = new Promise((resolve, reject) => { socket.once("error", reject); socket.once("data", resolve) })
|
|
472
|
+
socket.write("GET /version")
|
|
473
|
+
expect(Buffer.from(/** @type {Uint8Array} */ (await withTimeout(initialResponse, "spool initial response"))).toString()).toBe("docker:GET /version")
|
|
474
|
+
|
|
475
|
+
hub.kill("SIGKILL")
|
|
476
|
+
await withTimeout(new Promise((resolve) => hub.once("exit", () => resolve(undefined))), "hub exit")
|
|
477
|
+
if (targetSocket === undefined) throw new Error("Expected a live target socket")
|
|
478
|
+
const spooledMarker = Buffer.alloc(32, 0x61)
|
|
479
|
+
targetSocket.write(spooledMarker)
|
|
480
|
+
const spoolHasPayload = async () => {
|
|
481
|
+
const markerBase64 = spooledMarker.toString("base64")
|
|
482
|
+
for (const session of await readDirectoryOrEmpty(gatewaySpoolDirectory)) {
|
|
483
|
+
const files = await readDirectoryOrEmpty(join(gatewaySpoolDirectory, session))
|
|
484
|
+
if (!files.includes("chunks.jsonl")) continue
|
|
485
|
+
const lines = (await readFile(join(gatewaySpoolDirectory, session, "chunks.jsonl"), "utf8")).split("\n")
|
|
486
|
+
if (lines.some((line) => {
|
|
487
|
+
if (line === "") return false
|
|
488
|
+
try {
|
|
489
|
+
return /** @type {{dataBase64?: unknown}} */ (JSON.parse(line)).dataBase64 === markerBase64
|
|
490
|
+
} catch (error) {
|
|
491
|
+
if (error instanceof SyntaxError) return false
|
|
492
|
+
throw error
|
|
493
|
+
}
|
|
494
|
+
})) return true
|
|
495
|
+
}
|
|
496
|
+
return false
|
|
487
497
|
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
assert.equal(defaultSpooled, false, "spool must not fall back to the ephemeral default root")
|
|
498
|
+
await waitUntil(spoolHasPayload, "durable spool chunk under SOCKETDUCT_SPOOL_DIRECTORY")
|
|
499
|
+
const defaultGatewaySpoolRoot = join(tmpdir(), `socketduct-${typeof process.getuid === "function" ? process.getuid() : "user"}`, "reverse-gateway")
|
|
500
|
+
const defaultSpooled = (await readDirectoryOrEmpty(defaultGatewaySpoolRoot)).some((entry) => entry.startsWith("o2-spool_process_gateway"))
|
|
501
|
+
// Spool must not fall back to the ephemeral default root.
|
|
502
|
+
expect(defaultSpooled).toBe(false)
|
|
494
503
|
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
504
|
+
socket.destroy()
|
|
505
|
+
client.close()
|
|
506
|
+
await stopProcess(gateway)
|
|
507
|
+
}))
|
|
498
508
|
})
|