socketduct 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/changelog.d/20261010130509-bound-log-frame-compression-memory.md +1 -0
- package/docs/runbooks/jsonl-integrity-rollout.md +8 -6
- package/package.json +2 -2
- package/scripts/test.js +9 -14
- package/src/client/multiplexed-transport-set.js +20 -2
- package/src/logging/compressed-log-writer.js +1 -1
- package/src/logging/zstd-frame-codec.js +15 -1
- package/src/reverse/client.js +44 -1
- package/src/reverse/hub.js +10 -1
- package/src/reverse/runtime.js +11 -2
- package/test/e2e/gateway-process.test.js +93 -92
- package/test/e2e/logger-shutdown-process.test.js +40 -37
- package/test/e2e/reverse-client-recovery-process.test.js +107 -103
- package/test/e2e/reverse-gateway-process.test.js +328 -318
- package/test/integration/initial-target-recovery.test.js +333 -334
- package/test/integration/logging-cli.test.js +121 -119
- package/test/integration/logging-compressed-recovery.test.js +96 -95
- package/test/integration/multiplexed-compatibility.test.js +178 -177
- package/test/integration/multiplexed-corrections.test.js +360 -359
- package/test/integration/multiplexed-failures.test.js +253 -251
- package/test/integration/multiplexed-transport-set.test.js +334 -322
- package/test/integration/multiplexed-transport.test.js +135 -134
- package/test/integration/reverse-gateway-stream-id-reset.test.js +90 -89
- package/test/integration/reverse-gateway.test.js +2309 -2218
- package/test/integration/reverse-resume-continuity.test.js +219 -218
- package/test/integration/reverse-route-class.test.js +363 -278
- package/test/integration/socketduct-http-agent.test.js +314 -312
- package/test/integration/socketduct-https-agent.test.js +516 -485
- package/test/integration/stream-compression.test.js +862 -824
- package/test/integration/tcp-forwarding.test.js +2550 -2447
- package/test/integration/tls-forwarding.test.js +593 -592
- package/test/unit/auth/auth.test.js +43 -44
- package/test/unit/auth/static-token-store.test.js +9 -8
- package/test/unit/client/connect.test.js +41 -40
- package/test/unit/client/deferred-virtual-socket.test.js +95 -94
- package/test/unit/client/multiplexed-spool-budget.test.js +155 -154
- package/test/unit/client/multiplexed-transport-set.test.js +845 -732
- package/test/unit/client/socketduct-socket.test.js +768 -771
- package/test/unit/client/virtual-socket.test.js +198 -168
- package/test/unit/config/docker-gateway-files.test.js +224 -224
- package/test/unit/config/environment.test.js +472 -492
- package/test/unit/index.test.js +53 -52
- package/test/unit/logging/compressed-log-worker.test.js +23 -22
- package/test/unit/logging/compressed-log-writer.test.js +155 -154
- package/test/unit/logging/log-retention.test.js +52 -47
- package/test/unit/logging/log-segment-index.test.js +84 -81
- package/test/unit/logging/zstd-frame-codec.test.js +28 -19
- package/test/unit/protocol/compression.test.js +254 -298
- package/test/unit/protocol/frame.test.js +280 -276
- package/test/unit/protocol/v3-compression.test.js +21 -23
- package/test/unit/protocol/v3-frame.test.js +42 -41
- package/test/unit/release-configuration.test.js +15 -12
- package/test/unit/reverse/disconnect-attribution.test.js +65 -64
- package/test/unit/reverse-terminal-outcomes.test.js +53 -50
- package/test/unit/server/control-handler.test.js +232 -231
- package/test/unit/server/primary.test.js +51 -49
- package/test/unit/server/relay-runtime.test.js +280 -294
- package/test/unit/sessions/session.test.js +48 -62
- package/test/unit/spool/spool-file.test.js +401 -385
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/no-undefined-types, jsdoc/require-param, jsdoc/require-returns */
|
|
2
|
-
import
|
|
2
|
+
import {describe, expect, it} from "@velocious/testing"
|
|
3
3
|
import {EventEmitter} from "node:events"
|
|
4
4
|
import {join} from "node:path"
|
|
5
5
|
import {Duplex} from "node:stream"
|
|
6
|
-
import test from "node:test"
|
|
7
6
|
|
|
8
7
|
import {
|
|
9
8
|
MultiplexedSocketductTransportSet,
|
|
@@ -71,6 +70,7 @@ class FakeMember extends EventEmitter {
|
|
|
71
70
|
this.state = "configured"
|
|
72
71
|
this.generation = 0
|
|
73
72
|
this.physicalConnects = 0
|
|
73
|
+
this.gatewayUnavailable = false
|
|
74
74
|
/** @type {Promise<void>} */
|
|
75
75
|
this.connectPromise = new Promise((resolve, reject) => {
|
|
76
76
|
this.resolveConnect = resolve
|
|
@@ -135,6 +135,7 @@ class FakeMember extends EventEmitter {
|
|
|
135
135
|
status() {
|
|
136
136
|
return {
|
|
137
137
|
physicalState: this.state,
|
|
138
|
+
gatewayUnavailable: this.gatewayUnavailable,
|
|
138
139
|
connectionGeneration: this.generation,
|
|
139
140
|
physicalConnects: this.physicalConnects,
|
|
140
141
|
reconnectAttempts: this.state === "reconnecting" ? 1 : 0,
|
|
@@ -206,11 +207,11 @@ function gatewayOf(socket) {
|
|
|
206
207
|
return /** @type {{gatewayId: string}} */ (socket).gatewayId
|
|
207
208
|
}
|
|
208
209
|
|
|
209
|
-
/** @template T @param {T
|
|
210
|
+
/** @template T @param {Array<T | undefined>} values - Values. @param {number} index - Index. @returns {T} */
|
|
210
211
|
function at(values, index) {
|
|
211
212
|
const value = values[index]
|
|
212
|
-
|
|
213
|
-
return value
|
|
213
|
+
expect(value).toBeDefined()
|
|
214
|
+
return /** @type {T} */ (value)
|
|
214
215
|
}
|
|
215
216
|
|
|
216
217
|
/** @param {EventEmitter} emitter - Source. @param {string} event - Event. @returns {Promise<unknown[]>} */
|
|
@@ -222,785 +223,897 @@ function onceEvent(emitter, event) {
|
|
|
222
223
|
function memberPairs(members) {
|
|
223
224
|
return members.map((member) => {
|
|
224
225
|
const {connectionId, spoolDirectory} = /** @type {import('../../../src/client/multiplexed-connection.js').MultiplexedConnectionOptions} */ (member.options)
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
226
|
+
expect(connectionId).toEqual(expect.any(String))
|
|
227
|
+
expect(spoolDirectory).toEqual(expect.any(String))
|
|
228
|
+
if (connectionId === undefined || spoolDirectory === undefined) throw new Error("expected connectionId and spoolDirectory to be defined")
|
|
229
|
+
return /** @type {[string, string]} */ ([connectionId, spoolDirectory])
|
|
228
230
|
})
|
|
229
231
|
}
|
|
230
232
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
}
|
|
233
|
+
describe("multiplexed transport set", () => {
|
|
234
|
+
it("validates stable gateway identities, pinned endpoints, and process limits", () => {
|
|
235
|
+
for (const gatewayId of ["", " ../wan ", "wan/../../escape", "a", "x".repeat(65)]) {
|
|
236
|
+
expect(() => new MultiplexedSocketductTransportSet(options({
|
|
237
|
+
gateways: [{gatewayId, endpoint: {relay: {host: "127.0.0.1", port: 3101, token: "secret"}}}]
|
|
238
|
+
}))).toThrow(/gatewayId/i)
|
|
239
|
+
}
|
|
240
|
+
expect(() => new MultiplexedSocketductTransportSet(options({
|
|
241
|
+
gateways: [
|
|
242
|
+
{gatewayId: "duplicate", endpoint: {relay: {host: "127.0.0.1", port: 3101, token: "secret"}}},
|
|
243
|
+
{gatewayId: "duplicate", endpoint: {relay: {host: "127.0.0.1", port: 3102, token: "secret"}}}
|
|
244
|
+
]
|
|
245
|
+
}))).toThrow(/duplicate gatewayId/i)
|
|
246
|
+
expect(() => new MultiplexedSocketductTransportSet(options({
|
|
247
|
+
// @ts-expect-error Deliberately malformed entry must fail at runtime.
|
|
248
|
+
gateways: [{gatewayId: "starlink", relay: {host: "127.0.0.1", port: 3101, token: "secret"}}]
|
|
249
|
+
}))).toThrow(/endpoint/i)
|
|
250
|
+
expect(() => new MultiplexedSocketductTransportSet(options({
|
|
251
|
+
// @ts-expect-error Deliberately malformed port must fail at runtime.
|
|
252
|
+
gateways: [{gatewayId: "starlink", endpoint: {relay: {host: "127.0.0.1", port: "3101", token: "secret"}}}]
|
|
253
|
+
}))).toThrow(/host, port, and token/i)
|
|
254
|
+
expect(() => new MultiplexedSocketductTransportSet(options({
|
|
255
|
+
gateways: [{
|
|
256
|
+
gatewayId: "starlink",
|
|
257
|
+
// @ts-expect-error Misspelled pinning factory must fail instead of being ignored.
|
|
258
|
+
endpoint: {relay: {host: "127.0.0.1", port: 3101, token: "secret"}, physicalSocketFactory: () => {}}
|
|
259
|
+
}]
|
|
260
|
+
}))).toThrow(/unknown endpoint field/i)
|
|
261
|
+
expect(() => new MultiplexedSocketductTransportSet(options({
|
|
262
|
+
gateways: [
|
|
263
|
+
{gatewayId: "starlink", endpoint: {relay: {host: "relay.example", port: 3101, token: "secret"}}},
|
|
264
|
+
{gatewayId: "zyxel", endpoint: {relay: {host: "relay.example", port: 3101, token: "secret"}}}
|
|
265
|
+
]
|
|
266
|
+
}))).toThrow(/same unpinned endpoint/i)
|
|
267
|
+
expect(() => new MultiplexedSocketductTransportSet(options({minimumReadyMembers: 4}))).toThrow(/minimumReadyMembers/i)
|
|
268
|
+
for (const name of ["maxTotalBufferBytes", "maxTotalSpoolBytes", "maxTotalQueuedBytes", "maxPendingOpens"]) {
|
|
269
|
+
expect(() => new MultiplexedSocketductTransportSet(options({[name]: "1024"}))).toThrow(new RegExp(name))
|
|
270
|
+
}
|
|
271
|
+
})
|
|
269
272
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
273
|
+
it("route-class members are first-class reverse transports with route-class allocation metadata", async () => {
|
|
274
|
+
/** @type {FakeMember[]} */
|
|
275
|
+
const members = []
|
|
276
|
+
const set = new MultiplexedSocketductTransportSet(/** @type {import('../../../src/client/multiplexed-transport-set.js').TransportSetOptions} */ (/** @type {unknown} */ ({
|
|
277
|
+
connectionId: "route_class_process_identity",
|
|
278
|
+
minimumReadyMembers: 1,
|
|
279
|
+
routeClasses: [
|
|
280
|
+
{routeClass: "starlink", endpoint: {reverseHub: {host: "hub.example", port: 3100, token: "starlink-token"}, tls: {servername: "hub.example"}}},
|
|
281
|
+
{routeClass: "zyxel", endpoint: {reverseHub: {host: "hub.example", port: 3100, token: "zyxel-token"}, tls: {servername: "hub.example"}}}
|
|
282
|
+
]
|
|
283
|
+
})), {
|
|
284
|
+
createMember(memberOptions, routeClass) {
|
|
285
|
+
const member = new FakeMember(memberOptions, routeClass, undefined)
|
|
286
|
+
members.push(member)
|
|
287
|
+
return member
|
|
288
|
+
}
|
|
289
|
+
})
|
|
290
|
+
|
|
291
|
+
expect(members.map((member) => {
|
|
292
|
+
const memberOptions = /** @type {import('../../../src/reverse/client.js').ReverseGatewayConnectionOptions} */ (member.options)
|
|
293
|
+
return {gatewayId: memberOptions.gatewayId, routeClass: memberOptions.routeClass}
|
|
294
|
+
})).toEqual([
|
|
295
|
+
{gatewayId: undefined, routeClass: "starlink"},
|
|
296
|
+
{gatewayId: undefined, routeClass: "zyxel"}
|
|
297
|
+
])
|
|
298
|
+
const connectionIds = members.map((member) => member.options.connectionId)
|
|
299
|
+
expect(connectionIds.every((connectionId) => typeof connectionId === "string")).toBeTruthy()
|
|
300
|
+
expect(new Set(connectionIds).size).toBe(2)
|
|
301
|
+
|
|
302
|
+
let notFoundError
|
|
303
|
+
try {
|
|
304
|
+
set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet["recoverConnection"]>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "route_stream_01"})))
|
|
305
|
+
} catch (error) {
|
|
306
|
+
notFoundError = /** @type {Error} */ (error)
|
|
285
307
|
}
|
|
308
|
+
expect(notFoundError).toBeInstanceOf(Error)
|
|
309
|
+
expect(/** @type {{code?: string}} */ (notFoundError).code).toBe("SOCKETDUCT_RECOVERY_NOT_FOUND")
|
|
310
|
+
expect(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet["recoverConnection"]>[0]} */ (/** @type {unknown} */ ({gatewayId: "starlink", routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01"})))).toThrow(/at most one.*gatewayId.*routeClass/i)
|
|
311
|
+
expect(() => set.recoverConnection({routeClass: "missing", host: "target.internal", port: 443, identifier: "route_stream_01"})).toThrow(/unknown routeClass/i)
|
|
312
|
+
const recovered = set.recoverConnection({routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01", sessionName: "durable-route"})
|
|
313
|
+
expect(/** @type {{gatewayId?: string}} */ (recovered).gatewayId).toBe(undefined)
|
|
314
|
+
expect(/** @type {{routeClass?: string}} */ (recovered).routeClass).toBe("zyxel")
|
|
315
|
+
expect(Object.getOwnPropertyDescriptor(recovered, "routeClass")?.writable).toBe(false)
|
|
316
|
+
expect(members.map((member) => member.opens)).toEqual([0, 1])
|
|
317
|
+
expect(/** @type {{recover?: unknown}} */ (at(members, 1).recoveryConnectionCalls[0]).recover).toBe(true)
|
|
318
|
+
expect(Object.hasOwn(/** @type {object} */ (at(members, 1).recoveryConnectionCalls[0]), "routeClass")).toBe(false)
|
|
319
|
+
|
|
320
|
+
const started = set.start()
|
|
321
|
+
at(members, 0).ready()
|
|
322
|
+
await started
|
|
323
|
+
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
324
|
+
expect(/** @type {{gatewayId?: string}} */ (socket).gatewayId).toBe(undefined)
|
|
325
|
+
expect(/** @type {{routeClass?: string}} */ (socket).routeClass).toBe("starlink")
|
|
326
|
+
expect(set.status().members.map((member) => ({gatewayId: member.gatewayId, routeClass: member.routeClass}))).toEqual([
|
|
327
|
+
{gatewayId: undefined, routeClass: "starlink"},
|
|
328
|
+
{gatewayId: undefined, routeClass: "zyxel"}
|
|
329
|
+
])
|
|
330
|
+
await set.close()
|
|
286
331
|
})
|
|
287
332
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
{
|
|
293
|
-
|
|
294
|
-
])
|
|
295
|
-
const connectionIds = members.map((member) => member.options.connectionId)
|
|
296
|
-
assert(connectionIds.every((connectionId) => typeof connectionId === "string"))
|
|
297
|
-
assert.equal(new Set(connectionIds).size, 2)
|
|
298
|
-
|
|
299
|
-
assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "route_stream_01"}))), (error) => /** @type {{code?: string}} */ (error).code === "SOCKETDUCT_RECOVERY_NOT_FOUND")
|
|
300
|
-
assert.throws(() => set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet['recoverConnection']>[0]} */ (/** @type {unknown} */ ({gatewayId: "starlink", routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01"}))), /at most one.*gatewayId.*routeClass/i)
|
|
301
|
-
assert.throws(() => set.recoverConnection({routeClass: "missing", host: "target.internal", port: 443, identifier: "route_stream_01"}), /unknown routeClass/i)
|
|
302
|
-
const recovered = set.recoverConnection({routeClass: "zyxel", host: "target.internal", port: 443, identifier: "route_stream_01", sessionName: "durable-route"})
|
|
303
|
-
assert.equal(/** @type {{gatewayId?: string}} */ (recovered).gatewayId, undefined)
|
|
304
|
-
assert.equal(/** @type {{routeClass?: string}} */ (recovered).routeClass, "zyxel")
|
|
305
|
-
assert.equal(Object.getOwnPropertyDescriptor(recovered, "routeClass")?.writable, false)
|
|
306
|
-
assert.deepEqual(members.map((member) => member.opens), [0, 1])
|
|
307
|
-
assert.equal(/** @type {{recover?: unknown}} */ (at(members, 1).recoveryConnectionCalls[0]).recover, true)
|
|
308
|
-
assert.equal(Object.hasOwn(/** @type {object} */ (at(members, 1).recoveryConnectionCalls[0]), "routeClass"), false)
|
|
309
|
-
|
|
310
|
-
const started = set.start()
|
|
311
|
-
at(members, 0).ready()
|
|
312
|
-
await started
|
|
313
|
-
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
314
|
-
assert.equal(/** @type {{gatewayId?: string}} */ (socket).gatewayId, undefined)
|
|
315
|
-
assert.equal(/** @type {{routeClass?: string}} */ (socket).routeClass, "starlink")
|
|
316
|
-
assert.deepEqual(set.status().members.map((member) => ({gatewayId: member.gatewayId, routeClass: member.routeClass})), [
|
|
317
|
-
{gatewayId: undefined, routeClass: "starlink"},
|
|
318
|
-
{gatewayId: undefined, routeClass: "zyxel"}
|
|
319
|
-
])
|
|
320
|
-
await set.close()
|
|
321
|
-
})
|
|
333
|
+
it("logs transport-set startup, generation replacement, and terminal failure without error text", async () => {
|
|
334
|
+
/** @type {Array<{event: string} & Record<string, unknown>>} */
|
|
335
|
+
const entries = []
|
|
336
|
+
const logger = {log: (/** @type {string} */ event, /** @type {Record<string, unknown>} */ fields = {}) => entries.push({event, ...fields})}
|
|
337
|
+
const {set, members} = fixture({logger})
|
|
338
|
+
expect(members.every((member) => member.options.logger === logger)).toBeTruthy()
|
|
322
339
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
assert(entries.some(({event, gatewayId, generation}) => event === "transport_set.member_state" && gatewayId === "starlink" && generation === 1))
|
|
341
|
-
assert(entries.some(({event, gatewayId, generation}) => event === "transport_set.member_replaced" && gatewayId === "starlink" && generation === 2))
|
|
342
|
-
assert(entries.some(({event}) => event === "transport_set.terminal_failure"))
|
|
343
|
-
const serialized = JSON.stringify(entries)
|
|
344
|
-
assert.doesNotMatch(serialized, /alphabeticsecret|lowentropysecret|eyJhbGci|127\.0\.0\.1|secret/)
|
|
345
|
-
await set.close()
|
|
346
|
-
})
|
|
340
|
+
const started = set.start()
|
|
341
|
+
at(members, 0).ready()
|
|
342
|
+
await started
|
|
343
|
+
at(members, 0).degrade(Object.assign(new Error("alphabeticsecret eyJhbG...ture"), {code: "ECONNRESET"}))
|
|
344
|
+
at(members, 0).ready()
|
|
345
|
+
at(members, 0).fail(Object.assign(new Error("first lowentropysecret"), {code: "GATEWAY_DISCONNECTED"}))
|
|
346
|
+
at(members, 1).fail(new Error("second alphabeticsecret"))
|
|
347
|
+
at(members, 2).fail(new Error("third eyJhbG...ture"))
|
|
348
|
+
|
|
349
|
+
expect(entries.some(({event}) => event === "transport_set.started")).toBeTruthy()
|
|
350
|
+
expect(entries.some(({event, gatewayId, generation}) => event === "transport_set.member_state" && gatewayId === "starlink" && generation === 1)).toBeTruthy()
|
|
351
|
+
expect(entries.some(({event, gatewayId, generation}) => event === "transport_set.member_replaced" && gatewayId === "starlink" && generation === 2)).toBeTruthy()
|
|
352
|
+
expect(entries.some(({event}) => event === "transport_set.terminal_failure")).toBeTruthy()
|
|
353
|
+
const serialized = JSON.stringify(entries)
|
|
354
|
+
expect(serialized).not.toMatch(/alphabeticsecret|lowentropysecret|eyJhbGci|127\.0\.0\.1|secret/)
|
|
355
|
+
await set.close()
|
|
356
|
+
})
|
|
347
357
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
}
|
|
358
|
+
it("derives stable isolated connection identities and spool namespaces per gateway", () => {
|
|
359
|
+
const first = fixture()
|
|
360
|
+
const second = fixture()
|
|
361
|
+
const firstPairs = memberPairs(first.members)
|
|
362
|
+
const secondPairs = memberPairs(second.members)
|
|
363
|
+
expect(firstPairs).toEqual(secondPairs)
|
|
364
|
+
expect(new Set(firstPairs.map(([connectionId]) => connectionId)).size).toBe(3)
|
|
365
|
+
expect(new Set(firstPairs.map(([, spoolDirectory]) => spoolDirectory)).size).toBe(3)
|
|
366
|
+
const baseSpoolDirectory = options().spoolDirectory
|
|
367
|
+
expect(baseSpoolDirectory).toEqual(expect.any(String))
|
|
368
|
+
if (baseSpoolDirectory === undefined) throw new Error("expected the fixture spool directory")
|
|
369
|
+
for (const [connectionId, spoolDirectory] of firstPairs) {
|
|
370
|
+
expect(connectionId).toMatch(/^[A-Za-z0-9_-]{8,128}$/)
|
|
371
|
+
expect(spoolDirectory.startsWith(join(baseSpoolDirectory, "gateway-"))).toBe(true)
|
|
372
|
+
expect(spoolDirectory.includes("..")).toBe(false)
|
|
373
|
+
}
|
|
374
|
+
})
|
|
364
375
|
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
})
|
|
376
|
+
it("warms every configured member concurrently and coalesces start at minimum readiness", async () => {
|
|
377
|
+
const {set, members} = fixture({minimumReadyMembers: 1})
|
|
378
|
+
const firstStart = set.start()
|
|
379
|
+
const secondStart = set.start()
|
|
380
|
+
expect(firstStart).toBe(secondStart)
|
|
381
|
+
expect(members.map((member) => member.connectCalls)).toEqual([1, 1, 1])
|
|
382
|
+
expect(set.status().connectingMembers).toBe(3)
|
|
383
|
+
|
|
384
|
+
at(members, 1).ready()
|
|
385
|
+
await firstStart
|
|
386
|
+
expect(set.status().readyMembers).toBe(1)
|
|
387
|
+
expect(set.status().state).toBe("degraded")
|
|
388
|
+
expect(members.map((member) => member.connectCalls)).toEqual([1, 1, 1])
|
|
389
|
+
|
|
390
|
+
at(members, 0).ready()
|
|
391
|
+
at(members, 2).ready()
|
|
392
|
+
expect(set.status().readyMembers).toBe(3)
|
|
393
|
+
expect(set.status().state).toBe("ready")
|
|
394
|
+
await set.close()
|
|
395
|
+
})
|
|
385
396
|
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
at(members, 0).ready()
|
|
396
|
-
await started
|
|
397
|
-
assert.equal(gatewayOf(socket), "starlink")
|
|
398
|
-
assert.deepEqual(members.map((member) => member.opens), [1, 0, 0])
|
|
399
|
-
await set.close()
|
|
400
|
-
})
|
|
397
|
+
it("start followed immediately by createConnection returns a socket and waits for readiness", async () => {
|
|
398
|
+
const {set, members} = fixture()
|
|
399
|
+
const started = set.start()
|
|
400
|
+
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
401
|
+
expect(socket.connecting).toBe(true)
|
|
402
|
+
expect(socket.gatewayId).toBe(undefined)
|
|
403
|
+
expect(set.status().pendingOpens).toBe(1)
|
|
404
|
+
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
401
405
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
|
|
409
|
-
at(members, 1).ready()
|
|
410
|
-
assert.equal(gatewayOf(socket), "starlink")
|
|
411
|
-
assert.deepEqual(members.map((member) => member.opens), [1, 0, 0])
|
|
412
|
-
await set.close()
|
|
413
|
-
})
|
|
406
|
+
at(members, 0).ready()
|
|
407
|
+
await started
|
|
408
|
+
expect(gatewayOf(socket)).toBe("starlink")
|
|
409
|
+
expect(members.map((member) => member.opens)).toEqual([1, 0, 0])
|
|
410
|
+
await set.close()
|
|
411
|
+
})
|
|
414
412
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
const openOptions = {
|
|
420
|
-
host: "original-fallback.internal",
|
|
421
|
-
port: 80,
|
|
422
|
-
target,
|
|
423
|
-
sessionName: "original-session",
|
|
424
|
-
identifier: "snapshot_stream_01",
|
|
425
|
-
allowHalfOpen: true,
|
|
426
|
-
readableHighWaterMark: 7,
|
|
427
|
-
writableHighWaterMark: 9
|
|
428
|
-
}
|
|
429
|
-
const socket = set.createConnection(openOptions)
|
|
430
|
-
|
|
431
|
-
openOptions.host = "mutated-fallback.internal"
|
|
432
|
-
openOptions.port = 81
|
|
433
|
-
openOptions.sessionName = "mutated-session"
|
|
434
|
-
openOptions.identifier = "mutated_stream_01"
|
|
435
|
-
openOptions.allowHalfOpen = false
|
|
436
|
-
openOptions.readableHighWaterMark = 70
|
|
437
|
-
openOptions.writableHighWaterMark = 90
|
|
438
|
-
target.host = "mutated.internal"
|
|
439
|
-
target.port = 444
|
|
440
|
-
|
|
441
|
-
at(members, 0).ready()
|
|
442
|
-
await started
|
|
443
|
-
const accepted = at(at(members, 0).createConnectionCalls, 0)
|
|
444
|
-
assert.deepEqual(accepted, {
|
|
445
|
-
host: "original-fallback.internal",
|
|
446
|
-
port: 80,
|
|
447
|
-
target: {host: "original.internal", port: 443},
|
|
448
|
-
sessionName: "original-session",
|
|
449
|
-
identifier: "snapshot_stream_01",
|
|
450
|
-
allowHalfOpen: true,
|
|
451
|
-
readableHighWaterMark: 7,
|
|
452
|
-
writableHighWaterMark: 9
|
|
453
|
-
})
|
|
454
|
-
assert.notEqual(accepted, openOptions)
|
|
455
|
-
assert.notEqual(accepted.target, target)
|
|
456
|
-
assert.equal(socket.remoteAddress, "original.internal")
|
|
457
|
-
assert.equal(socket.remotePort, 443)
|
|
458
|
-
socket.destroy()
|
|
459
|
-
await set.close()
|
|
460
|
-
})
|
|
413
|
+
it("createConnection before start lazily warms the set and waits for minimum readiness", async () => {
|
|
414
|
+
const {set, members} = fixture({minimumReadyMembers: 2})
|
|
415
|
+
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
416
|
+
expect(members.map((member) => member.connectCalls)).toEqual([1, 1, 1])
|
|
461
417
|
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
sessionName: "null-target-fallback"
|
|
470
|
-
}))
|
|
471
|
-
const socket = set.createConnection(options)
|
|
472
|
-
assert.equal(socket.remoteAddress, "fallback.internal")
|
|
473
|
-
assert.equal(socket.remotePort, 443)
|
|
474
|
-
|
|
475
|
-
at(members, 0).ready()
|
|
476
|
-
await started
|
|
477
|
-
assert.deepEqual(at(at(members, 0).createConnectionCalls, 0), {
|
|
478
|
-
target: null,
|
|
479
|
-
host: "fallback.internal",
|
|
480
|
-
port: 443,
|
|
481
|
-
sessionName: "null-target-fallback"
|
|
482
|
-
})
|
|
483
|
-
socket.destroy()
|
|
484
|
-
await set.close()
|
|
485
|
-
})
|
|
418
|
+
at(members, 0).ready()
|
|
419
|
+
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
420
|
+
at(members, 1).ready()
|
|
421
|
+
expect(gatewayOf(socket)).toBe("starlink")
|
|
422
|
+
expect(members.map((member) => member.opens)).toEqual([1, 0, 0])
|
|
423
|
+
await set.close()
|
|
424
|
+
})
|
|
486
425
|
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
426
|
+
it("pending opens snapshot caller options and nested target at call time", async () => {
|
|
427
|
+
const {set, members} = fixture()
|
|
428
|
+
const started = set.start()
|
|
429
|
+
const target = {host: "original.internal", port: 443}
|
|
430
|
+
const openOptions = {
|
|
431
|
+
host: "original-fallback.internal",
|
|
432
|
+
port: 80,
|
|
433
|
+
target,
|
|
434
|
+
sessionName: "original-session",
|
|
435
|
+
identifier: "snapshot_stream_01",
|
|
436
|
+
allowHalfOpen: true,
|
|
437
|
+
readableHighWaterMark: 7,
|
|
438
|
+
writableHighWaterMark: 9
|
|
439
|
+
}
|
|
440
|
+
const socket = set.createConnection(openOptions)
|
|
441
|
+
|
|
442
|
+
openOptions.host = "mutated-fallback.internal"
|
|
443
|
+
openOptions.port = 81
|
|
444
|
+
openOptions.sessionName = "mutated-session"
|
|
445
|
+
openOptions.identifier = "mutated_stream_01"
|
|
446
|
+
openOptions.allowHalfOpen = false
|
|
447
|
+
openOptions.readableHighWaterMark = 70
|
|
448
|
+
openOptions.writableHighWaterMark = 90
|
|
449
|
+
target.host = "mutated.internal"
|
|
450
|
+
target.port = 444
|
|
451
|
+
|
|
452
|
+
at(members, 0).ready()
|
|
453
|
+
await started
|
|
454
|
+
const accepted = at(at(members, 0).createConnectionCalls, 0)
|
|
455
|
+
expect(accepted).toEqual({
|
|
456
|
+
host: "original-fallback.internal",
|
|
457
|
+
port: 80,
|
|
458
|
+
target: {host: "original.internal", port: 443},
|
|
459
|
+
sessionName: "original-session",
|
|
460
|
+
identifier: "snapshot_stream_01",
|
|
461
|
+
allowHalfOpen: true,
|
|
462
|
+
readableHighWaterMark: 7,
|
|
463
|
+
writableHighWaterMark: 9
|
|
464
|
+
})
|
|
465
|
+
expect(accepted).not.toBe(openOptions)
|
|
466
|
+
expect(accepted.target).not.toBe(target)
|
|
467
|
+
expect(socket.remoteAddress).toBe("original.internal")
|
|
468
|
+
expect(socket.remotePort).toBe(443)
|
|
469
|
+
socket.destroy()
|
|
470
|
+
await set.close()
|
|
471
|
+
})
|
|
472
|
+
|
|
473
|
+
it("pending opens preserve null target fallback to host and port", async () => {
|
|
474
|
+
const {set, members} = fixture()
|
|
475
|
+
const started = set.start()
|
|
476
|
+
const options = /** @type {import('../../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ (/** @type {unknown} */ ({
|
|
477
|
+
target: null,
|
|
478
|
+
host: "fallback.internal",
|
|
479
|
+
port: 443,
|
|
480
|
+
sessionName: "null-target-fallback"
|
|
481
|
+
}))
|
|
482
|
+
const socket = set.createConnection(options)
|
|
483
|
+
expect(socket.remoteAddress).toBe("fallback.internal")
|
|
484
|
+
expect(socket.remotePort).toBe(443)
|
|
485
|
+
|
|
486
|
+
at(members, 0).ready()
|
|
487
|
+
await started
|
|
488
|
+
expect(at(at(members, 0).createConnectionCalls, 0)).toEqual({
|
|
489
|
+
target: null,
|
|
490
|
+
host: "fallback.internal",
|
|
491
|
+
port: 443,
|
|
492
|
+
sessionName: "null-target-fallback"
|
|
493
|
+
})
|
|
494
|
+
socket.destroy()
|
|
495
|
+
await set.close()
|
|
496
|
+
})
|
|
513
497
|
|
|
514
|
-
|
|
515
|
-
for (const [count, expected] of /** @type {Array<[number, number[]]>} */ ([[25, [9, 8, 8]], [100, [34, 33, 33]]])) {
|
|
498
|
+
it("round-robins each virtual open fairly across currently ready sticky members", async () => {
|
|
516
499
|
const {set, members} = fixture()
|
|
517
500
|
const started = set.start()
|
|
518
501
|
members.forEach((member) => member.ready())
|
|
519
502
|
await started
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
503
|
+
|
|
504
|
+
const firstThirty = Array.from({length: 30}, () => set.createConnection({host: "target.internal", port: 443}))
|
|
505
|
+
expect(firstThirty.map(gatewayOf).slice(0, 6)).toEqual(["starlink", "zyxel", "o2gateway", "starlink", "zyxel", "o2gateway"])
|
|
506
|
+
expect(set.status().members.map((member) => member.assignments)).toEqual([10, 10, 10])
|
|
507
|
+
|
|
508
|
+
at(members, 1).degrade()
|
|
509
|
+
const whileDown = Array.from({length: 25}, () => set.createConnection({host: "target.internal", port: 443}))
|
|
510
|
+
expect(whileDown.every((socket) => gatewayOf(socket) !== "zyxel")).toBe(true)
|
|
511
|
+
const siblingAssignments = set.status().members.filter((member) => member.gatewayId !== "zyxel").map((member) => member.assignments - 10)
|
|
512
|
+
expect(Math.max(...siblingAssignments) - Math.min(...siblingAssignments) <= 1).toBeTruthy()
|
|
513
|
+
expect(gatewayOf(at(firstThirty, 1))).toBe("zyxel")
|
|
514
|
+
|
|
515
|
+
at(members, 1).ready()
|
|
516
|
+
const beforeRecovery = set.status().members.map((member) => member.assignments)
|
|
517
|
+
const afterRecovery = Array.from({length: 100}, () => set.createConnection({host: "target.internal", port: 443}))
|
|
518
|
+
expect(afterRecovery.some((socket) => gatewayOf(socket) === "zyxel")).toBe(true)
|
|
519
|
+
const recoveryAssignments = set.status().members.map((member, index) => member.assignments - at(beforeRecovery, index))
|
|
520
|
+
expect(Math.max(...recoveryAssignments) - Math.min(...recoveryAssignments) <= 1).toBeTruthy()
|
|
521
|
+
expect(set.status().virtualOpens).toBe(155)
|
|
523
522
|
await set.close()
|
|
524
|
-
}
|
|
525
|
-
})
|
|
523
|
+
})
|
|
526
524
|
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
})
|
|
539
|
-
assert.equal(sockets.every((socket) => socket.gatewayId === undefined), true)
|
|
540
|
-
assert.equal(set.status().pendingOpens, 3)
|
|
541
|
-
assert.equal(set.status().virtualOpens, 0)
|
|
542
|
-
assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
|
|
543
|
-
|
|
544
|
-
at(members, 1).ready()
|
|
545
|
-
await new Promise((resolve) => setImmediate(resolve))
|
|
546
|
-
assert.deepEqual(callbackOrder, ["first", "second", "third"])
|
|
547
|
-
assert.equal(sockets.every((socket) => socket.gatewayId === "zyxel"), true)
|
|
548
|
-
assert.deepEqual(at(members, 1).createConnectionCalls.map((call) => call.sessionName), ["first", "second", "third"])
|
|
549
|
-
assert.equal(set.status().pendingOpens, 0)
|
|
550
|
-
assert.equal(set.status().virtualOpens, 3)
|
|
551
|
-
await set.close()
|
|
552
|
-
})
|
|
525
|
+
it("25 and 100 continuously healthy opens have exact deterministic distributions", async () => {
|
|
526
|
+
for (const [count, expected] of /** @type {Array<[number, number[]]>} */ ([[25, [9, 8, 8]], [100, [34, 33, 33]]])) {
|
|
527
|
+
const {set, members} = fixture()
|
|
528
|
+
const started = set.start()
|
|
529
|
+
members.forEach((member) => member.ready())
|
|
530
|
+
await started
|
|
531
|
+
Array.from({length: count}, () => set.createConnection({host: "target.internal", port: 443}))
|
|
532
|
+
expect(set.status().members.map((member) => member.assignments)).toEqual(expected)
|
|
533
|
+
expect(members.map((member) => member.opens)).toEqual(expected)
|
|
534
|
+
await set.close()
|
|
535
|
+
}
|
|
536
|
+
})
|
|
553
537
|
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
assert.equal(set.status().state, "degraded")
|
|
571
|
-
assert.equal(set.status().pendingOpens, 1)
|
|
572
|
-
assert.deepEqual(set.status().members.map((member) => member.terminalErrorCode), [null, null, null])
|
|
573
|
-
at(members, 1).ready()
|
|
574
|
-
await ready
|
|
575
|
-
assert.equal(callbacks, 1)
|
|
576
|
-
assert.equal(socket.gatewayId, "zyxel")
|
|
577
|
-
await set.close()
|
|
578
|
-
})
|
|
538
|
+
it("opens skip a ready member whose gateway the hub reported unavailable", async () => {
|
|
539
|
+
const {set, members} = fixture()
|
|
540
|
+
const started = set.start()
|
|
541
|
+
members.forEach((member) => member.ready())
|
|
542
|
+
await started
|
|
543
|
+
at(members, 0).gatewayUnavailable = true
|
|
544
|
+
for (let offset = 0; offset < 12; offset += 1) {
|
|
545
|
+
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
546
|
+
expect(gatewayOf(socket) === "starlink").toBe(false)
|
|
547
|
+
}
|
|
548
|
+
expect(set.status().members.map((member) => member.assignments)).toEqual([0, 6, 6])
|
|
549
|
+
at(members, 0).gatewayUnavailable = false
|
|
550
|
+
const recovered = set.createConnection({host: "target.internal", port: 443})
|
|
551
|
+
expect(gatewayOf(recovered)).toBe("starlink")
|
|
552
|
+
await set.close()
|
|
553
|
+
})
|
|
579
554
|
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
socket.on("ready", () => events.push("ready"))
|
|
594
|
-
const ready = onceEvent(socket, "ready")
|
|
595
|
-
at(members, 0).ready()
|
|
596
|
-
await started
|
|
597
|
-
await ready
|
|
598
|
-
assert.deepEqual(events, ["callback", "connect", "ready"])
|
|
599
|
-
assert.equal(callbacks, 1)
|
|
600
|
-
assert.equal(socket.gatewayId, "starlink")
|
|
601
|
-
const descriptor = Object.getOwnPropertyDescriptor(socket, "gatewayId")
|
|
602
|
-
assert.equal(descriptor?.set, undefined)
|
|
603
|
-
assert.equal(descriptor?.configurable, false)
|
|
604
|
-
at(members, 0).degrade()
|
|
605
|
-
at(members, 1).ready()
|
|
606
|
-
assert.equal(socket.gatewayId, "starlink")
|
|
607
|
-
await set.close()
|
|
608
|
-
})
|
|
555
|
+
it("a stale gateway-unavailable mark cannot wedge the set when every ready member is marked", async () => {
|
|
556
|
+
const {set, members} = fixture()
|
|
557
|
+
const started = set.start()
|
|
558
|
+
members.forEach((member) => member.ready())
|
|
559
|
+
await started
|
|
560
|
+
members.forEach((member) => { member.gatewayUnavailable = true })
|
|
561
|
+
const socket = set.createConnection({host: "target.internal", port: 443}, (error) => {
|
|
562
|
+
expect(error).toBe(null)
|
|
563
|
+
})
|
|
564
|
+
expect(socket.gatewayId !== undefined).toBe(true)
|
|
565
|
+
expect(set.status().pendingOpens).toBe(0)
|
|
566
|
+
await set.close()
|
|
567
|
+
})
|
|
609
568
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
569
|
+
it("all-members reconnecting queues FIFO opens until one ready member recovers", async () => {
|
|
570
|
+
const {set, members} = fixture()
|
|
571
|
+
const started = set.start()
|
|
572
|
+
members.forEach((member) => member.ready())
|
|
573
|
+
await started
|
|
574
|
+
members.forEach((member) => member.degrade())
|
|
575
|
+
/** @type {string[]} */
|
|
576
|
+
const callbackOrder = []
|
|
577
|
+
const sockets = ["first", "second", "third"].map((sessionName) => set.createConnection({host: "target.internal", port: 443, sessionName}, (error) => {
|
|
578
|
+
expect(error).toBe(null)
|
|
579
|
+
callbackOrder.push(sessionName)
|
|
580
|
+
}))
|
|
581
|
+
expect(sockets.every((socket) => socket.gatewayId === undefined)).toBe(true)
|
|
582
|
+
expect(set.status().pendingOpens).toBe(3)
|
|
583
|
+
expect(set.status().virtualOpens).toBe(0)
|
|
584
|
+
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
585
|
+
|
|
586
|
+
at(members, 1).ready()
|
|
587
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
588
|
+
expect(callbackOrder).toEqual(["first", "second", "third"])
|
|
589
|
+
expect(sockets.every((socket) => socket.gatewayId === "zyxel")).toBe(true)
|
|
590
|
+
expect(at(members, 1).createConnectionCalls.map((call) => call.sessionName)).toEqual(["first", "second", "third"])
|
|
591
|
+
expect(set.status().pendingOpens).toBe(0)
|
|
592
|
+
expect(set.status().virtualOpens).toBe(3)
|
|
593
|
+
await set.close()
|
|
594
|
+
})
|
|
623
595
|
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
596
|
+
it("transient reverse-hub codes keep opens pending until a member reconnects", async () => {
|
|
597
|
+
const {set, members} = fixture()
|
|
598
|
+
const started = set.start()
|
|
599
|
+
members.forEach((member) => member.ready())
|
|
600
|
+
await started
|
|
601
|
+
const transientCodes = ["GATEWAY_UNAVAILABLE", "GATEWAY_DISCONNECTED", "GATEWAY_REPLACED", "HUB_SHUTDOWN"]
|
|
602
|
+
members.forEach((member, index) => member.degrade(Object.assign(new Error("transient reverse route"), {code: at(transientCodes, index)})))
|
|
603
|
+
at(members, 0).setState("reconnecting", Object.assign(new Error("transient hub shutdown"), {code: at(transientCodes, 3)}))
|
|
604
|
+
let callbacks = 0
|
|
605
|
+
const socket = set.createConnection({host: "target.internal", port: 443}, (error) => {
|
|
606
|
+
expect(error).toBe(null)
|
|
632
607
|
callbacks += 1
|
|
633
|
-
resolve({error, socket})
|
|
634
608
|
})
|
|
609
|
+
const ready = onceEvent(socket, "ready")
|
|
610
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
611
|
+
expect(callbacks).toBe(0)
|
|
612
|
+
expect(set.status().state).toBe("degraded")
|
|
613
|
+
expect(set.status().pendingOpens).toBe(1)
|
|
614
|
+
expect(set.status().members.map((member) => member.terminalErrorCode)).toEqual([null, null, null])
|
|
615
|
+
at(members, 1).ready()
|
|
616
|
+
await ready
|
|
617
|
+
expect(callbacks).toBe(1)
|
|
618
|
+
expect(socket.gatewayId).toBe("zyxel")
|
|
619
|
+
await set.close()
|
|
635
620
|
})
|
|
636
|
-
assert(pendingSocket !== undefined)
|
|
637
|
-
const closed = onceEvent(pendingSocket, "close")
|
|
638
|
-
const failures = members.map((member) => new Error(`${member.gatewayId} unavailable`))
|
|
639
|
-
members.forEach((member, index) => member.fail(at(failures, index)))
|
|
640
|
-
await assert.rejects(start, (error) => error instanceof AggregateError && error.errors.length === 3)
|
|
641
|
-
const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
|
|
642
|
-
await closed
|
|
643
|
-
assert(result.error instanceof AggregateError)
|
|
644
|
-
assert.equal(callbacks, 1)
|
|
645
|
-
assert.equal(set.status().state, "failed")
|
|
646
|
-
assert.equal(set.status().pendingOpens, 0)
|
|
647
|
-
assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
|
|
648
|
-
await set.close()
|
|
649
|
-
})
|
|
650
621
|
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
/** @type {Error | undefined} */
|
|
659
|
-
let pendingError
|
|
660
|
-
const pendingSocket = set.createConnection({host: "target.internal", port: 443}, (error) => { pendingError = error ?? undefined })
|
|
661
|
-
const pendingClosed = onceEvent(pendingSocket, "close")
|
|
662
|
-
const causes = [
|
|
663
|
-
Object.assign(new Error("starlink certificate rejected"), {code: "ERR_TLS_CERT_ALTNAME_INVALID"}),
|
|
664
|
-
Object.assign(new Error("zyxel authentication rejected"), {code: "AUTH_REQUIRED"}),
|
|
665
|
-
Object.assign(new Error("o2 protocol rejected"), {code: "PROTOCOL_MISMATCH"})
|
|
666
|
-
]
|
|
667
|
-
members.forEach((member, index) => member.fail(at(causes, index)))
|
|
668
|
-
await pendingClosed
|
|
669
|
-
assert(pendingError instanceof SocketductTransportSetTerminalError)
|
|
670
|
-
assert.equal(/** @type {{code?: string}} */ (pendingError).code, "SOCKETDUCT_TRANSPORT_SET_TERMINAL")
|
|
671
|
-
assert.deepEqual(pendingError.errors, causes)
|
|
672
|
-
const status = set.status()
|
|
673
|
-
assert.deepEqual(status.members.map((member) => member.terminalErrorCode), causes.map((error) => error.code))
|
|
674
|
-
assert.doesNotMatch(JSON.stringify(status), /certificate rejected|authentication rejected|protocol rejected/)
|
|
675
|
-
assert.equal(status.state, "failed")
|
|
676
|
-
|
|
677
|
-
let callbacks = 0
|
|
678
|
-
/** @type {import('../../../src/client/virtual-socket.js').VirtualSocket | undefined} */
|
|
679
|
-
let failedSocket
|
|
680
|
-
const callbackResult = new Promise((resolve) => {
|
|
681
|
-
failedSocket = set.createConnection({host: "target.internal", port: 443}, (error, socket) => {
|
|
622
|
+
it("pending assignment preserves callback, connect, ready, and immutable gateway ordering", async () => {
|
|
623
|
+
const {set, members} = fixture()
|
|
624
|
+
const started = set.start()
|
|
625
|
+
/** @type {string[]} */
|
|
626
|
+
const events = []
|
|
627
|
+
let callbacks = 0
|
|
628
|
+
const socket = set.createConnection({host: "target.internal", port: 443}, (error, callbackSocket) => {
|
|
682
629
|
callbacks += 1
|
|
683
|
-
|
|
630
|
+
expect(error).toBe(null)
|
|
631
|
+
expect(callbackSocket).toBe(socket)
|
|
632
|
+
events.push("callback")
|
|
684
633
|
})
|
|
634
|
+
socket.on("connect", () => events.push("connect"))
|
|
635
|
+
socket.on("ready", () => events.push("ready"))
|
|
636
|
+
const ready = onceEvent(socket, "ready")
|
|
637
|
+
at(members, 0).ready()
|
|
638
|
+
await started
|
|
639
|
+
await ready
|
|
640
|
+
expect(events).toEqual(["callback", "connect", "ready"])
|
|
641
|
+
expect(callbacks).toBe(1)
|
|
642
|
+
expect(socket.gatewayId).toBe("starlink")
|
|
643
|
+
const descriptor = Object.getOwnPropertyDescriptor(socket, "gatewayId")
|
|
644
|
+
expect(descriptor?.set).toBe(undefined)
|
|
645
|
+
expect(descriptor?.configurable).toBe(false)
|
|
646
|
+
at(members, 0).degrade()
|
|
647
|
+
at(members, 1).ready()
|
|
648
|
+
expect(socket.gatewayId).toBe("starlink")
|
|
649
|
+
await set.close()
|
|
685
650
|
})
|
|
686
|
-
assert(failedSocket !== undefined)
|
|
687
|
-
const closed = onceEvent(failedSocket, "close")
|
|
688
|
-
const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
|
|
689
|
-
await closed
|
|
690
|
-
assert.equal(result.error, pendingError)
|
|
691
|
-
assert.equal(callbacks, 1)
|
|
692
|
-
assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
|
|
693
|
-
await set.close()
|
|
694
|
-
})
|
|
695
651
|
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
await closed
|
|
710
|
-
assert.equal(/** @type {{code?: string}} */ (result.error).code, "SOCKETDUCT_TRANSPORT_SET_CLOSED")
|
|
711
|
-
await close
|
|
712
|
-
assert.deepEqual(members.map((member) => member.opens), [0, 0, 0])
|
|
713
|
-
assert.throws(() => set.createConnection({host: "target.internal", port: 443}), /closed/i)
|
|
714
|
-
})
|
|
652
|
+
it("destroy before assignment cancels the pending open and cannot leak a later member stream", async () => {
|
|
653
|
+
const {set, members} = fixture()
|
|
654
|
+
const started = set.start()
|
|
655
|
+
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
656
|
+
const closed = onceEvent(socket, "close")
|
|
657
|
+
socket.destroy()
|
|
658
|
+
await closed
|
|
659
|
+
expect(set.status().pendingOpens).toBe(0)
|
|
660
|
+
at(members, 0).ready()
|
|
661
|
+
await started
|
|
662
|
+
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
663
|
+
await set.close()
|
|
664
|
+
})
|
|
715
665
|
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
666
|
+
it("terminal startup failure reaches start and each pending socket exactly once", async () => {
|
|
667
|
+
const {set, members} = fixture()
|
|
668
|
+
const start = set.start()
|
|
669
|
+
let callbacks = 0
|
|
670
|
+
/** @type {import('../../../src/client/virtual-socket.js').VirtualSocket | undefined} */
|
|
671
|
+
let pendingSocket
|
|
672
|
+
const callbackResult = new Promise((resolve) => {
|
|
673
|
+
pendingSocket = set.createConnection({host: "target.internal", port: 443}, (error, socket) => {
|
|
674
|
+
callbacks += 1
|
|
675
|
+
resolve({error, socket})
|
|
676
|
+
})
|
|
677
|
+
})
|
|
678
|
+
expect(pendingSocket).toBeDefined()
|
|
679
|
+
const closed = onceEvent(/** @type {import("node:events").EventEmitter} */ (pendingSocket), "close")
|
|
680
|
+
const failures = members.map((member) => new Error(`${member.gatewayId} unavailable`))
|
|
681
|
+
members.forEach((member, index) => member.fail(at(failures, index)))
|
|
682
|
+
let startError
|
|
683
|
+
try {
|
|
684
|
+
await start
|
|
685
|
+
} catch (error) {
|
|
686
|
+
startError = /** @type {Error} */ (error)
|
|
687
|
+
}
|
|
688
|
+
expect(startError).toBeInstanceOf(AggregateError)
|
|
689
|
+
expect(/** @type {AggregateError} */ (startError).errors.length).toBe(3)
|
|
690
|
+
const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
|
|
691
|
+
await closed
|
|
692
|
+
expect(result.error).toBeInstanceOf(AggregateError)
|
|
693
|
+
expect(/** @type {AggregateError} */ (result.error).errors).toEqual(failures)
|
|
694
|
+
expect(callbacks).toBe(1)
|
|
695
|
+
expect(set.status().state).toBe("failed")
|
|
696
|
+
expect(set.status().pendingOpens).toBe(0)
|
|
697
|
+
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
698
|
+
await set.close()
|
|
699
|
+
})
|
|
743
700
|
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
701
|
+
it("terminal loss preserves ordered member causes for pending and future opens", async () => {
|
|
702
|
+
const {set, members} = fixture()
|
|
703
|
+
const started = set.start()
|
|
704
|
+
members.forEach((member) => member.ready())
|
|
705
|
+
await started
|
|
706
|
+
members.forEach((member) => member.degrade())
|
|
707
|
+
|
|
708
|
+
/** @type {Error | undefined} */
|
|
709
|
+
let pendingError
|
|
710
|
+
const pendingSocket = set.createConnection({host: "target.internal", port: 443}, (error) => { pendingError = error ?? undefined })
|
|
711
|
+
const pendingClosed = onceEvent(/** @type {import("node:events").EventEmitter} */ (pendingSocket), "close")
|
|
712
|
+
const causes = [
|
|
713
|
+
Object.assign(new Error("starlink certificate rejected"), {code: "ERR_TLS_CERT_ALTNAME_INVALID"}),
|
|
714
|
+
Object.assign(new Error("zyxel authentication rejected"), {code: "AUTH_REQUIRED"}),
|
|
715
|
+
Object.assign(new Error("o2 protocol rejected"), {code: "PROTOCOL_MISMATCH"})
|
|
716
|
+
]
|
|
717
|
+
members.forEach((member, index) => member.fail(at(causes, index)))
|
|
718
|
+
await pendingClosed
|
|
719
|
+
expect(pendingError).toBeInstanceOf(SocketductTransportSetTerminalError)
|
|
720
|
+
if (!(pendingError instanceof SocketductTransportSetTerminalError)) throw new Error("expected terminal error")
|
|
721
|
+
expect(pendingError.code).toBe("SOCKETDUCT_TRANSPORT_SET_TERMINAL")
|
|
722
|
+
expect(pendingError.errors).toEqual(causes)
|
|
723
|
+
const status = set.status()
|
|
724
|
+
expect(status.members.map((member) => member.terminalErrorCode)).toEqual(causes.map((error) => error.code))
|
|
725
|
+
expect(JSON.stringify(status)).not.toMatch(/certificate rejected|authentication rejected|protocol rejected/)
|
|
726
|
+
expect(status.state).toBe("failed")
|
|
727
|
+
|
|
728
|
+
let callbacks = 0
|
|
729
|
+
/** @type {import('../../../src/client/virtual-socket.js').VirtualSocket | undefined} */
|
|
730
|
+
let failedSocket
|
|
731
|
+
const callbackResult = new Promise((resolve) => {
|
|
732
|
+
failedSocket = set.createConnection({host: "target.internal", port: 443}, (error, socket) => {
|
|
733
|
+
callbacks += 1
|
|
734
|
+
resolve({error, socket})
|
|
735
|
+
})
|
|
736
|
+
})
|
|
737
|
+
expect(failedSocket).toBeDefined()
|
|
738
|
+
if (failedSocket === undefined) throw new Error("expected failedSocket")
|
|
739
|
+
const closed = onceEvent(failedSocket, "close")
|
|
740
|
+
const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
|
|
741
|
+
await closed
|
|
742
|
+
expect(result.error).toBe(pendingError)
|
|
743
|
+
expect(callbacks).toBe(1)
|
|
744
|
+
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
745
|
+
await set.close()
|
|
746
|
+
})
|
|
770
747
|
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
748
|
+
it("close fails a pending socket without allocating a member and rejects later opens", async () => {
|
|
749
|
+
const {set, members} = fixture()
|
|
750
|
+
const start = set.start()
|
|
751
|
+
/** @type {import('../../../src/client/virtual-socket.js').VirtualSocket | undefined} */
|
|
752
|
+
let pendingSocket
|
|
753
|
+
const callbackResult = new Promise((resolve) => {
|
|
754
|
+
pendingSocket = set.createConnection({host: "target.internal", port: 443}, (error, socket) => resolve({error, socket}))
|
|
755
|
+
})
|
|
756
|
+
expect(pendingSocket).toBeDefined()
|
|
757
|
+
const closed = onceEvent(/** @type {import("node:events").EventEmitter} */ (pendingSocket), "close")
|
|
758
|
+
const close = set.close()
|
|
759
|
+
let startError
|
|
760
|
+
try {
|
|
761
|
+
await start
|
|
762
|
+
} catch (error) {
|
|
763
|
+
startError = /** @type {Error} */ (error)
|
|
785
764
|
}
|
|
765
|
+
expect(startError).toBeInstanceOf(Error)
|
|
766
|
+
expect(/** @type {{code?: string}} */ (startError).code).toBe("SOCKETDUCT_TRANSPORT_SET_CLOSED")
|
|
767
|
+
const result = /** @type {{error: Error, socket: import('../../../src/client/virtual-socket.js').VirtualSocket}} */ (await callbackResult)
|
|
768
|
+
await closed
|
|
769
|
+
expect(/** @type {{code?: string}} */ (result.error).code).toBe("SOCKETDUCT_TRANSPORT_SET_CLOSED")
|
|
770
|
+
await close
|
|
771
|
+
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
772
|
+
expect(() => set.createConnection({host: "target.internal", port: 443})).toThrow(/closed/i)
|
|
786
773
|
})
|
|
787
|
-
const drained = onceEvent(socket, "drain")
|
|
788
|
-
|
|
789
|
-
assert.equal(socket.write(payload, completeWrite), false)
|
|
790
|
-
assert.equal(set.status().budgets.dataBytes, payload.byteLength)
|
|
791
|
-
assert.throws(
|
|
792
|
-
() => socket.write(/** @type {never} */ ({unsupported: true})),
|
|
793
|
-
(error) => /** @type {{code?: string}} */ (error).code === "ERR_INVALID_ARG_TYPE"
|
|
794
|
-
)
|
|
795
|
-
assert.equal(set.status().budgets.dataBytes, payload.byteLength)
|
|
796
|
-
|
|
797
|
-
at(members, 0).ready()
|
|
798
|
-
await Promise.all([started, written, drained])
|
|
799
|
-
assert.equal(callbacks, 1)
|
|
800
|
-
assert.deepEqual(at(at(members, 0).sockets, 0).writes, [expected])
|
|
801
|
-
assert.equal(set.status().budgets.dataBytes, 0)
|
|
802
|
-
socket.destroy()
|
|
803
|
-
await set.close()
|
|
804
|
-
})
|
|
805
774
|
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
775
|
+
it("bounds pending open count and pending application bytes", async () => {
|
|
776
|
+
const countFixture = fixture({maxPendingOpens: 1})
|
|
777
|
+
const first = countFixture.set.createConnection({host: "target.internal", port: 443})
|
|
778
|
+
let limitError
|
|
779
|
+
try {
|
|
780
|
+
countFixture.set.createConnection({host: "target.internal", port: 443})
|
|
781
|
+
} catch (error) {
|
|
782
|
+
limitError = /** @type {Error} */ (error)
|
|
783
|
+
}
|
|
784
|
+
expect(limitError).toBeInstanceOf(SocketductTransportSetLimitError)
|
|
785
|
+
expect(/** @type {{code?: string}} */ (limitError).code).toBe("SOCKETDUCT_SET_PENDING_OPEN_LIMIT")
|
|
786
|
+
const firstClosed = onceEvent(first, "close")
|
|
787
|
+
first.destroy()
|
|
788
|
+
await firstClosed
|
|
789
|
+
await countFixture.set.close()
|
|
790
|
+
|
|
791
|
+
const byteFixture = fixture({maxTotalBufferBytes: 4, maxTotalQueuedBytes: 4})
|
|
792
|
+
const socket = byteFixture.set.createConnection({host: "target.internal", port: 443})
|
|
793
|
+
const streamError = onceEvent(socket, "error")
|
|
794
|
+
const socketClosed = onceEvent(socket, "close")
|
|
795
|
+
socket.write("four")
|
|
796
|
+
const callbackError = new Promise((resolve) => {
|
|
797
|
+
expect(socket.write("!", (error) => resolve(error))).toBe(false)
|
|
822
798
|
})
|
|
823
|
-
|
|
799
|
+
const [error] = await streamError
|
|
800
|
+
expect(error).toBe(await callbackError)
|
|
801
|
+
await socketClosed
|
|
802
|
+
expect(byteFixture.set.status().budgets.dataBytes).toBe(0)
|
|
803
|
+
expect(byteFixture.set.status().budgets.queuedBytes).toBe(0)
|
|
804
|
+
await byteFixture.set.close()
|
|
824
805
|
})
|
|
825
806
|
|
|
826
|
-
|
|
807
|
+
it("pending sockets retain timeout and Writable backpressure behavior", async () => {
|
|
808
|
+
const {set, members} = fixture()
|
|
809
|
+
const started = set.start()
|
|
810
|
+
const socket = set.createConnection({
|
|
811
|
+
host: "target.internal",
|
|
812
|
+
port: 443,
|
|
813
|
+
writableHighWaterMark: 4
|
|
814
|
+
})
|
|
815
|
+
const timedOut = onceEvent(socket, "timeout")
|
|
816
|
+
socket.setTimeout(1)
|
|
817
|
+
await timedOut
|
|
818
|
+
expect(socket.destroyed).toBe(false)
|
|
819
|
+
expect(socket.connecting).toBe(true)
|
|
820
|
+
socket.setTimeout(0)
|
|
821
|
+
|
|
822
|
+
const drained = onceEvent(socket, "drain")
|
|
823
|
+
const written = new Promise((resolve, reject) => {
|
|
824
|
+
expect(socket.write("four", (error) => error ? reject(error) : resolve(undefined))).toBe(false)
|
|
825
|
+
})
|
|
826
|
+
expect(set.status().budgets.dataBytes).toBe(4)
|
|
827
|
+
at(members, 0).ready()
|
|
828
|
+
await Promise.all([started, written, drained])
|
|
829
|
+
expect(set.status().budgets.dataBytes).toBe(0)
|
|
830
|
+
socket.destroy()
|
|
831
|
+
await set.close()
|
|
832
|
+
})
|
|
827
833
|
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
}
|
|
834
|
+
it("pending sockets reserve and write non-Uint8 typed-array views", async () => {
|
|
835
|
+
const payload = new Uint16Array([0x1234, 0xabcd])
|
|
836
|
+
const expected = Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength)
|
|
837
|
+
const {set, members} = fixture({maxTotalBufferBytes: payload.byteLength, maxTotalQueuedBytes: payload.byteLength})
|
|
838
|
+
const started = set.start()
|
|
839
|
+
const socket = set.createConnection({host: "target.internal", port: 443, writableHighWaterMark: payload.byteLength})
|
|
840
|
+
let callbacks = 0
|
|
841
|
+
/** @type {(error?: Error | null) => void} */
|
|
842
|
+
let completeWrite = () => {}
|
|
843
|
+
const written = new Promise((resolve, reject) => {
|
|
844
|
+
completeWrite = (error) => {
|
|
845
|
+
callbacks += 1
|
|
846
|
+
if (error) reject(error)
|
|
847
|
+
else resolve(undefined)
|
|
848
|
+
}
|
|
849
|
+
})
|
|
850
|
+
const drained = onceEvent(socket, "drain")
|
|
851
|
+
|
|
852
|
+
expect(socket.write(payload, completeWrite)).toBe(false)
|
|
853
|
+
expect(set.status().budgets.dataBytes).toBe(payload.byteLength)
|
|
854
|
+
let unsupportedError
|
|
855
|
+
try {
|
|
856
|
+
socket.write(/** @type {never} */ ({unsupported: true}))
|
|
857
|
+
} catch (error) {
|
|
858
|
+
unsupportedError = /** @type {Error} */ (error)
|
|
859
|
+
}
|
|
860
|
+
expect(unsupportedError).toBeInstanceOf(Error)
|
|
861
|
+
expect(/** @type {{code?: string}} */ (unsupportedError).code).toBe("ERR_INVALID_ARG_TYPE")
|
|
862
|
+
expect(set.status().budgets.dataBytes).toBe(payload.byteLength)
|
|
863
|
+
|
|
864
|
+
at(members, 0).ready()
|
|
865
|
+
await Promise.all([started, written, drained])
|
|
866
|
+
expect(callbacks).toBe(1)
|
|
867
|
+
const actualWrites = at(at(members, 0).sockets, 0).writes
|
|
868
|
+
expect(actualWrites.length).toBe(1)
|
|
869
|
+
expect(Buffer.from(at(actualWrites, 0)).equals(expected)).toBe(true)
|
|
870
|
+
expect(set.status().budgets.dataBytes).toBe(0)
|
|
871
|
+
socket.destroy()
|
|
872
|
+
await set.close()
|
|
873
|
+
})
|
|
836
874
|
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
/** @type {() => void} */
|
|
847
|
-
let completeEnd = () => {}
|
|
848
|
-
const ended = new Promise((resolve) => {
|
|
849
|
-
completeEnd = () => {
|
|
850
|
-
callbacks += 1
|
|
875
|
+
it("pending socket end payload reserves and sends bytes after assignment", async () => {
|
|
876
|
+
const {set, members} = fixture({maxTotalBufferBytes: 5, maxTotalQueuedBytes: 5})
|
|
877
|
+
const started = set.start()
|
|
878
|
+
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
879
|
+
let callbacks = 0
|
|
880
|
+
/** @type {string[]} */
|
|
881
|
+
const events = []
|
|
882
|
+
const finished = new Promise((resolve) => socket.once("finish", () => {
|
|
883
|
+
events.push("finish")
|
|
851
884
|
resolve(undefined)
|
|
852
|
-
}
|
|
885
|
+
}))
|
|
886
|
+
const ended = new Promise((resolve) => {
|
|
887
|
+
const returned = socket.end("final", () => {
|
|
888
|
+
callbacks += 1
|
|
889
|
+
events.push("callback")
|
|
890
|
+
resolve(undefined)
|
|
891
|
+
})
|
|
892
|
+
expect(returned).toBe(socket)
|
|
893
|
+
})
|
|
894
|
+
|
|
895
|
+
expect(set.status().budgets.dataBytes).toBe(5)
|
|
896
|
+
|
|
897
|
+
at(members, 0).ready()
|
|
898
|
+
await Promise.all([started, ended, finished])
|
|
899
|
+
expect(callbacks).toBe(1)
|
|
900
|
+
expect(events).toEqual(["callback", "finish"])
|
|
901
|
+
const actualWrites = at(at(members, 0).sockets, 0).writes
|
|
902
|
+
expect(actualWrites.length).toBe(1)
|
|
903
|
+
expect(Buffer.from(at(actualWrites, 0)).equals(Buffer.from("final"))).toBe(true)
|
|
904
|
+
expect(set.status().budgets.dataBytes).toBe(0)
|
|
905
|
+
await set.close()
|
|
853
906
|
})
|
|
854
907
|
|
|
855
|
-
|
|
856
|
-
|
|
908
|
+
it("pending socket end reserves and sends DataView bytes", async () => {
|
|
909
|
+
const buffer = new ArrayBuffer(6)
|
|
910
|
+
const payload = new DataView(buffer, 1, 4)
|
|
911
|
+
payload.setUint32(0, 0x1234abcd)
|
|
912
|
+
const expected = Buffer.from(buffer, payload.byteOffset, payload.byteLength)
|
|
913
|
+
const {set, members} = fixture({maxTotalBufferBytes: payload.byteLength, maxTotalQueuedBytes: payload.byteLength})
|
|
914
|
+
const started = set.start()
|
|
915
|
+
const socket = set.createConnection({host: "target.internal", port: 443})
|
|
916
|
+
let callbacks = 0
|
|
917
|
+
/** @type {() => void} */
|
|
918
|
+
let completeEnd = () => {}
|
|
919
|
+
const ended = new Promise((resolve) => {
|
|
920
|
+
completeEnd = () => {
|
|
921
|
+
callbacks += 1
|
|
922
|
+
resolve(undefined)
|
|
923
|
+
}
|
|
924
|
+
})
|
|
857
925
|
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
assert.equal(callbacks, 1)
|
|
861
|
-
assert.deepEqual(at(at(members, 0).sockets, 0).writes, [expected])
|
|
862
|
-
assert.equal(set.status().budgets.dataBytes, 0)
|
|
863
|
-
await set.close()
|
|
864
|
-
})
|
|
926
|
+
expect(socket.end(payload, completeEnd)).toBe(socket)
|
|
927
|
+
expect(set.status().budgets.dataBytes).toBe(payload.byteLength)
|
|
865
928
|
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
gatewayId: "zyxel",
|
|
876
|
-
host: "target.internal",
|
|
877
|
-
port: 443,
|
|
878
|
-
identifier: "stable_stream_01",
|
|
879
|
-
sessionName: "durable"
|
|
880
|
-
})
|
|
881
|
-
await new Promise((resolve) => setImmediate(resolve))
|
|
882
|
-
const recoveredLater = set.recoverConnection({
|
|
883
|
-
gatewayId: "zyxel",
|
|
884
|
-
host: "target.internal",
|
|
885
|
-
port: 443,
|
|
886
|
-
identifier: "stable_stream_02",
|
|
887
|
-
sessionName: "durable-later"
|
|
888
|
-
})
|
|
889
|
-
assert.equal(gatewayOf(recovered), "zyxel")
|
|
890
|
-
assert.equal(gatewayOf(recoveredLater), "zyxel")
|
|
891
|
-
assert.deepEqual(members.map((member) => member.opens), [0, 2, 0])
|
|
892
|
-
assert.equal(at(members, 1).options.connectionId === undefined, false)
|
|
893
|
-
assert.equal(at(members, 1).createConnectionCalls.length, 0)
|
|
894
|
-
assert.equal(/** @type {{recover?: unknown}} */ (at(members, 1).recoveryConnectionCalls[0]).recover, true)
|
|
895
|
-
assert.equal(at(members, 1).recoveryConnectionCalls.length, 2)
|
|
896
|
-
assert.equal(Object.hasOwn(/** @type {object} */ (at(members, 1).recoveryConnectionCalls[0]), "gatewayId"), false)
|
|
897
|
-
|
|
898
|
-
const starting = set.start()
|
|
899
|
-
assert.throws(() => set.recoverConnection({gatewayId: "zyxel", host: "target.internal", port: 443, identifier: "stable_stream_04"}), /before start/i)
|
|
900
|
-
members.forEach((member) => member.ready())
|
|
901
|
-
await starting
|
|
902
|
-
await set.close()
|
|
903
|
-
})
|
|
929
|
+
at(members, 0).ready()
|
|
930
|
+
await Promise.all([started, ended])
|
|
931
|
+
expect(callbacks).toBe(1)
|
|
932
|
+
const actualWrites = at(at(members, 0).sockets, 0).writes
|
|
933
|
+
expect(actualWrites.length).toBe(1)
|
|
934
|
+
expect(Buffer.from(at(actualWrites, 0)).equals(expected)).toBe(true)
|
|
935
|
+
expect(set.status().budgets.dataBytes).toBe(0)
|
|
936
|
+
await set.close()
|
|
937
|
+
})
|
|
904
938
|
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
identifier: "
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
()
|
|
940
|
-
(
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
939
|
+
it("recovery requires one original configured selector and registration before start", async () => {
|
|
940
|
+
const {set, members} = fixture()
|
|
941
|
+
let notFoundError
|
|
942
|
+
try {
|
|
943
|
+
set.recoverConnection(/** @type {Parameters<MultiplexedSocketductTransportSet["recoverConnection"]>[0]} */ (/** @type {unknown} */ ({host: "target.internal", port: 443, identifier: "stable_stream_01"})))
|
|
944
|
+
} catch (error) {
|
|
945
|
+
notFoundError = /** @type {Error} */ (error)
|
|
946
|
+
}
|
|
947
|
+
expect(notFoundError).toBeInstanceOf(Error)
|
|
948
|
+
expect(/** @type {{code?: string}} */ (notFoundError).code).toBe("SOCKETDUCT_RECOVERY_NOT_FOUND")
|
|
949
|
+
expect(() => set.recoverConnection({gatewayId: "unknown", host: "target.internal", port: 443, identifier: "stable_stream_01"})).toThrow(/unknown gatewayId/i)
|
|
950
|
+
expect(() => set.createConnection(/** @type {import('../../../src/client/multiplexed-connection.js').VirtualConnectionOptions} */ (/** @type {unknown} */ ({gatewayId: "zyxel", host: "target.internal", port: 443})))).toThrow(/recoverConnection/i)
|
|
951
|
+
expect(() => set.createConnection({recover: true, host: "target.internal", port: 443, identifier: "stable_stream_01"})).toThrow(/recoverConnection/i)
|
|
952
|
+
expect(() => set.createTlsConnection({gatewayId: "zyxel", host: "target.internal", port: 443})).toThrow(/recoverConnection/i)
|
|
953
|
+
|
|
954
|
+
const recovered = set.recoverConnection({
|
|
955
|
+
gatewayId: "zyxel",
|
|
956
|
+
host: "target.internal",
|
|
957
|
+
port: 443,
|
|
958
|
+
identifier: "stable_stream_01",
|
|
959
|
+
sessionName: "durable"
|
|
960
|
+
})
|
|
961
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
962
|
+
const recoveredLater = set.recoverConnection({
|
|
963
|
+
gatewayId: "zyxel",
|
|
964
|
+
host: "target.internal",
|
|
965
|
+
port: 443,
|
|
966
|
+
identifier: "stable_stream_02",
|
|
967
|
+
sessionName: "durable-later"
|
|
968
|
+
})
|
|
969
|
+
expect(gatewayOf(recovered)).toBe("zyxel")
|
|
970
|
+
expect(gatewayOf(recoveredLater)).toBe("zyxel")
|
|
971
|
+
expect(members.map((member) => member.opens)).toEqual([0, 2, 0])
|
|
972
|
+
expect(at(members, 1).options.connectionId === undefined).toBe(false)
|
|
973
|
+
expect(at(members, 1).createConnectionCalls.length).toBe(0)
|
|
974
|
+
expect(/** @type {{recover?: unknown}} */ (at(members, 1).recoveryConnectionCalls[0]).recover).toBe(true)
|
|
975
|
+
expect(at(members, 1).recoveryConnectionCalls.length).toBe(2)
|
|
976
|
+
expect(Object.hasOwn(/** @type {object} */ (at(members, 1).recoveryConnectionCalls[0]), "gatewayId")).toBe(false)
|
|
977
|
+
|
|
978
|
+
const starting = set.start()
|
|
979
|
+
expect(() => set.recoverConnection({gatewayId: "zyxel", host: "target.internal", port: 443, identifier: "stable_stream_04"})).toThrow(/before start/i)
|
|
980
|
+
members.forEach((member) => member.ready())
|
|
981
|
+
await starting
|
|
982
|
+
await set.close()
|
|
983
|
+
})
|
|
944
984
|
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
assert.equal(descriptor?.writable, false)
|
|
954
|
-
await set.close()
|
|
955
|
-
|
|
956
|
-
const recoveryFixture = fixture()
|
|
957
|
-
const recovered = recoveryFixture.set.recoverConnection({gatewayId: "zyxel", host: "target.internal", port: 443, identifier: "stable_stream_03"})
|
|
958
|
-
assert.equal(gatewayOf(recovered), "zyxel")
|
|
959
|
-
await recoveryFixture.set.close()
|
|
960
|
-
})
|
|
985
|
+
it("identifier-only recovery finds exactly one durable member and restores stored options", async () => {
|
|
986
|
+
const unique = fixture()
|
|
987
|
+
at(unique.members, 1).recoveryDescriptors.set("stable_stream_05", {
|
|
988
|
+
identifier: "stable_stream_05",
|
|
989
|
+
target: {host: "target.internal", port: 443},
|
|
990
|
+
sessionName: "durable-request",
|
|
991
|
+
manualReceiveAck: true
|
|
992
|
+
})
|
|
961
993
|
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
994
|
+
const recovered = unique.set.recoverConnection({identifier: "stable_stream_05"})
|
|
995
|
+
expect(gatewayOf(recovered)).toBe("zyxel")
|
|
996
|
+
expect(at(unique.members, 1).recoveryConnectionCalls).toEqual([{
|
|
997
|
+
identifier: "stable_stream_05",
|
|
998
|
+
target: {host: "target.internal", port: 443},
|
|
999
|
+
sessionName: "durable-request",
|
|
1000
|
+
manualReceiveAck: true,
|
|
1001
|
+
recover: true
|
|
1002
|
+
}])
|
|
1003
|
+
let missingError
|
|
1004
|
+
try {
|
|
1005
|
+
unique.set.recoverConnection({identifier: "missing_stream_01"})
|
|
1006
|
+
} catch (error) {
|
|
1007
|
+
missingError = /** @type {Error} */ (error)
|
|
1008
|
+
}
|
|
1009
|
+
expect(missingError).toBeInstanceOf(Error)
|
|
1010
|
+
expect(/** @type {{code?: string}} */ (missingError).code).toBe("SOCKETDUCT_RECOVERY_NOT_FOUND")
|
|
1011
|
+
await unique.set.close()
|
|
1012
|
+
|
|
1013
|
+
const ambiguous = fixture()
|
|
1014
|
+
const descriptor = {
|
|
1015
|
+
identifier: "duplicate_stream_01",
|
|
1016
|
+
target: {host: "target.internal", port: 443},
|
|
1017
|
+
sessionName: "duplicate",
|
|
1018
|
+
manualReceiveAck: false
|
|
1019
|
+
}
|
|
1020
|
+
at(ambiguous.members, 0).recoveryDescriptors.set(descriptor.identifier, descriptor)
|
|
1021
|
+
at(ambiguous.members, 2).recoveryDescriptors.set(descriptor.identifier, descriptor)
|
|
1022
|
+
let ambiguousError
|
|
1023
|
+
try {
|
|
1024
|
+
ambiguous.set.recoverConnection({identifier: descriptor.identifier})
|
|
1025
|
+
} catch (error) {
|
|
1026
|
+
ambiguousError = /** @type {Error} */ (error)
|
|
1027
|
+
}
|
|
1028
|
+
expect(ambiguousError).toBeInstanceOf(Error)
|
|
1029
|
+
expect(/** @type {{code?: string}} */ (ambiguousError).code).toBe("SOCKETDUCT_RECOVERY_AMBIGUOUS")
|
|
1030
|
+
await ambiguous.set.close()
|
|
1031
|
+
})
|
|
1032
|
+
|
|
1033
|
+
it("returns the stable selected gatewayId on ordinary and recovered sockets", async () => {
|
|
1034
|
+
const {set, members} = fixture()
|
|
1035
|
+
const started = set.start()
|
|
1036
|
+
members.forEach((member) => member.ready())
|
|
1037
|
+
await started
|
|
1038
|
+
const ordinary = set.createConnection({host: "target.internal", port: 443})
|
|
1039
|
+
expect(gatewayOf(ordinary)).toBe("starlink")
|
|
1040
|
+
const descriptor = Object.getOwnPropertyDescriptor(ordinary, "gatewayId")
|
|
1041
|
+
expect(descriptor?.writable).toBe(false)
|
|
1042
|
+
await set.close()
|
|
1043
|
+
|
|
1044
|
+
const recoveryFixture = fixture()
|
|
1045
|
+
const recovered = recoveryFixture.set.recoverConnection({gatewayId: "zyxel", host: "target.internal", port: 443, identifier: "stable_stream_03"})
|
|
1046
|
+
expect(gatewayOf(recovered)).toBe("zyxel")
|
|
1047
|
+
await recoveryFixture.set.close()
|
|
1048
|
+
})
|
|
1049
|
+
|
|
1050
|
+
it("shares aggregate data, spool, and queued-byte reservations across members", async () => {
|
|
1051
|
+
const {set, members} = fixture({maxTotalBufferBytes: 10, maxTotalSpoolBytes: 8, maxTotalQueuedBytes: 12})
|
|
1052
|
+
const budgets = members.map((member) => {
|
|
1053
|
+
const budget = member.options.aggregateBudget
|
|
1054
|
+
expect(budget).toBeDefined()
|
|
1055
|
+
return /** @type {NonNullable<typeof budget>} */ (budget)
|
|
1056
|
+
})
|
|
1057
|
+
expect(new Set(budgets).size).toBe(1)
|
|
1058
|
+
at(budgets, 0).reserveData(6)
|
|
1059
|
+
let spoolLimitError
|
|
1060
|
+
try {
|
|
1061
|
+
at(budgets, 1).reserveData(3)
|
|
1062
|
+
} catch (error) {
|
|
1063
|
+
spoolLimitError = /** @type {Error} */ (error)
|
|
1064
|
+
}
|
|
1065
|
+
expect(spoolLimitError).toBeInstanceOf(Error)
|
|
1066
|
+
expect(/** @type {{code?: string}} */ (spoolLimitError).code).toBe("SOCKETDUCT_SET_SPOOL_LIMIT")
|
|
1067
|
+
at(budgets, 2).reserveControl(6)
|
|
1068
|
+
let queueLimitError
|
|
1069
|
+
try {
|
|
1070
|
+
at(budgets, 1).reserveControl(1)
|
|
1071
|
+
} catch (error) {
|
|
1072
|
+
queueLimitError = /** @type {Error} */ (error)
|
|
1073
|
+
}
|
|
1074
|
+
expect(queueLimitError).toBeInstanceOf(Error)
|
|
1075
|
+
expect(/** @type {{code?: string}} */ (queueLimitError).code).toBe("SOCKETDUCT_SET_QUEUE_LIMIT")
|
|
1076
|
+
expect(set.status().budgets).toEqual({
|
|
1077
|
+
dataBytes: 6,
|
|
1078
|
+
spoolBytes: 6,
|
|
1079
|
+
queuedBytes: 12,
|
|
1080
|
+
maxDataBytes: 10,
|
|
1081
|
+
maxSpoolBytes: 8,
|
|
1082
|
+
maxQueuedBytes: 12
|
|
1083
|
+
})
|
|
1084
|
+
at(budgets, 0).releaseMemory(6)
|
|
1085
|
+
at(budgets, 0).releaseSpool(6)
|
|
1086
|
+
at(budgets, 2).releaseControl(6)
|
|
1087
|
+
await set.close()
|
|
1088
|
+
})
|
|
987
1089
|
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1090
|
+
it("close during startup cannot resurrect members and concurrent close is ordered", async () => {
|
|
1091
|
+
const firstError = new Error("starlink close")
|
|
1092
|
+
const thirdError = new Error("o2 close")
|
|
1093
|
+
const {set, members} = fixture({}, {starlink: firstError, o2gateway: thirdError})
|
|
1094
|
+
const start = set.start()
|
|
1095
|
+
const firstClose = set.close()
|
|
1096
|
+
const secondClose = set.close()
|
|
1097
|
+
expect(firstClose).toBe(secondClose)
|
|
1098
|
+
let startError
|
|
1099
|
+
try {
|
|
1100
|
+
await start
|
|
1101
|
+
} catch (error) {
|
|
1102
|
+
startError = /** @type {Error} */ (error)
|
|
1103
|
+
}
|
|
1104
|
+
expect(startError).toBeInstanceOf(Error)
|
|
1105
|
+
expect(/** @type {{code?: string}} */ (startError).code).toBe("SOCKETDUCT_TRANSPORT_SET_CLOSED")
|
|
1106
|
+
let closeError
|
|
1107
|
+
try {
|
|
1108
|
+
await firstClose
|
|
1109
|
+
} catch (error) {
|
|
1110
|
+
closeError = /** @type {Error} */ (error)
|
|
1111
|
+
}
|
|
1112
|
+
expect(closeError).toBeInstanceOf(AggregateError)
|
|
1113
|
+
expect(/** @type {AggregateError} */ (closeError).errors).toEqual([firstError, thirdError])
|
|
1114
|
+
expect(members.map((member) => member.closeCalls)).toEqual([1, 1, 1])
|
|
1115
|
+
members.forEach((member) => member.ready())
|
|
1116
|
+
expect(set.status().state).toBe("closed")
|
|
1117
|
+
expect(() => set.createConnection({host: "target.internal", port: 443})).toThrow(/closed/i)
|
|
1118
|
+
})
|
|
1006
1119
|
})
|