socketduct 0.1.32 → 0.1.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261026183000-route-class-availability-scope.md +1 -0
- package/package.json +2 -2
- package/src/client/multiplexed-transport-set.js +15 -8
- package/src/reverse/client.js +29 -143
- package/test/integration/multiplexed-transport.test.js +160 -0
- package/test/integration/reverse-route-class.test.js +44 -47
- package/test/unit/client/multiplexed-transport-set.test.js +24 -15
- package/changelog.d/20261025123000-route-class-down-registration.md +0 -1
- package/changelog.d/20261025203000-node-24-lts.md +0 -5
- package/changelog.d/20261026100000-reverse-client-drain.md +0 -1
- package/test/integration/multiplexed-drain.test.js +0 -179
- package/test/integration/reverse-drain.test.js +0 -138
|
@@ -0,0 +1 @@
|
|
|
1
|
+
A reset for one stream no longer marks the member's whole route class unavailable, so a single unreachable target cannot disable every route class and stall all later opens in a transport set.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "socketduct",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.33",
|
|
4
4
|
"description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.js",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"./relay": "./src/relay.js"
|
|
17
17
|
},
|
|
18
18
|
"engines": {
|
|
19
|
-
"node": "
|
|
19
|
+
"node": ">=24.0.0"
|
|
20
20
|
},
|
|
21
21
|
"scripts": {
|
|
22
22
|
"gateway:deploy": "node scripts/deploy-gateway.js",
|
|
@@ -494,14 +494,13 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
|
|
|
494
494
|
return this
|
|
495
495
|
}
|
|
496
496
|
|
|
497
|
-
/** @returns {{state: string, configuredMembers: number, connectingMembers: number, readyMembers: number, degradedMembers: number, closedMembers: number, minimumReadyMembers: number, pendingOpens: number, maxPendingOpens: number, virtualOpens: number, physicalConnects: number, reconnectAttempts: number, budgets: {dataBytes: number, spoolBytes: number, queuedBytes: number, maxDataBytes: number, maxSpoolBytes: number, maxQueuedBytes: number}, members: Array<{gatewayId?: string, routeClass?: string, state: string,
|
|
497
|
+
/** @returns {{state: string, configuredMembers: number, connectingMembers: number, readyMembers: number, degradedMembers: number, closedMembers: number, minimumReadyMembers: number, pendingOpens: number, maxPendingOpens: number, virtualOpens: number, physicalConnects: number, reconnectAttempts: number, budgets: {dataBytes: number, spoolBytes: number, queuedBytes: number, maxDataBytes: number, maxSpoolBytes: number, maxQueuedBytes: number}, members: Array<{gatewayId?: string, routeClass?: string, state: string, assignments: number, physicalConnects: number, reconnectAttempts: number, reconnectSuccesses: number, virtualStreams: number, queuedBytes: number, spoolBytes: number, spoolCleanupFailures: number, terminalErrorCode: string | null}>}} Privacy-safe aggregate and per-member status. */
|
|
498
498
|
status() {
|
|
499
499
|
const members = this.#members.map((member) => {
|
|
500
500
|
const status = member.connection.status()
|
|
501
501
|
return {
|
|
502
502
|
...memberAllocation(member),
|
|
503
503
|
state: status.physicalState,
|
|
504
|
-
gatewayUnavailable: status.gatewayUnavailable === true,
|
|
505
504
|
assignments: member.assignments,
|
|
506
505
|
physicalConnects: status.physicalConnects,
|
|
507
506
|
reconnectAttempts: status.reconnectAttempts,
|
|
@@ -578,25 +577,33 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
|
|
|
578
577
|
}
|
|
579
578
|
|
|
580
579
|
/**
|
|
581
|
-
* Selects the next ready member whose
|
|
582
|
-
*
|
|
583
|
-
*
|
|
584
|
-
*
|
|
585
|
-
* (the member keeps probing the hub, and its state change re-drains this set).
|
|
580
|
+
* Selects the next ready member, preferring members whose gateway the hub has
|
|
581
|
+
* not recently reported unavailable. When every ready member carries a
|
|
582
|
+
* gateway-unavailable mark, any ready member may still be selected so a stale
|
|
583
|
+
* mark can never wedge the set.
|
|
586
584
|
* @returns {{member: TransportMember, previousCursor: number} | undefined}
|
|
587
585
|
*/
|
|
588
586
|
#selectReadyMember() {
|
|
589
587
|
if (this.#closed) throw closedError()
|
|
590
588
|
const previousCursor = this.#cursor
|
|
589
|
+
/** @type {{member: TransportMember, index: number} | undefined} */
|
|
590
|
+
let fallback
|
|
591
591
|
for (let offset = 0; offset < this.#members.length; offset += 1) {
|
|
592
592
|
const index = (this.#cursor + offset) % this.#members.length
|
|
593
593
|
const member = /** @type {TransportMember} */ (this.#members[index])
|
|
594
594
|
const status = member.connection.status()
|
|
595
595
|
if (status.physicalState !== "ready") continue
|
|
596
|
-
if (status.gatewayUnavailable === true)
|
|
596
|
+
if (status.gatewayUnavailable === true) {
|
|
597
|
+
fallback ??= {member, index}
|
|
598
|
+
continue
|
|
599
|
+
}
|
|
597
600
|
this.#cursor = (index + 1) % this.#members.length
|
|
598
601
|
return {member, previousCursor}
|
|
599
602
|
}
|
|
603
|
+
if (fallback !== undefined) {
|
|
604
|
+
this.#cursor = (fallback.index + 1) % this.#members.length
|
|
605
|
+
return {member: fallback.member, previousCursor}
|
|
606
|
+
}
|
|
600
607
|
return undefined
|
|
601
608
|
}
|
|
602
609
|
|
package/src/reverse/client.js
CHANGED
|
@@ -32,8 +32,6 @@ import {attachDisconnectAttribution, createDisconnectAttribution, parseDisconnec
|
|
|
32
32
|
const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
|
|
33
33
|
const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
|
|
34
34
|
const MANIFEST_PAGE_STREAMS = 64
|
|
35
|
-
const DEFAULT_GATEWAY_PROBE_DELAY_MS = 1_000
|
|
36
|
-
const DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS = 30_000
|
|
37
35
|
|
|
38
36
|
/**
|
|
39
37
|
* @typedef {object} ReverseGatewayConnectionOptions
|
|
@@ -52,8 +50,6 @@ const DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS = 30_000
|
|
|
52
50
|
* @property {number} [connectTimeoutMs] - TLS/auth deadline.
|
|
53
51
|
* @property {number} [reconnectDelayMs] - Initial reconnect delay.
|
|
54
52
|
* @property {number} [maxReconnectDelayMs] - Maximum reconnect delay.
|
|
55
|
-
* @property {number} [gatewayProbeDelayMs] - Initial re-reach delay while the hub reports no selectable gateway.
|
|
56
|
-
* @property {number} [maxGatewayProbeDelayMs] - Maximum re-reach delay while the hub reports no selectable gateway.
|
|
57
53
|
* @property {number} [initialGeneration] - Monotonic seed for controlled replacement; defaults to wall-clock microseconds.
|
|
58
54
|
* @property {boolean} [continuousReconnect] - Keep the member warm after loss, default true.
|
|
59
55
|
* @property {import('../protocol/compression.js').CompressionLimits} [compressionLimits] - zstd safety bounds.
|
|
@@ -144,9 +140,6 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
144
140
|
/** @type {number} */ #generation
|
|
145
141
|
/** @type {string | undefined} */ #codec
|
|
146
142
|
/** @type {boolean} */ #closing = false
|
|
147
|
-
/** @type {boolean} */ #draining = false
|
|
148
|
-
/** @type {Promise<void> | undefined} */ #closePromise
|
|
149
|
-
/** @type {Array<() => void>} */ #drainWaiters = []
|
|
150
143
|
/** @type {boolean} */ #detached = false
|
|
151
144
|
/** @type {boolean} */ #preserveSpools = false
|
|
152
145
|
/** @type {boolean} */ #fatal = false
|
|
@@ -175,10 +168,7 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
175
168
|
/** @type {number} */ #spoolBytes = 0
|
|
176
169
|
/** @type {number} */ #spoolCleanupFailures = 0
|
|
177
170
|
/** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
|
|
178
|
-
/** @type {number | undefined} - Wall-clock time the hub
|
|
179
|
-
/** @type {ReturnType<typeof setTimeout> | undefined} */ #gatewayProbeTimer
|
|
180
|
-
/** @type {number} */ #gatewayProbeDelay
|
|
181
|
-
/** @type {number} */ #gatewayProbeAttempts = 0
|
|
171
|
+
/** @type {number | undefined} - Wall-clock time of the last hub-reported gateway unavailability. */ #gatewayUnavailableSince
|
|
182
172
|
/** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */ #disconnectAttribution
|
|
183
173
|
|
|
184
174
|
/** @param {ReverseGatewayConnectionOptions} options - Hub route and safety policy. */
|
|
@@ -194,7 +184,6 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
194
184
|
})
|
|
195
185
|
this.#generation = options.initialGeneration ?? Date.now() * 1000
|
|
196
186
|
this.#reconnectDelay = options.reconnectDelayMs ?? 250
|
|
197
|
-
this.#gatewayProbeDelay = options.gatewayProbeDelayMs ?? DEFAULT_GATEWAY_PROBE_DELAY_MS
|
|
198
187
|
this.#sendWindowLimit = this.#initialWindow()
|
|
199
188
|
this.createConnection = this.createConnection.bind(this)
|
|
200
189
|
}
|
|
@@ -250,7 +239,6 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
250
239
|
/** Creates a fresh or synchronously recovered reverse stream. */
|
|
251
240
|
#createConnection(/** @type {import('../client/multiplexed-connection.js').VirtualConnectionOptions} */ options, /** @type {((error: Error | null, socket: VirtualSocket) => void) | undefined} */ callback, /** @type {boolean} */ recovering) {
|
|
252
241
|
if (this.#closing) throw closedError()
|
|
253
|
-
if (this.#draining) throw Object.assign(new Error("Reverse gateway member is draining"), {code: "SOCKETDUCT_TRANSPORT_SET_DRAINING"})
|
|
254
242
|
if (options?.manualReceiveAck !== undefined && typeof options.manualReceiveAck !== "boolean") throw new TypeError("manualReceiveAck must be a boolean")
|
|
255
243
|
if (!recovering && this.#state !== "ready") throw Object.assign(new Error("Reverse gateway member is not ready"), {code: "SOCKETDUCT_TRANSPORT_SET_NOT_READY"})
|
|
256
244
|
if (this.#streams.size >= (this.#options.maxStreams ?? DEFAULT_MAX_STREAMS)) throw new Error("Maximum reverse virtual stream count exceeded")
|
|
@@ -372,92 +360,30 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
372
360
|
* @returns {void}
|
|
373
361
|
*/
|
|
374
362
|
#markGatewayUnavailable(code) {
|
|
375
|
-
const firstMark = this.#
|
|
376
|
-
this.#
|
|
363
|
+
const firstMark = this.#gatewayUnavailableSince === undefined
|
|
364
|
+
this.#gatewayUnavailableSince = Date.now()
|
|
377
365
|
if (firstMark) {
|
|
378
366
|
logDiagnostic(this.#options.logger, "reverse_client.gateway_unavailable", {
|
|
379
367
|
code,
|
|
380
368
|
generation: this.#generation,
|
|
381
369
|
...this.#routeIdentityFields()
|
|
382
370
|
})
|
|
383
|
-
this.#emitStateChange()
|
|
384
371
|
}
|
|
385
|
-
this.#scheduleGatewayProbe()
|
|
386
372
|
}
|
|
387
373
|
|
|
388
374
|
/**
|
|
389
|
-
*
|
|
390
|
-
*
|
|
391
|
-
*
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
this.#gatewayProbeAttempts = 0
|
|
399
|
-
if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
|
|
400
|
-
this.#gatewayProbeTimer = undefined
|
|
401
|
-
logDiagnostic(this.#options.logger, "reverse_client.gateway_available", {
|
|
402
|
-
downMs,
|
|
403
|
-
generation: this.#generation,
|
|
404
|
-
...this.#routeIdentityFields()
|
|
405
|
-
})
|
|
406
|
-
this.#emitStateChange()
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
/**
|
|
410
|
-
* Keeps reaching the hub while this member is down, so a returning gateway is
|
|
411
|
-
* detected and taken into service again without operator action. Only a fresh
|
|
412
|
-
* hub selection can report a route class available, so the probe re-selects.
|
|
413
|
-
* @returns {void}
|
|
414
|
-
*/
|
|
415
|
-
#scheduleGatewayProbe() {
|
|
416
|
-
if (this.#closing || this.#fatal || this.#gatewayDownSince === undefined || this.#gatewayProbeTimer !== undefined) return
|
|
417
|
-
const delay = this.#gatewayProbeDelay
|
|
418
|
-
this.#gatewayProbeDelay = Math.min(this.#gatewayProbeDelay * 2, this.#options.maxGatewayProbeDelayMs ?? DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS)
|
|
419
|
-
this.#gatewayProbeTimer = setTimeout(() => {
|
|
420
|
-
this.#gatewayProbeTimer = undefined
|
|
421
|
-
this.#probeGateway()
|
|
422
|
-
}, delay)
|
|
423
|
-
if (!this.#referenced) this.#gatewayProbeTimer.unref?.()
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
/** Re-runs hub selection for a down route class and re-reads its availability. @returns {void} */
|
|
427
|
-
#probeGateway() {
|
|
428
|
-
if (this.#closing || this.#fatal || this.#gatewayDownSince === undefined) return
|
|
429
|
-
this.#gatewayProbeAttempts += 1
|
|
430
|
-
logDiagnostic(this.#options.logger, "reverse_client.gateway_probe", {
|
|
431
|
-
attempt: this.#gatewayProbeAttempts,
|
|
432
|
-
generation: this.#generation,
|
|
433
|
-
physicalState: this.#state,
|
|
434
|
-
...this.#routeIdentityFields()
|
|
435
|
-
})
|
|
436
|
-
const physical = this.#physical
|
|
437
|
-
if (physical !== undefined && !physical.destroyed) {
|
|
438
|
-
physical.destroy()
|
|
439
|
-
return
|
|
440
|
-
}
|
|
441
|
-
if (this.#connectPromise === undefined) {
|
|
442
|
-
this.#connectPromise = new Promise((resolve, reject) => { this.#resolveConnect = resolve; this.#rejectConnect = reject })
|
|
443
|
-
this.#connectPromise.catch(() => {})
|
|
444
|
-
}
|
|
445
|
-
this.#connectPhysical()
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
/** Notifies owners that availability changed without changing the physical state. @returns {void} */
|
|
449
|
-
#emitStateChange() {
|
|
450
|
-
this.emit("stateChange", this.#state, undefined)
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
/**
|
|
454
|
-
* Whether the hub currently has this member's route class registered down. The
|
|
455
|
-
* state never expires on its own: a down member stays out of service until the
|
|
456
|
-
* hub reports a selectable gateway again.
|
|
457
|
-
* @returns {boolean} True while the route class is registered down.
|
|
375
|
+
* Whether the hub last reported this member's route class as having no selectable
|
|
376
|
+
* gateway. The report is a property of the live connection, not a short-lived cache:
|
|
377
|
+
* it arrives once per connection at selection time and only changes when the hub says
|
|
378
|
+
* otherwise or an open succeeds. Expiring it on a timer made the transport set pick a
|
|
379
|
+
* route class the hub had already declared unserviceable, and the reset that followed
|
|
380
|
+
* failed the caller's operation — a build lost its first transport open this way.
|
|
381
|
+
* The transport set still falls back to marked members when every member is marked, so
|
|
382
|
+
* a mark can never wedge the set.
|
|
383
|
+
* @returns {boolean} True while the hub reports no selectable gateway for this member.
|
|
458
384
|
*/
|
|
459
385
|
#gatewayUnavailable() {
|
|
460
|
-
return this.#
|
|
386
|
+
return this.#gatewayUnavailableSince !== undefined
|
|
461
387
|
}
|
|
462
388
|
|
|
463
389
|
status() {
|
|
@@ -467,7 +393,6 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
467
393
|
return {
|
|
468
394
|
physicalState: this.#state,
|
|
469
395
|
gatewayUnavailable: this.#gatewayUnavailable(),
|
|
470
|
-
gatewayProbeAttempts: this.#gatewayProbeAttempts,
|
|
471
396
|
connectionGeneration: this.#generation,
|
|
472
397
|
virtualStreams: this.#streams.size,
|
|
473
398
|
streamStates,
|
|
@@ -491,52 +416,13 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
491
416
|
ref() { this.#referenced = true; this.#physical?.ref(); this.#reconnectTimer?.ref?.(); return this }
|
|
492
417
|
unref() { this.#referenced = false; this.#physical?.unref(); this.#reconnectTimer?.unref?.(); return this }
|
|
493
418
|
|
|
494
|
-
/**
|
|
495
|
-
* Graceful counterpart of `close()`: refuses new virtual streams, lets the open
|
|
496
|
-
* ones finish on their own, then releases the shared transport. Transport sets
|
|
497
|
-
* that are torn down while they carry work must use this instead of `close()`,
|
|
498
|
-
* otherwise live streams are reset at the relay mid-transfer.
|
|
499
|
-
* @returns {Promise<void>}
|
|
500
|
-
*/
|
|
501
|
-
async drain() {
|
|
502
|
-
if (this.#closePromise !== undefined) return this.#closePromise
|
|
503
|
-
this.#draining = true
|
|
504
|
-
this.#closePromise = (async () => {
|
|
505
|
-
logDiagnostic(this.#options.logger, "reverse_client.draining", {virtualStreams: this.#streams.size})
|
|
506
|
-
await this.#waitForStreamsToDrain()
|
|
507
|
-
await this.close()
|
|
508
|
-
})()
|
|
509
|
-
|
|
510
|
-
return this.#closePromise
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
/** @returns {Promise<void>} */
|
|
514
|
-
#waitForStreamsToDrain() {
|
|
515
|
-
if (this.#streams.size === 0) return Promise.resolve()
|
|
516
|
-
|
|
517
|
-
return new Promise((resolve) => { this.#drainWaiters.push(resolve) })
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
/** @returns {void} */
|
|
521
|
-
#releaseDrainWaiters() {
|
|
522
|
-
if (this.#drainWaiters.length === 0) return
|
|
523
|
-
const waiters = this.#drainWaiters.splice(0)
|
|
524
|
-
for (const resolve of waiters) resolve()
|
|
525
|
-
}
|
|
526
|
-
|
|
527
419
|
async close() {
|
|
528
420
|
if (this.#closing) return
|
|
529
|
-
if (this.#streams.size > 0 && !this.#draining) {
|
|
530
|
-
logDiagnostic(this.#options.logger, "reverse_client.closed_with_live_streams", {streamsKilled: this.#streams.size})
|
|
531
|
-
}
|
|
532
421
|
this.#closing = true
|
|
533
422
|
if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
|
|
534
423
|
this.#reconnectTimer = undefined
|
|
535
|
-
if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
|
|
536
|
-
this.#gatewayProbeTimer = undefined
|
|
537
424
|
this.#setState("closed")
|
|
538
425
|
for (const stream of [...this.#streams.values()]) stream.socket.destroy()
|
|
539
|
-
this.#releaseDrainWaiters()
|
|
540
426
|
const physical = this.#physical
|
|
541
427
|
this.#physical = undefined
|
|
542
428
|
if (physical !== undefined && !physical.destroyed) {
|
|
@@ -560,8 +446,6 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
560
446
|
this.#closing = true
|
|
561
447
|
if (this.#reconnectTimer !== undefined) clearTimeout(this.#reconnectTimer)
|
|
562
448
|
this.#reconnectTimer = undefined
|
|
563
|
-
if (this.#gatewayProbeTimer !== undefined) clearTimeout(this.#gatewayProbeTimer)
|
|
564
|
-
this.#gatewayProbeTimer = undefined
|
|
565
449
|
this.#setState("closed")
|
|
566
450
|
const physical = this.#physical
|
|
567
451
|
physical?.pause()
|
|
@@ -774,7 +658,7 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
774
658
|
const selection = this.#routeIdentityFields()
|
|
775
659
|
if (detail.generation !== generation || ("gatewayId" in selection ? detail.gatewayId !== selection.gatewayId || Object.hasOwn(detail, "routeClass") : detail.routeClass !== selection.routeClass || Object.hasOwn(detail, "gatewayId"))) throw fatalError("GATEWAY_SELECTION_FAILED")
|
|
776
660
|
if (typeof detail.resumed !== "boolean") throw fatalError("GATEWAY_SELECTION_FAILED")
|
|
777
|
-
if (detail.selectable === true) this.#
|
|
661
|
+
if (detail.selectable === true) this.#gatewayUnavailableSince = undefined
|
|
778
662
|
else if (detail.selectable === false) this.#markGatewayUnavailable("GATEWAY_SELECT_UNAVAILABLE")
|
|
779
663
|
selected = true
|
|
780
664
|
if (detail.resumed) {
|
|
@@ -901,7 +785,13 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
901
785
|
stream.sendWindow = Math.min(stream.sendWindow, detail.initialWindow)
|
|
902
786
|
stream.state = stream.localEnded ? "half-closed" : "open"
|
|
903
787
|
stream.opened = true
|
|
904
|
-
this.#
|
|
788
|
+
if (this.#gatewayUnavailableSince !== undefined) {
|
|
789
|
+
this.#gatewayUnavailableSince = undefined
|
|
790
|
+
logDiagnostic(this.#options.logger, "reverse_client.gateway_available", {
|
|
791
|
+
generation: this.#generation,
|
|
792
|
+
...this.#routeIdentityFields()
|
|
793
|
+
})
|
|
794
|
+
}
|
|
905
795
|
logDiagnostic(this.#options.logger, "virtual_stream.opened", {
|
|
906
796
|
streamId: stream.id,
|
|
907
797
|
generation: this.#generation,
|
|
@@ -955,9 +845,14 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
955
845
|
const detail = decodeV3Control(frame.payload)
|
|
956
846
|
const code = typeof detail.code === "string" ? detail.code : "STREAM_RESET"
|
|
957
847
|
stream.state = "reset"
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
848
|
+
// A stream reset describes this stream's target, not the health of the
|
|
849
|
+
// member's whole route class. The hub resolves gateways per (route class,
|
|
850
|
+
// target) and the transport set retries a failed open across members, so a
|
|
851
|
+
// single unreachable target would otherwise mark every route class down.
|
|
852
|
+
// Such a mark never expires on its own, which deadlocks the set: nothing is
|
|
853
|
+
// selected, so no open can succeed, so no mark can ever be cleared.
|
|
854
|
+
// Route-class availability is owned by the hub's GATEWAY_SELECTED response,
|
|
855
|
+
// which is class-scoped and re-read on every probe.
|
|
961
856
|
const error = attachDisconnectAttribution(new SocketductStreamError("Reverse gateway stream was reset", code), validHubAttribution(detail) ?? (code === "GATEWAY_DISCONNECTED" ? this.#disconnectAttribution : undefined))
|
|
962
857
|
stream.socket._receiveError(error)
|
|
963
858
|
return
|
|
@@ -1244,7 +1139,6 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
1244
1139
|
while (stream.pendingWrites.length > 0) this.#finishWrite(stream, pendingWriteError)
|
|
1245
1140
|
if (!this.#preserveSpools) this.#trackSpoolCleanup(stream)
|
|
1246
1141
|
this.#progressPendingWrites()
|
|
1247
|
-
if (this.#streams.size === 0) this.#releaseDrainWaiters()
|
|
1248
1142
|
}
|
|
1249
1143
|
|
|
1250
1144
|
/** @param {ReverseClientStream} stream - Stream. @param {number} sequence - ACK. */
|
|
@@ -1330,11 +1224,6 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
1330
1224
|
this.#reconnectDelay = Math.min(this.#reconnectDelay * 2, this.#options.maxReconnectDelayMs ?? 30_000)
|
|
1331
1225
|
this.#reconnectTimer = setTimeout(() => {
|
|
1332
1226
|
this.#reconnectTimer = undefined
|
|
1333
|
-
// clearTimeout() cannot stop a callback that is already queued, so a
|
|
1334
|
-
// detach()/close() that lands in that window used to reconnect anyway:
|
|
1335
|
-
// it reopened a physical transport after the caller had deliberately
|
|
1336
|
-
// detached (observed as ENOENT reading a deleted recovery CA file).
|
|
1337
|
-
if (this.#closing || this.#fatal) return
|
|
1338
1227
|
if (this.#connectPromise === undefined) {
|
|
1339
1228
|
this.#connectPromise = new Promise((resolve, reject) => { this.#resolveConnect = resolve; this.#rejectConnect = reject })
|
|
1340
1229
|
this.#connectPromise.catch(() => {})
|
|
@@ -1452,9 +1341,6 @@ function validateOptions(options) {
|
|
|
1452
1341
|
positiveInteger(options.maxConnectionBufferBytes, "maxConnectionBufferBytes", DEFAULT_CONNECTION_BUFFER_BYTES)
|
|
1453
1342
|
if (options.spoolDirectory !== undefined && (typeof options.spoolDirectory !== "string" || options.spoolDirectory.length === 0)) throw new TypeError("spoolDirectory must be a non-empty string")
|
|
1454
1343
|
positiveInteger(options.maxAgeMs, "maxAgeMs", 24 * 60 * 60 * 1000)
|
|
1455
|
-
const gatewayProbeDelay = positiveInteger(options.gatewayProbeDelayMs, "gatewayProbeDelayMs", DEFAULT_GATEWAY_PROBE_DELAY_MS)
|
|
1456
|
-
const maxGatewayProbeDelay = positiveInteger(options.maxGatewayProbeDelayMs, "maxGatewayProbeDelayMs", DEFAULT_MAX_GATEWAY_PROBE_DELAY_MS)
|
|
1457
|
-
if (maxGatewayProbeDelay < gatewayProbeDelay) throw new RangeError("maxGatewayProbeDelayMs must not be smaller than gatewayProbeDelayMs")
|
|
1458
1344
|
if (options.initialGeneration !== undefined && (!Number.isSafeInteger(options.initialGeneration) || options.initialGeneration < 0)) throw new RangeError("initialGeneration must be a non-negative safe integer")
|
|
1459
1345
|
}
|
|
1460
1346
|
|
|
@@ -207,3 +207,163 @@ describe("multiplexed transport", () => {
|
|
|
207
207
|
sockets.forEach((socket) => socket.destroy())
|
|
208
208
|
}))
|
|
209
209
|
})
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* `close()` terminates live virtual streams, which the relay answers with a
|
|
213
|
+
* stream reset — that is how in-flight work on the other side of a transport
|
|
214
|
+
* set died when the owning process shut down. `drain()` is the graceful verb:
|
|
215
|
+
* it refuses new streams, waits for the open ones, and only then releases the
|
|
216
|
+
* transport.
|
|
217
|
+
*/
|
|
218
|
+
|
|
219
|
+
/** @returns {Promise<void>} */
|
|
220
|
+
async function settleTurns() {
|
|
221
|
+
for (let index = 0; index < 10; index += 1) await new Promise((resolve) => setImmediate(resolve))
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* @param {Array<import('../../src/client/virtual-socket.js').VirtualSocket>} streams - Open streams.
|
|
226
|
+
* @param {number} index - Index.
|
|
227
|
+
* @returns {import('../../src/client/virtual-socket.js').VirtualSocket} - The opened stream.
|
|
228
|
+
*/
|
|
229
|
+
function streamAt(streams, index) {
|
|
230
|
+
const stream = streams[index]
|
|
231
|
+
|
|
232
|
+
if (stream === undefined) throw new Error(`Expected stream ${index} to be open`)
|
|
233
|
+
|
|
234
|
+
return stream
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* @param {(callback: () => Promise<void> | void) => void} after - Cleanup registrar.
|
|
239
|
+
* @param {number} streamCount - Streams to open.
|
|
240
|
+
*/
|
|
241
|
+
async function openDrainStreams(after, streamCount) {
|
|
242
|
+
/** @type {Set<import('node:net').Socket>} */
|
|
243
|
+
const targetSockets = new Set()
|
|
244
|
+
const target = createServer((socket) => {
|
|
245
|
+
targetSockets.add(socket)
|
|
246
|
+
socket.on("close", () => targetSockets.delete(socket))
|
|
247
|
+
socket.on("data", (chunk) => socket.write(chunk))
|
|
248
|
+
})
|
|
249
|
+
const targetPort = await listen(target)
|
|
250
|
+
|
|
251
|
+
after(async () => {
|
|
252
|
+
for (const socket of targetSockets) socket.destroy()
|
|
253
|
+
await new Promise((resolve) => target.close(() => resolve(undefined)))
|
|
254
|
+
})
|
|
255
|
+
|
|
256
|
+
const relay = await startMultiplexedTcpRelay({
|
|
257
|
+
tokens: new StaticTokenStore([{username: "edge", token: "valid-token"}]),
|
|
258
|
+
authorizeTarget: ({target: requested}) => requested.host === "127.0.0.1" && requested.port === targetPort
|
|
259
|
+
})
|
|
260
|
+
|
|
261
|
+
after(() => relay.close())
|
|
262
|
+
|
|
263
|
+
const connection = new MultiplexedSocketductConnection({
|
|
264
|
+
relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"}
|
|
265
|
+
})
|
|
266
|
+
|
|
267
|
+
after(() => connection.close())
|
|
268
|
+
/** @type {Array<import('../../src/client/virtual-socket.js').VirtualSocket>} */
|
|
269
|
+
const streams = []
|
|
270
|
+
|
|
271
|
+
for (let index = 0; index < streamCount; index += 1) {
|
|
272
|
+
streams.push(connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: `drain-${index}`}))
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
await Promise.all(streams.map((stream) => onceEvent(stream, "connect")))
|
|
276
|
+
expect(connection.status().virtualStreams).toBe(streamCount)
|
|
277
|
+
|
|
278
|
+
return {connection, relay, streams, targetPort}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
describe("multiplexed transport drain", () => {
|
|
282
|
+
it("drain() keeps in-flight virtual streams alive until they finish", async () => withTestCleanup(async (after) => {
|
|
283
|
+
const {connection, relay, streams} = await openDrainStreams(after, 2)
|
|
284
|
+
|
|
285
|
+
const draining = connection.drain()
|
|
286
|
+
|
|
287
|
+
await settleTurns()
|
|
288
|
+
|
|
289
|
+
// drain() must not tear down streams that are still open.
|
|
290
|
+
expect(connection.status().virtualStreams).toBe(2)
|
|
291
|
+
expect(streamAt(streams, 0).destroyed).toBe(false)
|
|
292
|
+
expect(streamAt(streams, 1).destroyed).toBe(false)
|
|
293
|
+
// The physical transport must not be dropped while streams are live.
|
|
294
|
+
expect(relay.status().authenticatedConnections).toBe(1)
|
|
295
|
+
|
|
296
|
+
for (const stream of streams) stream.destroy()
|
|
297
|
+
await draining
|
|
298
|
+
|
|
299
|
+
expect(connection.status().virtualStreams).toBe(0)
|
|
300
|
+
// The physical transport is released once the last stream ends.
|
|
301
|
+
expect(relay.status().authenticatedConnections).toBe(0)
|
|
302
|
+
}))
|
|
303
|
+
|
|
304
|
+
it("drain() keeps working streams usable instead of resetting them", async () => withTestCleanup(async (after) => {
|
|
305
|
+
const {connection, streams} = await openDrainStreams(after, 1)
|
|
306
|
+
const draining = connection.drain()
|
|
307
|
+
|
|
308
|
+
await settleTurns()
|
|
309
|
+
|
|
310
|
+
streamAt(streams, 0).write("still-works")
|
|
311
|
+
/** @type {string} */
|
|
312
|
+
let echoed = ""
|
|
313
|
+
|
|
314
|
+
await new Promise((resolve, reject) => {
|
|
315
|
+
/** @param {Buffer} chunk - Echoed data. */
|
|
316
|
+
const onData = (chunk) => {
|
|
317
|
+
echoed += chunk.toString()
|
|
318
|
+
if (echoed.length < 11) return
|
|
319
|
+
streamAt(streams, 0).off("error", reject)
|
|
320
|
+
resolve(undefined)
|
|
321
|
+
}
|
|
322
|
+
streamAt(streams, 0).on("error", reject)
|
|
323
|
+
streamAt(streams, 0).on("data", onData)
|
|
324
|
+
})
|
|
325
|
+
|
|
326
|
+
// Data written during the drain must still reach the target and come back.
|
|
327
|
+
expect(echoed).toBe("still-works")
|
|
328
|
+
|
|
329
|
+
streamAt(streams, 0).destroy()
|
|
330
|
+
await draining
|
|
331
|
+
}))
|
|
332
|
+
|
|
333
|
+
it("drain() rejects new virtual streams while draining", async () => withTestCleanup(async (after) => {
|
|
334
|
+
const {connection, streams, targetPort} = await openDrainStreams(after, 1)
|
|
335
|
+
const draining = connection.drain()
|
|
336
|
+
|
|
337
|
+
await settleTurns()
|
|
338
|
+
|
|
339
|
+
// A draining connection must refuse new virtual streams instead of opening
|
|
340
|
+
// them on a doomed transport.
|
|
341
|
+
/** @type {unknown} */
|
|
342
|
+
let caughtError
|
|
343
|
+
|
|
344
|
+
try {
|
|
345
|
+
connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "late"})
|
|
346
|
+
} catch (error) {
|
|
347
|
+
caughtError = error
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
expect(caughtError).toBeInstanceOf(Error)
|
|
351
|
+
expect(String(/** @type {Error} */ (caughtError).message)).toMatch(/connection is closed/i)
|
|
352
|
+
// The stream that was already open must survive.
|
|
353
|
+
expect(connection.status().virtualStreams).toBe(1)
|
|
354
|
+
|
|
355
|
+
streamAt(streams, 0).destroy()
|
|
356
|
+
await draining
|
|
357
|
+
}))
|
|
358
|
+
|
|
359
|
+
it("close() still terminates in-flight virtual streams immediately", async () => withTestCleanup(async (after) => {
|
|
360
|
+
const {connection, relay, streams} = await openDrainStreams(after, 2)
|
|
361
|
+
|
|
362
|
+
await connection.close()
|
|
363
|
+
|
|
364
|
+
expect(connection.status().virtualStreams).toBe(0)
|
|
365
|
+
expect(streamAt(streams, 0).destroyed).toBe(true)
|
|
366
|
+
expect(streamAt(streams, 1).destroyed).toBe(true)
|
|
367
|
+
expect(relay.status().authenticatedConnections).toBe(0)
|
|
368
|
+
}))
|
|
369
|
+
})
|
|
@@ -56,15 +56,6 @@ async function waitUntil(predicate, label) {
|
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
/** @param {() => boolean} predicate - Completion predicate. @param {string} label - Failure label. @param {number} timeoutMs - Deadline. */
|
|
60
|
-
async function waitUntilWithin(predicate, label, timeoutMs) {
|
|
61
|
-
const deadline = Date.now() + timeoutMs
|
|
62
|
-
while (!predicate()) {
|
|
63
|
-
if (Date.now() >= deadline) throw new Error(`Timed out: ${label}`)
|
|
64
|
-
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
59
|
/** @param {string} prefix - Echo response prefix. */
|
|
69
60
|
async function createEchoTarget(prefix) {
|
|
70
61
|
const sockets = new Set()
|
|
@@ -480,58 +471,42 @@ describe("reverse gateway route-class", () => {
|
|
|
480
471
|
stream.destroy()
|
|
481
472
|
}))
|
|
482
473
|
|
|
483
|
-
it("a
|
|
484
|
-
const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-
|
|
474
|
+
it("a reset for one target without a gateway does not mark the whole route class unavailable", async () => withTestCleanup(async (after) => {
|
|
475
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-target-scope-"))
|
|
485
476
|
after(() => rm(root, {recursive: true, force: true}))
|
|
486
477
|
const certificates = generateTestCertificates(root)
|
|
487
478
|
const targetA = await createEchoTarget("a:")
|
|
488
479
|
after(() => targetA.close())
|
|
480
|
+
const targetB = await createEchoTarget("b:")
|
|
481
|
+
after(() => targetB.close())
|
|
489
482
|
|
|
483
|
+
// The route class is serviceable, but only target A has a configured gateway.
|
|
484
|
+
// Target B has none, so the hub resets opens for it with GATEWAY_UNAVAILABLE.
|
|
485
|
+
// That statement is about target B, not about the route class as a whole.
|
|
490
486
|
const hub = await startReverseGatewayHub({
|
|
491
487
|
tokens: new StaticTokenStore([
|
|
492
488
|
{username: "home-a", token: "gateway-a-token"},
|
|
493
489
|
{username: "tensorbuzz", token: "client-token"}
|
|
494
490
|
]),
|
|
495
491
|
tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
|
|
496
|
-
knownGatewayIds: ["server-a-starlink"],
|
|
492
|
+
knownGatewayIds: ["server-a-starlink", "server-b-starlink"],
|
|
497
493
|
knownRouteClasses: ["starlink"],
|
|
498
|
-
authorizeGateway: ({principal, gatewayId}) => principal.username === "home-a" && gatewayId === "server-a-starlink",
|
|
494
|
+
authorizeGateway: ({principal, gatewayId}) => principal.username === "home-a" && (gatewayId === "server-a-starlink" || gatewayId === "server-b-starlink"),
|
|
499
495
|
authorizeClientGateway: () => false,
|
|
500
496
|
authorizeClientRouteClass: ({principal, routeClass}) => principal.username === "tensorbuzz" && routeClass === "starlink",
|
|
501
|
-
resolveRouteClassGateways: () => ["server-a-starlink"],
|
|
502
|
-
resolveOpenGateways: ({routeClass, target}) =>
|
|
503
|
-
|
|
497
|
+
resolveRouteClassGateways: () => ["server-a-starlink", "server-b-starlink"],
|
|
498
|
+
resolveOpenGateways: ({routeClass, target}) => {
|
|
499
|
+
if (routeClass !== "starlink") return []
|
|
500
|
+
return target.port === targetA.port ? ["server-a-starlink"] : ["server-b-starlink"]
|
|
501
|
+
},
|
|
502
|
+
authorizeOpen: ({gatewayId, target}) => (gatewayId === "server-a-starlink" && target.port === targetA.port) || (gatewayId === "server-b-starlink" && target.port === targetB.port)
|
|
504
503
|
})
|
|
505
504
|
after(() => hub.close())
|
|
506
505
|
const tls = {caFile: certificates.caCertFile, servername: "localhost"}
|
|
507
506
|
|
|
508
|
-
const client = new ReverseGatewayConnection({
|
|
509
|
-
routeClass: "starlink",
|
|
510
|
-
connectionId: "route_class_down_client",
|
|
511
|
-
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
512
|
-
tls,
|
|
513
|
-
gatewayProbeDelayMs: 20,
|
|
514
|
-
maxGatewayProbeDelayMs: 40,
|
|
515
|
-
reconnectDelayMs: 10,
|
|
516
|
-
maxReconnectDelayMs: 20
|
|
517
|
-
})
|
|
518
|
-
after(() => client.close())
|
|
519
|
-
|
|
520
|
-
await client.connect()
|
|
521
|
-
expect(client.status().gatewayUnavailable).toBe(true)
|
|
522
|
-
|
|
523
|
-
// The member keeps reaching the hub while its route class is down.
|
|
524
|
-
await waitUntilWithin(() => client.status().gatewayProbeAttempts >= 3, "gateway re-reaching", 4_000)
|
|
525
|
-
|
|
526
|
-
// A down registration never expires on its own: the previous five-second
|
|
527
|
-
// mark would have silently returned this member to service with no gateway.
|
|
528
|
-
await new Promise((resolve) => setTimeout(resolve, 5_100))
|
|
529
|
-
expect(client.status().gatewayUnavailable).toBe(true)
|
|
530
|
-
expect(client.status().gatewayProbeAttempts >= 3).toBe(true)
|
|
531
|
-
|
|
532
507
|
const gatewayA = new OutboundReverseGateway({
|
|
533
508
|
gatewayId: "server-a-starlink",
|
|
534
|
-
connectionId: "
|
|
509
|
+
connectionId: "target_scope_gateway_a",
|
|
535
510
|
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
|
|
536
511
|
tls,
|
|
537
512
|
authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port,
|
|
@@ -542,11 +517,33 @@ describe("reverse gateway route-class", () => {
|
|
|
542
517
|
await gatewayA.start()
|
|
543
518
|
await waitUntil(() => hub.status().healthyGateways === 1, "gateway A registration")
|
|
544
519
|
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
520
|
+
const client = new ReverseGatewayConnection({
|
|
521
|
+
routeClass: "starlink",
|
|
522
|
+
connectionId: "target_scope_client",
|
|
523
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
524
|
+
tls
|
|
525
|
+
})
|
|
526
|
+
after(() => client.close())
|
|
527
|
+
await client.connect()
|
|
528
|
+
expect(client.status().gatewayUnavailable).toBe(false)
|
|
529
|
+
|
|
530
|
+
// Target B's gateway is configured in the route table but is not registered
|
|
531
|
+
// (the maintenance/offline case), so the hub resets its opens as unavailable.
|
|
532
|
+
const unavailableStream = client.createConnection({host: targetB.host, port: targetB.port})
|
|
533
|
+
const reset = await nextError(unavailableStream)
|
|
534
|
+
expect(reset.code).toBe("GATEWAY_UNAVAILABLE")
|
|
535
|
+
unavailableStream.destroy()
|
|
536
|
+
|
|
537
|
+
// The route class is healthy, so the reset must not take it out of service.
|
|
538
|
+
// A member marked unavailable this way is never selected again, which would
|
|
539
|
+
// stall every later open in the set until the caller's own deadline.
|
|
540
|
+
expect({value: client.status().gatewayUnavailable, context: "a target-scoped reset must not mark the route class unavailable"}).toMatchObject({value: false})
|
|
541
|
+
|
|
542
|
+
// A healthy target on the same route class still opens and exchanges data.
|
|
543
|
+
const healthyStream = client.createConnection({host: targetA.host, port: targetA.port})
|
|
544
|
+
await onceEvent(healthyStream, "connect")
|
|
545
|
+
expect(await exchange(healthyStream, "probe")).toBe("a:probe")
|
|
546
|
+
healthyStream.destroy()
|
|
551
547
|
}))
|
|
548
|
+
|
|
552
549
|
})
|
|
@@ -558,30 +558,16 @@ describe("multiplexed transport set", () => {
|
|
|
558
558
|
await set.close()
|
|
559
559
|
})
|
|
560
560
|
|
|
561
|
-
it("a
|
|
561
|
+
it("a stale gateway-unavailable mark cannot wedge the set when every ready member is marked", async () => {
|
|
562
562
|
const {set, members} = fixture()
|
|
563
563
|
const started = set.start()
|
|
564
564
|
members.forEach((member) => member.ready())
|
|
565
565
|
await started
|
|
566
566
|
members.forEach((member) => { member.gatewayUnavailable = true })
|
|
567
|
-
let callbacks = 0
|
|
568
567
|
const socket = set.createConnection({host: "target.internal", port: 443}, (error) => {
|
|
569
568
|
expect(error).toBe(null)
|
|
570
|
-
callbacks += 1
|
|
571
569
|
})
|
|
572
|
-
await new Promise((resolve) => setImmediate(resolve))
|
|
573
|
-
expect(callbacks).toBe(0)
|
|
574
|
-
expect(socket.gatewayId).toBe(undefined)
|
|
575
|
-
expect(set.status().pendingOpens).toBe(1)
|
|
576
|
-
expect(members.map((member) => member.opens)).toEqual([0, 0, 0])
|
|
577
|
-
expect(set.status().members.map((member) => member.gatewayUnavailable)).toEqual([true, true, true])
|
|
578
|
-
|
|
579
|
-
at(members, 2).gatewayUnavailable = false
|
|
580
|
-
at(members, 2).setState("ready")
|
|
581
|
-
await new Promise((resolve) => setImmediate(resolve))
|
|
582
|
-
expect(callbacks).toBe(1)
|
|
583
570
|
expect(socket.gatewayId !== undefined).toBe(true)
|
|
584
|
-
expect(members.map((member) => member.opens)).toEqual([0, 0, 1])
|
|
585
571
|
expect(set.status().pendingOpens).toBe(0)
|
|
586
572
|
await set.close()
|
|
587
573
|
})
|
|
@@ -1136,4 +1122,27 @@ describe("multiplexed transport set", () => {
|
|
|
1136
1122
|
expect(set.status().state).toBe("closed")
|
|
1137
1123
|
expect(() => set.createConnection({host: "target.internal", port: 443})).toThrow(/closed/i)
|
|
1138
1124
|
})
|
|
1125
|
+
|
|
1126
|
+
it("drain() drains every member instead of closing them", async () => {
|
|
1127
|
+
const {set, members} = fixture()
|
|
1128
|
+
const started = set.start()
|
|
1129
|
+
const draining = set.drain()
|
|
1130
|
+
|
|
1131
|
+
/** @type {unknown} */
|
|
1132
|
+
let startError
|
|
1133
|
+
|
|
1134
|
+
try {
|
|
1135
|
+
await started
|
|
1136
|
+
} catch (error) {
|
|
1137
|
+
startError = error
|
|
1138
|
+
}
|
|
1139
|
+
await draining
|
|
1140
|
+
|
|
1141
|
+
expect(/** @type {{code?: string}} */ (startError).code).toBe("SOCKETDUCT_TRANSPORT_SET_CLOSED")
|
|
1142
|
+
expect(members.length).toBe(3)
|
|
1143
|
+
expect(members.map((member) => member.drainCalls)).toEqual([1, 1, 1])
|
|
1144
|
+
expect(members.map((member) => member.closeCalls)).toEqual([0, 0, 0])
|
|
1145
|
+
expect(set.status().state).toBe("closed")
|
|
1146
|
+
expect(() => set.createConnection({host: "target.internal", port: 443})).toThrow(/closed/i)
|
|
1147
|
+
})
|
|
1139
1148
|
})
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
A route class the hub reports down is never selected while it stays down: the member keeps re-reaching the hub on a bounded backoff and returns to service automatically once a gateway registers again, and `status()` now exposes the down state and the probe attempts.
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
## Node.js 24 LTS
|
|
2
|
-
|
|
3
|
-
The package, its test suite and the reverse-gateway integration tests are validated on Node.js 24 (current LTS). `engines.node` is now pinned to `^24.0.0`.
|
|
4
|
-
|
|
5
|
-
Why: on this development host the suite is also run under Node.js 26, where the compressed reverse-gateway round-trip tests fail with corrupted payloads (16 failures, `Buffer wasn't equal to Buffer`). The same suite passes 42/42 under Node.js 24, so the corruption tracks the runtime, not the wire format. Pinning to LTS keeps the validated runtime and the declared support range identical.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
`ReverseGatewayConnection` gains `drain()`, matching the multiplexed connection: it refuses new virtual streams with `SOCKETDUCT_TRANSPORT_SET_DRAINING`, waits for the open ones to finish, and only then releases the hub transport, so tearing down a reverse-mode transport set no longer resets work that is still in flight; `close()` keeps its immediate-termination behaviour and now logs how many live streams it terminated.
|
|
@@ -1,179 +0,0 @@
|
|
|
1
|
-
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-returns */
|
|
2
|
-
import {describe, expect, it, withTestCleanup} from "@velocious/testing"
|
|
3
|
-
import {createServer} from "node:net"
|
|
4
|
-
|
|
5
|
-
import {StaticTokenStore} from "../../src/auth/static-token-store.js"
|
|
6
|
-
import {MultiplexedSocketductConnection} from "../../src/client/multiplexed-connection.js"
|
|
7
|
-
import {startMultiplexedTcpRelay} from "../../src/proxy/multiplexed-relay.js"
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* `close()` terminates live virtual streams, which the relay answers with a
|
|
11
|
-
* stream reset — that is how in-flight work on the other side of a transport
|
|
12
|
-
* set died when the owning process shut down. `drain()` is the graceful verb:
|
|
13
|
-
* it refuses new streams, waits for the open ones, then releases the transport.
|
|
14
|
-
*/
|
|
15
|
-
|
|
16
|
-
/** @param {import('node:net').Server} server - Server. @returns {Promise<number>} */
|
|
17
|
-
async function listen(server) {
|
|
18
|
-
await new Promise((resolve, reject) => {
|
|
19
|
-
server.once("error", reject)
|
|
20
|
-
server.listen(0, "127.0.0.1", () => {
|
|
21
|
-
server.off("error", reject)
|
|
22
|
-
resolve(undefined)
|
|
23
|
-
})
|
|
24
|
-
})
|
|
25
|
-
const address = server.address()
|
|
26
|
-
|
|
27
|
-
if (address === null || typeof address === "string") throw new Error("Expected a TCP address")
|
|
28
|
-
|
|
29
|
-
return address.port
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* @param {import('node:events').EventEmitter} emitter - Emitter.
|
|
34
|
-
* @param {string} event - Event.
|
|
35
|
-
* @returns {Promise<void>}
|
|
36
|
-
*/
|
|
37
|
-
async function onceEvent(emitter, event) {
|
|
38
|
-
await new Promise((resolve, reject) => {
|
|
39
|
-
emitter.once("error", reject)
|
|
40
|
-
emitter.once(event, () => {
|
|
41
|
-
emitter.off("error", reject)
|
|
42
|
-
resolve(undefined)
|
|
43
|
-
})
|
|
44
|
-
})
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* @param {Array<import("../../src/client/virtual-socket.js").VirtualSocket>} streams - Open streams.
|
|
49
|
-
* @param {number} index - Index.
|
|
50
|
-
* @returns {import("../../src/client/virtual-socket.js").VirtualSocket} - The stream.
|
|
51
|
-
*/
|
|
52
|
-
function streamAt(streams, index) {
|
|
53
|
-
const stream = streams[index]
|
|
54
|
-
|
|
55
|
-
if (stream === undefined) throw new Error(`Expected stream ${index} to be open`)
|
|
56
|
-
|
|
57
|
-
return stream
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
/** @returns {Promise<void>} */
|
|
61
|
-
async function settleTurns() {
|
|
62
|
-
for (let index = 0; index < 10; index += 1) await new Promise((resolve) => setImmediate(resolve))
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Relay plus one client connection with `streamCount` echoed streams open.
|
|
67
|
-
* @param {(cleanup: () => void | Promise<void>) => void} after - Cleanup registrar.
|
|
68
|
-
* @param {number} streamCount - Streams to open.
|
|
69
|
-
*/
|
|
70
|
-
async function openStreams(after, streamCount) {
|
|
71
|
-
/** @type {Set<import('node:net').Socket>} */
|
|
72
|
-
const targetSockets = new Set()
|
|
73
|
-
const target = createServer((socket) => {
|
|
74
|
-
targetSockets.add(socket)
|
|
75
|
-
socket.on("close", () => targetSockets.delete(socket))
|
|
76
|
-
socket.on("data", (chunk) => socket.write(chunk))
|
|
77
|
-
})
|
|
78
|
-
const targetPort = await listen(target)
|
|
79
|
-
|
|
80
|
-
after(() => {
|
|
81
|
-
for (const socket of targetSockets) socket.destroy()
|
|
82
|
-
|
|
83
|
-
return new Promise((resolve) => target.close(() => resolve(undefined)))
|
|
84
|
-
})
|
|
85
|
-
|
|
86
|
-
const relay = await startMultiplexedTcpRelay({
|
|
87
|
-
tokens: new StaticTokenStore([{username: "edge", token: "valid-token"}]),
|
|
88
|
-
authorizeTarget: ({target: requested}) => requested.host === "127.0.0.1" && requested.port === targetPort
|
|
89
|
-
})
|
|
90
|
-
|
|
91
|
-
after(() => relay.close())
|
|
92
|
-
|
|
93
|
-
const connection = new MultiplexedSocketductConnection({
|
|
94
|
-
relay: {host: "127.0.0.1", port: relay.port, token: "valid-token"}
|
|
95
|
-
})
|
|
96
|
-
|
|
97
|
-
after(() => connection.close())
|
|
98
|
-
|
|
99
|
-
/** @type {Array<import("../../src/client/virtual-socket.js").VirtualSocket>} */
|
|
100
|
-
const streams = []
|
|
101
|
-
|
|
102
|
-
for (let index = 0; index < streamCount; index += 1) {
|
|
103
|
-
streams.push(connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: `drain-${index}`}))
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
await Promise.all(streams.map((stream) => onceEvent(stream, "connect")))
|
|
107
|
-
expect(connection.status().virtualStreams).toBe(streamCount)
|
|
108
|
-
|
|
109
|
-
return {connection, relay, streams, targetPort}
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
describe("multiplexed connection drain", () => {
|
|
113
|
-
it("drain() keeps in-flight virtual streams alive until they finish", async () => withTestCleanup(async (after) => {
|
|
114
|
-
const {connection, relay, streams} = await openStreams(after, 2)
|
|
115
|
-
const draining = connection.drain()
|
|
116
|
-
|
|
117
|
-
await settleTurns()
|
|
118
|
-
|
|
119
|
-
expect(connection.status().virtualStreams).toBe(2)
|
|
120
|
-
expect(streamAt(streams, 0).destroyed).toBeFalse()
|
|
121
|
-
expect(streamAt(streams, 1).destroyed).toBeFalse()
|
|
122
|
-
expect(relay.status().authenticatedConnections).toBe(1)
|
|
123
|
-
|
|
124
|
-
for (const stream of streams) stream.destroy()
|
|
125
|
-
await draining
|
|
126
|
-
|
|
127
|
-
expect(connection.status().virtualStreams).toBe(0)
|
|
128
|
-
expect(relay.status().authenticatedConnections).toBe(0)
|
|
129
|
-
}))
|
|
130
|
-
|
|
131
|
-
it("drain() keeps working streams usable instead of resetting them", async () => withTestCleanup(async (after) => {
|
|
132
|
-
const {connection, streams} = await openStreams(after, 1)
|
|
133
|
-
const draining = connection.drain()
|
|
134
|
-
|
|
135
|
-
await settleTurns()
|
|
136
|
-
|
|
137
|
-
streamAt(streams, 0).write("still-works")
|
|
138
|
-
|
|
139
|
-
/** @type {string} */
|
|
140
|
-
let echoed = ""
|
|
141
|
-
|
|
142
|
-
await new Promise((resolve, reject) => {
|
|
143
|
-
streamAt(streams, 0).once("error", reject)
|
|
144
|
-
streamAt(streams, 0).on("data", (chunk) => {
|
|
145
|
-
echoed += chunk.toString()
|
|
146
|
-
if (echoed.length >= 11) resolve(undefined)
|
|
147
|
-
})
|
|
148
|
-
})
|
|
149
|
-
|
|
150
|
-
expect(echoed).toBe("still-works")
|
|
151
|
-
|
|
152
|
-
streamAt(streams, 0).destroy()
|
|
153
|
-
await draining
|
|
154
|
-
}))
|
|
155
|
-
|
|
156
|
-
it("drain() rejects new virtual streams while draining", async () => withTestCleanup(async (after) => {
|
|
157
|
-
const {connection, streams, targetPort} = await openStreams(after, 1)
|
|
158
|
-
const draining = connection.drain()
|
|
159
|
-
|
|
160
|
-
await settleTurns()
|
|
161
|
-
|
|
162
|
-
expect(() => connection.createConnection({host: "127.0.0.1", port: targetPort, sessionName: "late"})).toThrow(/connection is closed/i)
|
|
163
|
-
expect(connection.status().virtualStreams).toBe(1)
|
|
164
|
-
|
|
165
|
-
streamAt(streams, 0).destroy()
|
|
166
|
-
await draining
|
|
167
|
-
}))
|
|
168
|
-
|
|
169
|
-
it("close() still terminates in-flight virtual streams immediately", async () => withTestCleanup(async (after) => {
|
|
170
|
-
const {connection, relay, streams} = await openStreams(after, 2)
|
|
171
|
-
|
|
172
|
-
await connection.close()
|
|
173
|
-
|
|
174
|
-
expect(connection.status().virtualStreams).toBe(0)
|
|
175
|
-
expect(streamAt(streams, 0).destroyed).toBeTrue()
|
|
176
|
-
expect(streamAt(streams, 1).destroyed).toBeTrue()
|
|
177
|
-
expect(relay.status().authenticatedConnections).toBe(0)
|
|
178
|
-
}))
|
|
179
|
-
})
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
|
|
2
|
-
import {describe, expect, it, withTestCleanup} from "@velocious/testing"
|
|
3
|
-
import {mkdtemp, rm} from "node:fs/promises"
|
|
4
|
-
import {createServer} from "node:net"
|
|
5
|
-
import {tmpdir} from "node:os"
|
|
6
|
-
import {join} from "node:path"
|
|
7
|
-
|
|
8
|
-
import {StaticTokenStore} from "../../src/auth/static-token-store.js"
|
|
9
|
-
import {OutboundReverseGateway, ReverseGatewayConnection, startReverseGatewayHub} from "../../src/reverse/index.js"
|
|
10
|
-
import {generateTestCertificates} from "../helpers/tls-certificates.js"
|
|
11
|
-
|
|
12
|
-
/** @param {import('node:net').Server} server - Server. */
|
|
13
|
-
async function listen(server) {
|
|
14
|
-
await new Promise((resolve, reject) => {
|
|
15
|
-
server.once("error", reject)
|
|
16
|
-
server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
|
|
17
|
-
})
|
|
18
|
-
const address = server.address()
|
|
19
|
-
if (address === null || typeof address === "string") throw new Error("Server did not bind to a TCP port")
|
|
20
|
-
return address.port
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/** @param {string} prefix - Echo prefix. */
|
|
24
|
-
async function createEchoTarget(prefix) {
|
|
25
|
-
const server = createServer((socket) => {
|
|
26
|
-
socket.on("data", (chunk) => socket.write(`${prefix}${chunk.toString("utf8")}`))
|
|
27
|
-
})
|
|
28
|
-
const port = await listen(server)
|
|
29
|
-
return {host: "127.0.0.1", port, close: () => new Promise((resolve) => server.close(resolve))}
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/** @param {import('node:events').EventEmitter} emitter - Source. @param {string} event - Event. */
|
|
33
|
-
async function onceEvent(emitter, event) {
|
|
34
|
-
await new Promise((resolve, reject) => {
|
|
35
|
-
emitter.once("error", reject)
|
|
36
|
-
emitter.once(event, () => { emitter.off("error", reject); resolve(undefined) })
|
|
37
|
-
})
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/** @param {import('node:stream').Duplex} socket - Stream. @param {string} message - Payload. */
|
|
41
|
-
async function exchange(socket, message) {
|
|
42
|
-
return await new Promise((resolve) => {
|
|
43
|
-
socket.once("data", (chunk) => resolve(chunk.toString("utf8")))
|
|
44
|
-
socket.write(message)
|
|
45
|
-
})
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/** @param {() => boolean} predicate - Completion predicate. @param {string} label - Failure label. */
|
|
49
|
-
async function waitUntil(predicate, label) {
|
|
50
|
-
const deadline = Date.now() + 4_000
|
|
51
|
-
while (!predicate()) {
|
|
52
|
-
if (Date.now() >= deadline) throw new Error(`Timed out: ${label}`)
|
|
53
|
-
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
describe("reverse gateway drain", () => {
|
|
58
|
-
it("refuses new streams, lets open streams finish, then releases the transport", async () => withTestCleanup(async (after) => {
|
|
59
|
-
const root = await mkdtemp(join(tmpdir(), "socketduct-reverse-drain-"))
|
|
60
|
-
after(() => rm(root, {recursive: true, force: true}))
|
|
61
|
-
const certificates = generateTestCertificates(root)
|
|
62
|
-
const target = await createEchoTarget("t:")
|
|
63
|
-
after(() => target.close())
|
|
64
|
-
|
|
65
|
-
const tokens = new StaticTokenStore([
|
|
66
|
-
{username: "home-a", token: "gateway-a-token"},
|
|
67
|
-
{username: "tensorbuzz", token: "client-token"}
|
|
68
|
-
])
|
|
69
|
-
const hub = await startReverseGatewayHub({
|
|
70
|
-
tokens,
|
|
71
|
-
tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
|
|
72
|
-
knownGatewayIds: ["server-a-starlink"],
|
|
73
|
-
knownRouteClasses: ["starlink"],
|
|
74
|
-
authorizeGateway: ({gatewayId}) => gatewayId === "server-a-starlink",
|
|
75
|
-
authorizeClientGateway: ({gatewayId}) => gatewayId === "server-a-starlink",
|
|
76
|
-
authorizeClientRouteClass: () => true,
|
|
77
|
-
resolveOpenGateways: ({routeClass, target: requestedTarget}) => (
|
|
78
|
-
routeClass === "starlink" && requestedTarget.port === target.port ? ["server-a-starlink"] : []
|
|
79
|
-
),
|
|
80
|
-
authorizeOpen: ({target: requestedTarget}) => requestedTarget.host === target.host && requestedTarget.port === target.port
|
|
81
|
-
})
|
|
82
|
-
after(() => hub.close())
|
|
83
|
-
const tls = {caFile: certificates.caCertFile, servername: "localhost"}
|
|
84
|
-
const gateway = new OutboundReverseGateway({
|
|
85
|
-
gatewayId: "server-a-starlink",
|
|
86
|
-
connectionId: "drain_gateway_a",
|
|
87
|
-
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
|
|
88
|
-
tls,
|
|
89
|
-
authorizeTarget: (requestedTarget) => requestedTarget.host === target.host && requestedTarget.port === target.port,
|
|
90
|
-
reconnectDelayMs: 10,
|
|
91
|
-
maxReconnectDelayMs: 20
|
|
92
|
-
})
|
|
93
|
-
after(() => gateway.close())
|
|
94
|
-
await gateway.start()
|
|
95
|
-
|
|
96
|
-
const client = new ReverseGatewayConnection({
|
|
97
|
-
routeClass: "starlink",
|
|
98
|
-
connectionId: "drain_client",
|
|
99
|
-
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
100
|
-
tls
|
|
101
|
-
})
|
|
102
|
-
await client.connect()
|
|
103
|
-
expect(hub.status().clientPhysicalConnections).toBe(1)
|
|
104
|
-
|
|
105
|
-
const stream = client.createConnection({host: target.host, port: target.port})
|
|
106
|
-
await onceEvent(stream, "connect")
|
|
107
|
-
expect(await exchange(stream, "before")).toBe("t:before")
|
|
108
|
-
expect(client.status().virtualStreams).toBe(1)
|
|
109
|
-
|
|
110
|
-
expect(typeof client.drain).toBe("function")
|
|
111
|
-
|
|
112
|
-
let drained = false
|
|
113
|
-
const drainPromise = client.drain().then(() => { drained = true })
|
|
114
|
-
|
|
115
|
-
// New work is refused while the set drains.
|
|
116
|
-
/** @type {Error & {code?: string} | undefined} */
|
|
117
|
-
let refused
|
|
118
|
-
try {
|
|
119
|
-
client.createConnection({host: target.host, port: target.port})
|
|
120
|
-
} catch (caught) {
|
|
121
|
-
refused = /** @type {Error & {code?: string}} */ (caught)
|
|
122
|
-
}
|
|
123
|
-
expect(refused?.code).toBe("SOCKETDUCT_TRANSPORT_SET_DRAINING")
|
|
124
|
-
|
|
125
|
-
// The transport stays up while the open stream is still usable.
|
|
126
|
-
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
127
|
-
expect(drained).toBe(false)
|
|
128
|
-
expect(await exchange(stream, "during")).toBe("t:during")
|
|
129
|
-
expect(hub.status().clientPhysicalConnections).toBe(1)
|
|
130
|
-
|
|
131
|
-
// Finishing the last stream lets the drain complete and release the transport.
|
|
132
|
-
stream.destroy()
|
|
133
|
-
await drainPromise
|
|
134
|
-
expect(drained).toBe(true)
|
|
135
|
-
await waitUntil(() => hub.status().clientPhysicalConnections === 0, "hub releases the drained member")
|
|
136
|
-
expect(client.status().physicalState).toBe("closed")
|
|
137
|
-
}))
|
|
138
|
-
})
|