socketduct 0.1.25 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/changelog.d/20261010130509-bound-log-frame-compression-memory.md +1 -0
- package/changelog.d/20261025070000-reclaim-unreadable-retention-lock.md +14 -0
- package/docs/runbooks/jsonl-integrity-rollout.md +8 -6
- package/package.json +2 -2
- package/scripts/test.js +9 -14
- package/src/logging/compressed-log-writer.js +1 -1
- package/src/logging/log-retention.js +36 -1
- package/src/logging/zstd-frame-codec.js +15 -1
- package/test/e2e/gateway-process.test.js +93 -92
- package/test/e2e/logger-shutdown-process.test.js +40 -37
- package/test/e2e/reverse-client-recovery-process.test.js +107 -103
- package/test/e2e/reverse-gateway-process.test.js +328 -318
- package/test/integration/initial-target-recovery.test.js +333 -334
- package/test/integration/logging-cli.test.js +121 -119
- package/test/integration/logging-compressed-recovery.test.js +96 -95
- package/test/integration/multiplexed-compatibility.test.js +178 -177
- package/test/integration/multiplexed-corrections.test.js +360 -359
- package/test/integration/multiplexed-failures.test.js +253 -251
- package/test/integration/multiplexed-transport-set.test.js +334 -322
- package/test/integration/multiplexed-transport.test.js +135 -134
- package/test/integration/reverse-gateway-stream-id-reset.test.js +90 -89
- package/test/integration/reverse-gateway.test.js +2309 -2218
- package/test/integration/reverse-resume-continuity.test.js +219 -218
- package/test/integration/reverse-route-class.test.js +358 -341
- package/test/integration/socketduct-http-agent.test.js +314 -312
- package/test/integration/socketduct-https-agent.test.js +516 -485
- package/test/integration/stream-compression.test.js +862 -824
- package/test/integration/tcp-forwarding.test.js +2550 -2447
- package/test/integration/tls-forwarding.test.js +593 -592
- package/test/unit/auth/auth.test.js +43 -44
- package/test/unit/auth/static-token-store.test.js +9 -8
- package/test/unit/client/connect.test.js +41 -40
- package/test/unit/client/deferred-virtual-socket.test.js +95 -94
- package/test/unit/client/multiplexed-spool-budget.test.js +155 -154
- package/test/unit/client/multiplexed-transport-set.test.js +830 -748
- package/test/unit/client/socketduct-socket.test.js +768 -771
- package/test/unit/client/virtual-socket.test.js +198 -168
- package/test/unit/config/docker-gateway-files.test.js +224 -224
- package/test/unit/config/environment.test.js +472 -492
- package/test/unit/index.test.js +53 -52
- package/test/unit/logging/compressed-log-worker.test.js +23 -22
- package/test/unit/logging/compressed-log-writer.test.js +155 -154
- package/test/unit/logging/log-retention.test.js +71 -48
- package/test/unit/logging/log-segment-index.test.js +84 -81
- package/test/unit/logging/zstd-frame-codec.test.js +28 -19
- package/test/unit/protocol/compression.test.js +254 -298
- package/test/unit/protocol/frame.test.js +280 -276
- package/test/unit/protocol/v3-compression.test.js +21 -23
- package/test/unit/protocol/v3-frame.test.js +42 -41
- package/test/unit/release-configuration.test.js +15 -12
- package/test/unit/reverse/disconnect-attribution.test.js +65 -64
- package/test/unit/reverse-terminal-outcomes.test.js +53 -50
- package/test/unit/server/control-handler.test.js +232 -231
- package/test/unit/server/primary.test.js +51 -49
- package/test/unit/server/relay-runtime.test.js +280 -294
- package/test/unit/sessions/session.test.js +48 -62
- package/test/unit/spool/spool-file.test.js +401 -385
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/require-param, jsdoc/require-returns */
|
|
2
|
-
import
|
|
2
|
+
import {describe, expect, it, withTestCleanup} from "@velocious/testing"
|
|
3
3
|
import {mkdtemp, rm} from "node:fs/promises"
|
|
4
4
|
import {createServer} from "node:net"
|
|
5
5
|
import {tmpdir} from "node:os"
|
|
6
6
|
import {join} from "node:path"
|
|
7
|
-
import test from "node:test"
|
|
8
7
|
|
|
9
8
|
import {StaticTokenStore} from "../../src/auth/static-token-store.js"
|
|
10
9
|
import {V3FrameType} from "../../src/protocol/v3-frame.js"
|
|
@@ -18,7 +17,7 @@ async function listen(server) {
|
|
|
18
17
|
server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(undefined) })
|
|
19
18
|
})
|
|
20
19
|
const address = server.address()
|
|
21
|
-
|
|
20
|
+
if (address === null || typeof address === "string") throw new Error("Server did not bind to a TCP port")
|
|
22
21
|
return address.port
|
|
23
22
|
}
|
|
24
23
|
|
|
@@ -86,365 +85,383 @@ async function exchange(socket, payload) {
|
|
|
86
85
|
return await withTimeout(response, `response for ${payload}`)
|
|
87
86
|
}
|
|
88
87
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
88
|
+
describe("reverse gateway route-class", () => {
|
|
89
|
+
it("route-class clients fail closed per OPEN while unaffected routes and legacy exact clients remain usable", async () => withTestCleanup(async (after) => {
|
|
90
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-"))
|
|
91
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
92
|
+
const certificates = generateTestCertificates(root)
|
|
93
|
+
const targetA = await createEchoTarget("a:")
|
|
94
|
+
const targetB = await createEchoTarget("b:")
|
|
95
|
+
after(() => targetA.close())
|
|
96
|
+
after(() => targetB.close())
|
|
97
97
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
98
|
+
const tokens = new StaticTokenStore([
|
|
99
|
+
{username: "home-a", token: "gateway-a-token"},
|
|
100
|
+
{username: "home-b", token: "gateway-b-token"},
|
|
101
|
+
{username: "tensorbuzz", token: "client-token"},
|
|
102
|
+
{username: "untrusted", token: "untrusted-token"}
|
|
103
|
+
])
|
|
104
|
+
let ambiguousTargetA = false
|
|
105
|
+
const hub = await startReverseGatewayHub({
|
|
106
|
+
tokens,
|
|
107
|
+
tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
|
|
108
|
+
knownGatewayIds: ["server-a-starlink", "server-b-starlink"],
|
|
109
|
+
knownRouteClasses: ["starlink"],
|
|
110
|
+
authorizeGateway: ({principal, gatewayId}) => (
|
|
111
|
+
(principal.username === "home-a" && gatewayId === "server-a-starlink")
|
|
112
|
+
|| (principal.username === "home-b" && gatewayId === "server-b-starlink")
|
|
113
|
+
),
|
|
114
|
+
authorizeClientGateway: ({principal, gatewayId}) => principal.username === "tensorbuzz" && gatewayId === "server-a-starlink",
|
|
115
|
+
authorizeClientRouteClass: ({principal}) => principal.username === "tensorbuzz",
|
|
116
|
+
resolveOpenGateways: ({principal, routeClass, target}) => {
|
|
117
|
+
if (principal.username !== "tensorbuzz" || routeClass !== "starlink" || target.host !== "127.0.0.1") return []
|
|
118
|
+
if (target.port === targetA.port) return ambiguousTargetA ? ["server-a-starlink", "server-b-starlink"] : ["server-a-starlink"]
|
|
119
|
+
if (target.port === targetB.port) return ["server-b-starlink"]
|
|
120
|
+
return []
|
|
121
|
+
},
|
|
122
|
+
authorizeOpen: ({principal, gatewayId, target}) => principal.username === "tensorbuzz" && (
|
|
123
|
+
(gatewayId === "server-a-starlink" && target.host === targetA.host && target.port === targetA.port)
|
|
124
|
+
|| (gatewayId === "server-b-starlink" && target.host === targetB.host && target.port === targetB.port)
|
|
125
|
+
)
|
|
126
|
+
})
|
|
127
|
+
after(() => hub.close())
|
|
128
|
+
const tls = {caFile: certificates.caCertFile, servername: "localhost"}
|
|
129
|
+
const gatewayA = new OutboundReverseGateway({
|
|
130
|
+
gatewayId: "server-a-starlink",
|
|
131
|
+
connectionId: "route_class_gateway_a",
|
|
132
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
|
|
133
|
+
tls,
|
|
134
|
+
authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port,
|
|
135
|
+
reconnectDelayMs: 10,
|
|
136
|
+
maxReconnectDelayMs: 20
|
|
137
|
+
})
|
|
138
|
+
const gatewayB = new OutboundReverseGateway({
|
|
139
|
+
gatewayId: "server-b-starlink",
|
|
140
|
+
connectionId: "route_class_gateway_b",
|
|
141
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-b-token"},
|
|
142
|
+
tls,
|
|
143
|
+
authorizeTarget: (target) => target.host === targetB.host && target.port === targetB.port,
|
|
144
|
+
reconnectDelayMs: 10,
|
|
145
|
+
maxReconnectDelayMs: 20
|
|
146
|
+
})
|
|
147
|
+
after(() => gatewayA.close())
|
|
148
|
+
after(() => gatewayB.close())
|
|
149
|
+
await Promise.all([gatewayA.start(), gatewayB.start()])
|
|
150
150
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
151
|
+
const unknown = new ReverseGatewayConnection({
|
|
152
|
+
routeClass: "missing",
|
|
153
|
+
connectionId: "route_class_unknown_client",
|
|
154
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
155
|
+
tls,
|
|
156
|
+
continuousReconnect: false
|
|
157
|
+
})
|
|
158
|
+
after(() => unknown.close())
|
|
159
|
+
/** @type {Error & {code?: string} | undefined} */
|
|
160
|
+
let error
|
|
161
|
+
try {
|
|
162
|
+
await withTimeout(unknown.connect(), "unknown route class")
|
|
163
|
+
} catch (caught) {
|
|
164
|
+
error = /** @type {Error & {code?: string}} */ (caught)
|
|
165
|
+
}
|
|
166
|
+
expect(error).toBeDefined()
|
|
167
|
+
expect(error?.code).toBe("UNKNOWN_ROUTE_CLASS")
|
|
160
168
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
169
|
+
const unauthorized = new ReverseGatewayConnection({
|
|
170
|
+
routeClass: "starlink",
|
|
171
|
+
connectionId: "route_class_unauthorized_client",
|
|
172
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "untrusted-token"},
|
|
173
|
+
tls,
|
|
174
|
+
continuousReconnect: false
|
|
175
|
+
})
|
|
176
|
+
after(() => unauthorized.close())
|
|
177
|
+
/** @type {Error & {code?: string} | undefined} */
|
|
178
|
+
let unauthorizedError
|
|
179
|
+
try {
|
|
180
|
+
await withTimeout(unauthorized.connect(), "unauthorized route class")
|
|
181
|
+
} catch (caught) {
|
|
182
|
+
unauthorizedError = /** @type {Error & {code?: string}} */ (caught)
|
|
183
|
+
}
|
|
184
|
+
expect(unauthorizedError).toBeDefined()
|
|
185
|
+
expect(unauthorizedError?.code).toBe("ROUTE_CLASS_UNAUTHORIZED")
|
|
170
186
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
187
|
+
const client = new ReverseGatewayConnection({
|
|
188
|
+
routeClass: "starlink",
|
|
189
|
+
connectionId: "route_class_tensorbuzz_client",
|
|
190
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
191
|
+
tls
|
|
192
|
+
})
|
|
193
|
+
after(() => client.close())
|
|
194
|
+
await client.connect()
|
|
195
|
+
expect(client.status().physicalConnects).toBe(1)
|
|
196
|
+
expect(hub.status().clientPhysicalConnections).toBe(1)
|
|
181
197
|
|
|
182
|
-
|
|
183
|
-
|
|
198
|
+
const denied = client.createConnection({host: "127.0.0.1", port: 65_534})
|
|
199
|
+
expect((await nextError(denied)).code).toBe("TARGET_UNAUTHORIZED")
|
|
184
200
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
201
|
+
ambiguousTargetA = true
|
|
202
|
+
const ambiguous = client.createConnection({host: targetA.host, port: targetA.port})
|
|
203
|
+
expect((await nextError(ambiguous)).code).toBe("AMBIGUOUS_ROUTE")
|
|
204
|
+
ambiguousTargetA = false
|
|
189
205
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
206
|
+
const legacy = new ReverseGatewayConnection({
|
|
207
|
+
gatewayId: "server-a-starlink",
|
|
208
|
+
connectionId: "legacy_exact_gateway_client",
|
|
209
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
210
|
+
tls
|
|
211
|
+
})
|
|
212
|
+
after(() => legacy.close())
|
|
213
|
+
await legacy.connect()
|
|
214
|
+
const legacyStream = legacy.createConnection({host: targetA.host, port: targetA.port})
|
|
215
|
+
await onceEvent(legacyStream, "connect")
|
|
216
|
+
expect(await exchange(legacyStream, "legacy")).toBe("a:legacy")
|
|
217
|
+
legacyStream.destroy()
|
|
218
|
+
await legacy.close()
|
|
203
219
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
220
|
+
const streamA = client.createConnection({host: targetA.host, port: targetA.port})
|
|
221
|
+
const streamB = client.createConnection({host: targetB.host, port: targetB.port})
|
|
222
|
+
await Promise.all([onceEvent(streamA, "connect"), onceEvent(streamB, "connect")])
|
|
223
|
+
expect(await Promise.all([exchange(streamA, "one"), exchange(streamB, "two")])).toEqual(["a:one", "b:two"])
|
|
208
224
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
225
|
+
const resumed = Promise.all([onceEvent(streamA, "ready"), onceEvent(streamB, "ready")])
|
|
226
|
+
await hub.closeActiveGateway("server-a-starlink")
|
|
227
|
+
await withTimeout(resumed, "route-class streams resume after gateway transport replacement")
|
|
228
|
+
await waitUntil(() => hub.status().healthyGateways === 2, "gateway A re-registration")
|
|
229
|
+
expect(await exchange(streamA, "resumed")).toBe("a:resumed")
|
|
230
|
+
expect(await exchange(streamB, "still-open")).toBe("b:still-open")
|
|
215
231
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
232
|
+
const anotherA = client.createConnection({host: targetA.host, port: targetA.port})
|
|
233
|
+
await onceEvent(anotherA, "connect")
|
|
234
|
+
expect(await exchange(anotherA, "new-a")).toBe("a:new-a")
|
|
235
|
+
const anotherB = client.createConnection({host: targetB.host, port: targetB.port})
|
|
236
|
+
await onceEvent(anotherB, "connect")
|
|
237
|
+
expect(await exchange(anotherB, "new")).toBe("b:new")
|
|
222
238
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
})
|
|
239
|
+
streamA.destroy()
|
|
240
|
+
anotherA.destroy()
|
|
241
|
+
await gatewayA.close()
|
|
242
|
+
const replacementA = new OutboundReverseGateway({
|
|
243
|
+
gatewayId: "server-a-starlink",
|
|
244
|
+
connectionId: "route_class_gateway_a",
|
|
245
|
+
initialGeneration: gatewayA.status().connectionGeneration,
|
|
246
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
|
|
247
|
+
tls,
|
|
248
|
+
authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port
|
|
249
|
+
})
|
|
250
|
+
after(() => replacementA.close())
|
|
251
|
+
await replacementA.start()
|
|
252
|
+
const rejoinedA = client.createConnection({host: targetA.host, port: targetA.port})
|
|
253
|
+
await onceEvent(rejoinedA, "connect")
|
|
254
|
+
expect(await exchange(rejoinedA, "rejoined")).toBe("a:rejoined")
|
|
255
|
+
expect(client.status().physicalConnects).toBeGreaterThanOrEqual(2)
|
|
256
|
+
expect(hub.status().clientPhysicalConnections).toBe(1)
|
|
257
|
+
streamB.destroy()
|
|
258
|
+
anotherB.destroy()
|
|
259
|
+
rejoinedA.destroy()
|
|
260
|
+
}))
|
|
245
261
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
262
|
+
it("a backpressured gateway does not block DATA or a new OPEN for another route-class gateway", async () => withTestCleanup(async (after) => {
|
|
263
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-backpressure-"))
|
|
264
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
265
|
+
const certificates = generateTestCertificates(root)
|
|
266
|
+
const targetA = await createEchoTarget("a:")
|
|
267
|
+
const targetB = await createEchoTarget("b:")
|
|
268
|
+
after(() => targetA.close())
|
|
269
|
+
after(() => targetB.close())
|
|
254
270
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
271
|
+
const hub = await startReverseGatewayHub({
|
|
272
|
+
tokens: new StaticTokenStore([
|
|
273
|
+
{username: "home-a", token: "gateway-a-token"},
|
|
274
|
+
{username: "home-b", token: "gateway-b-token"},
|
|
275
|
+
{username: "tensorbuzz", token: "client-token"}
|
|
276
|
+
]),
|
|
277
|
+
tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
|
|
278
|
+
knownGatewayIds: ["server-a-starlink", "server-b-starlink"],
|
|
279
|
+
knownRouteClasses: ["starlink"],
|
|
280
|
+
authorizeGateway: ({principal, gatewayId}) => (
|
|
281
|
+
(principal.username === "home-a" && gatewayId === "server-a-starlink")
|
|
282
|
+
|| (principal.username === "home-b" && gatewayId === "server-b-starlink")
|
|
283
|
+
),
|
|
284
|
+
authorizeClientGateway: () => false,
|
|
285
|
+
authorizeClientRouteClass: ({principal, routeClass}) => principal.username === "tensorbuzz" && routeClass === "starlink",
|
|
286
|
+
resolveOpenGateways: ({routeClass, target}) => {
|
|
287
|
+
if (routeClass !== "starlink" || target.host !== "127.0.0.1") return []
|
|
288
|
+
if (target.port === targetA.port) return ["server-a-starlink"]
|
|
289
|
+
if (target.port === targetB.port) return ["server-b-starlink"]
|
|
290
|
+
return []
|
|
291
|
+
},
|
|
292
|
+
authorizeOpen: ({gatewayId, target}) => (
|
|
293
|
+
(gatewayId === "server-a-starlink" && target.port === targetA.port)
|
|
294
|
+
|| (gatewayId === "server-b-starlink" && target.port === targetB.port)
|
|
295
|
+
),
|
|
296
|
+
maxOutputBytesPerConnection: 1_024
|
|
297
|
+
})
|
|
298
|
+
after(() => hub.close())
|
|
299
|
+
/** @type {import('node:tls').TLSSocket[]} */
|
|
300
|
+
const accepted = []
|
|
301
|
+
hub.server.on("secureConnection", (socket) => accepted.push(socket))
|
|
302
|
+
const tls = {caFile: certificates.caCertFile, servername: "localhost"}
|
|
303
|
+
const gatewayA = new OutboundReverseGateway({
|
|
304
|
+
gatewayId: "server-a-starlink",
|
|
305
|
+
connectionId: "backpressure_gateway_a",
|
|
306
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
|
|
307
|
+
tls,
|
|
308
|
+
authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port,
|
|
309
|
+
reconnectDelayMs: 10_000,
|
|
310
|
+
maxReconnectDelayMs: 10_000
|
|
311
|
+
})
|
|
312
|
+
const gatewayB = new OutboundReverseGateway({
|
|
313
|
+
gatewayId: "server-b-starlink",
|
|
314
|
+
connectionId: "backpressure_gateway_b",
|
|
315
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-b-token"},
|
|
316
|
+
tls,
|
|
317
|
+
authorizeTarget: (target) => target.host === targetB.host && target.port === targetB.port,
|
|
318
|
+
reconnectDelayMs: 10_000,
|
|
319
|
+
maxReconnectDelayMs: 10_000
|
|
320
|
+
})
|
|
321
|
+
after(() => gatewayA.close())
|
|
322
|
+
after(() => gatewayB.close())
|
|
323
|
+
await gatewayA.start()
|
|
324
|
+
await gatewayB.start()
|
|
325
|
+
expect(accepted.length).toBe(2)
|
|
310
326
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
327
|
+
const client = new ReverseGatewayConnection({
|
|
328
|
+
routeClass: "starlink",
|
|
329
|
+
connectionId: "backpressure_route_client",
|
|
330
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
331
|
+
tls,
|
|
332
|
+
continuousReconnect: false,
|
|
333
|
+
reconnectDelayMs: 10_000,
|
|
334
|
+
maxReconnectDelayMs: 10_000
|
|
335
|
+
})
|
|
336
|
+
after(() => client.close())
|
|
337
|
+
await client.connect()
|
|
338
|
+
const streamA = client.createConnection({host: targetA.host, port: targetA.port})
|
|
339
|
+
const streamB = client.createConnection({host: targetB.host, port: targetB.port})
|
|
340
|
+
await Promise.all([onceEvent(streamA, "connect"), onceEvent(streamB, "connect")])
|
|
325
341
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
342
|
+
const gatewayASocket = /** @type {import('node:tls').TLSSocket} */ (accepted[0])
|
|
343
|
+
const writable = /** @type {{write: (chunk: Buffer) => boolean}} */ (/** @type {unknown} */ (gatewayASocket))
|
|
344
|
+
const originalWrite = writable.write
|
|
345
|
+
let blocked = false
|
|
346
|
+
/** @type {() => void} */
|
|
347
|
+
let markBlocked = () => {}
|
|
348
|
+
const backpressured = new Promise((resolve) => { markBlocked = () => resolve(undefined) })
|
|
349
|
+
writable.write = (chunk) => {
|
|
350
|
+
const acceptedWrite = Reflect.apply(originalWrite, gatewayASocket, [chunk])
|
|
351
|
+
if (!blocked && chunk[5] === V3FrameType.DATA) {
|
|
352
|
+
blocked = true
|
|
353
|
+
markBlocked()
|
|
354
|
+
return false
|
|
355
|
+
}
|
|
356
|
+
return acceptedWrite
|
|
339
357
|
}
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
if (blocked) gatewayASocket.emit("drain")
|
|
345
|
-
})
|
|
358
|
+
after(() => {
|
|
359
|
+
writable.write = originalWrite
|
|
360
|
+
if (blocked) gatewayASocket.emit("drain")
|
|
361
|
+
})
|
|
346
362
|
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
})
|
|
363
|
+
streamA.write("blocked")
|
|
364
|
+
await withTimeout(backpressured, "gateway A backpressure")
|
|
365
|
+
const newStreamB = client.createConnection({host: targetB.host, port: targetB.port})
|
|
366
|
+
after(() => { newStreamB.destroy() })
|
|
367
|
+
const newStreamBExchange = (async () => {
|
|
368
|
+
await onceEvent(newStreamB, "connect")
|
|
369
|
+
return await exchange(newStreamB, "new-open")
|
|
370
|
+
})()
|
|
371
|
+
expect(await withTimeout(Promise.all([
|
|
372
|
+
exchange(streamB, "existing"),
|
|
373
|
+
newStreamBExchange
|
|
374
|
+
]), "healthy gateway progress while gateway A is backpressured")).toEqual(["b:existing", "b:new-open"])
|
|
375
|
+
expect(client.status().physicalConnects).toBe(1)
|
|
376
|
+
expect(hub.status().queuedOutputBytes).toBeGreaterThan(0)
|
|
377
|
+
const streamAError = nextError(streamA)
|
|
378
|
+
const queuedAErrors = Array.from({length: 8}, () => {
|
|
379
|
+
const socket = client.createConnection({host: targetA.host, port: targetA.port, sessionName: "a".repeat(256)})
|
|
380
|
+
after(() => { socket.destroy() })
|
|
381
|
+
return nextError(socket)
|
|
382
|
+
})
|
|
383
|
+
const outputError = /** @type {Error & Record<string, unknown>} */ (await withTimeout(streamAError, "gateway A output limit"))
|
|
384
|
+
expect(outputError.code).toBe("OUTPUT_BACKPRESSURE_LIMIT")
|
|
385
|
+
expect(outputError.disconnectReason).toBe("output_backpressure_limit")
|
|
386
|
+
expect(outputError.incidentOrigin).toBe("reverse_hub")
|
|
387
|
+
expect(outputError.physicalRole).toBe("gateway")
|
|
388
|
+
expect(/** @type {string} */ (outputError.disconnectIncidentId)).toMatch(/^[A-Za-z0-9_-]{22}$/)
|
|
389
|
+
await withTimeout(Promise.all(queuedAErrors), "queued gateway A stream cleanup")
|
|
390
|
+
expect(await exchange(streamB, "after-limit")).toBe("b:after-limit")
|
|
391
|
+
expect(hub.status().healthyGateways).toBe(1)
|
|
392
|
+
expect(hub.status().clientPhysicalConnections).toBe(1)
|
|
393
|
+
expect(client.status().physicalConnects).toBe(1)
|
|
394
|
+
writable.write = originalWrite
|
|
395
|
+
expect(hub.status().queuedOutputBytes).toBe(0)
|
|
396
|
+
streamA.destroy()
|
|
397
|
+
streamB.destroy()
|
|
398
|
+
}))
|
|
383
399
|
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
400
|
+
it("hub announces route-class availability at selection and the client exposes gateway unavailability", async () => withTestCleanup(async (after) => {
|
|
401
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-route-class-selectable-"))
|
|
402
|
+
after(() => rm(root, {recursive: true, force: true}))
|
|
403
|
+
const certificates = generateTestCertificates(root)
|
|
404
|
+
const targetA = await createEchoTarget("a:")
|
|
405
|
+
after(() => targetA.close())
|
|
390
406
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
407
|
+
const hub = await startReverseGatewayHub({
|
|
408
|
+
tokens: new StaticTokenStore([
|
|
409
|
+
{username: "home-a", token: "gateway-a-token"},
|
|
410
|
+
{username: "tensorbuzz", token: "client-token"}
|
|
411
|
+
]),
|
|
412
|
+
tls: {certFile: certificates.serverCertFile, keyFile: certificates.serverKeyFile},
|
|
413
|
+
knownGatewayIds: ["server-a-starlink"],
|
|
414
|
+
knownRouteClasses: ["starlink"],
|
|
415
|
+
authorizeGateway: ({principal, gatewayId}) => principal.username === "home-a" && gatewayId === "server-a-starlink",
|
|
416
|
+
authorizeClientGateway: () => false,
|
|
417
|
+
authorizeClientRouteClass: ({principal, routeClass}) => principal.username === "tensorbuzz" && routeClass === "starlink",
|
|
418
|
+
resolveRouteClassGateways: () => ["server-a-starlink"],
|
|
419
|
+
resolveOpenGateways: ({routeClass, target}) => (routeClass === "starlink" && target.host === targetA.host ? ["server-a-starlink"] : []),
|
|
420
|
+
authorizeOpen: ({gatewayId, target}) => gatewayId === "server-a-starlink" && target.port === targetA.port
|
|
421
|
+
})
|
|
422
|
+
after(() => hub.close())
|
|
423
|
+
const tls = {caFile: certificates.caCertFile, servername: "localhost"}
|
|
408
424
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
425
|
+
const client = new ReverseGatewayConnection({
|
|
426
|
+
routeClass: "starlink",
|
|
427
|
+
connectionId: "route_class_selectable_client",
|
|
428
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
429
|
+
tls
|
|
430
|
+
})
|
|
431
|
+
after(() => client.close())
|
|
416
432
|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
433
|
+
// No gateway registered for the route class yet: the hub must announce selectable=false
|
|
434
|
+
// and the client must surface the member as gateway-unavailable before any open is attempted.
|
|
435
|
+
await client.connect()
|
|
436
|
+
expect(client.status().physicalState).toBe("ready")
|
|
437
|
+
expect(client.status().gatewayUnavailable).toBe(true)
|
|
422
438
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
439
|
+
// Once the gateway registers, a replacement selection announces selectable=true and clears the mark.
|
|
440
|
+
const gatewayA = new OutboundReverseGateway({
|
|
441
|
+
gatewayId: "server-a-starlink",
|
|
442
|
+
connectionId: "selectable_gateway_a",
|
|
443
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "gateway-a-token"},
|
|
444
|
+
tls,
|
|
445
|
+
authorizeTarget: (target) => target.host === targetA.host && target.port === targetA.port,
|
|
446
|
+
reconnectDelayMs: 10,
|
|
447
|
+
maxReconnectDelayMs: 20
|
|
448
|
+
})
|
|
449
|
+
after(() => gatewayA.close())
|
|
450
|
+
await gatewayA.start()
|
|
451
|
+
await waitUntil(() => hub.status().healthyGateways === 1, "gateway A registration")
|
|
436
452
|
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
453
|
+
const replacement = new ReverseGatewayConnection({
|
|
454
|
+
routeClass: "starlink",
|
|
455
|
+
connectionId: "selectable_client_replacement",
|
|
456
|
+
hub: {host: "127.0.0.1", port: hub.port, token: "client-token"},
|
|
457
|
+
tls
|
|
458
|
+
})
|
|
459
|
+
after(() => replacement.close())
|
|
460
|
+
await replacement.connect()
|
|
461
|
+
expect(replacement.status().gatewayUnavailable).toBe(false)
|
|
462
|
+
const stream = replacement.createConnection({host: targetA.host, port: targetA.port})
|
|
463
|
+
await onceEvent(stream, "connect")
|
|
464
|
+
expect(await exchange(stream, "probe")).toBe("a:probe")
|
|
465
|
+
stream.destroy()
|
|
466
|
+
}))
|
|
450
467
|
})
|