socketduct 0.1.18 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +17 -5
- package/README.md +5 -5
- package/changelog.d/20260925143000-reverse-client-process-recovery.md +1 -0
- package/changelog.d/20260925170000-disconnect-attribution-compressed-logs.md +1 -0
- package/docs/architecture.md +11 -9
- package/docs/docker.md +5 -4
- package/docs/logging.md +131 -159
- package/docs/protocol.md +43 -7
- package/docs/reverse-gateway.md +75 -9
- package/docs/running.md +16 -13
- package/docs/security.md +16 -5
- package/docs/session-lifecycle.md +14 -3
- package/docs/socket-api.md +54 -20
- package/package.json +5 -1
- package/scripts/verify-zstd-interoperability.mjs +41 -0
- package/src/cli/socketduct.js +143 -0
- package/src/client/multiplexed-transport-set.js +12 -5
- package/src/client/virtual-socket.js +52 -0
- package/src/config/environment.js +56 -7
- package/src/index.js +64 -26
- package/src/logging/compressed-log-reader.js +256 -0
- package/src/logging/compressed-log-worker.js +430 -0
- package/src/logging/compressed-log-writer.js +538 -0
- package/src/logging/log-batch.js +72 -0
- package/src/logging/log-retention.js +241 -0
- package/src/logging/log-segment-index.js +176 -0
- package/src/logging/structured-logger.js +136 -30
- package/src/logging/zstd-frame-codec.js +73 -0
- package/src/reverse/client.js +372 -82
- package/src/reverse/disconnect-attribution.js +191 -0
- package/src/reverse/gateway.js +68 -18
- package/src/reverse/hub.js +215 -32
- package/src/reverse/runtime.js +5 -3
- package/src/spool/spool-file.js +136 -7
- package/tensorbuzz.yml +4 -2
- package/test/e2e/logger-shutdown-process.test.js +86 -0
- package/test/e2e/reverse-client-recovery-process.test.js +170 -0
- package/test/e2e/reverse-gateway-process.test.js +15 -14
- package/test/fixtures/reverse-client-recovery.mjs +123 -0
- package/test/integration/logging-cli.test.js +168 -0
- package/test/integration/logging-compressed-recovery.test.js +186 -0
- package/test/integration/reverse-gateway.test.js +597 -1
- package/test/integration/reverse-route-class.test.js +6 -1
- package/test/unit/client/multiplexed-transport-set.test.js +12 -1
- package/test/unit/config/docker-gateway-files.test.js +2 -0
- package/test/unit/config/environment.test.js +54 -14
- package/test/unit/index.test.js +3 -0
- package/test/unit/logging/compressed-log-worker.test.js +29 -0
- package/test/unit/logging/compressed-log-writer.test.js +183 -0
- package/test/unit/logging/log-retention.test.js +86 -0
- package/test/unit/logging/log-segment-index.test.js +104 -0
- package/test/unit/logging/structured-logger.test.js +14 -1
- package/test/unit/logging/zstd-frame-codec.test.js +24 -0
- package/test/unit/reverse/disconnect-attribution.test.js +82 -0
- package/test/unit/spool/spool-file.test.js +133 -0
package/docs/reverse-gateway.md
CHANGED
|
@@ -125,8 +125,10 @@ in-memory gateway registry across independent cluster workers.
|
|
|
125
125
|
|
|
126
126
|
Gateway registration reconnects with bounded exponential delay and a higher
|
|
127
127
|
generation. An established Docker stream survives a transient physical loss
|
|
128
|
-
when the hub process and its retained state,
|
|
128
|
+
when the hub process and its retained state, outbound gateway process, and
|
|
129
129
|
original target socket remain available through the configured resume window.
|
|
130
|
+
The original client object may reconnect in place, or a replacement reverse
|
|
131
|
+
client process may reconstruct a new `VirtualSocket` from durable spool state.
|
|
130
132
|
Socketduct keeps
|
|
131
133
|
the app-facing socket open, durably spools each direction's unacknowledged raw
|
|
132
134
|
chunks before sending them, reconciles the two receiver-committed ACK positions,
|
|
@@ -145,14 +147,39 @@ mismatches, invalid manifests, spool/buffer limits, and authorization failures
|
|
|
145
147
|
fail closed.
|
|
146
148
|
|
|
147
149
|
Reverse replay state is bounded per stream and connection and stored in private
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
`
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
150
|
+
spool directories. ACKed chunks are truncated, and terminal close/reset removes
|
|
151
|
+
the spool. `status().spoolBytes` reports retained durable payload. Reverse-client
|
|
152
|
+
replacement requires the same explicit `connectionId`, an explicit persistent
|
|
153
|
+
`spoolDirectory`, and the same stream identifier, name, target, and one original
|
|
154
|
+
selector (`gatewayId` or `routeClass`). Register every stream with
|
|
155
|
+
`registerRecoveredConnection({...options, recover: true})` before `connect()`;
|
|
156
|
+
registration synchronously rejects missing, expired, corrupt, wrong-owner,
|
|
157
|
+
mismatched, noncontiguous, or oversized state before any network attempt. The
|
|
158
|
+
stored generation is advanced and persisted before AUTH, recovered outbound
|
|
159
|
+
replay starts above the gateway high-water position, and inbound delivery starts
|
|
160
|
+
at the durable receive ACK because unread bytes in the dead `VirtualSocket` no
|
|
161
|
+
longer exist. Receive ACKs advance only after the application consumes bytes,
|
|
162
|
+
not when `VirtualSocket` merely accepts them into its readable buffer. Recovery
|
|
163
|
+
uses the committed recovery record to discard an uncommitted chunk tail and
|
|
164
|
+
repair derivable usage accounting after an abrupt process loss; structural,
|
|
165
|
+
identity, sequence, quota, and non-tail corruption still fail closed. Recovery
|
|
166
|
+
never sends `OPEN`.
|
|
167
|
+
|
|
168
|
+
The default reverse spool is under `/tmp`, and the sample reverse-gateway
|
|
169
|
+
Compose mounts `/tmp` as tmpfs. Those defaults do not survive a container or
|
|
170
|
+
host replacement. A caller claiming process/container recovery must supply a
|
|
171
|
+
private persistent directory; Socketduct maintains `0700` directories and
|
|
172
|
+
`0600` files and never writes hub credentials or session tokens there.
|
|
173
|
+
|
|
174
|
+
This boundary does not reconstruct a hub after hub process loss: the current
|
|
175
|
+
manifests have no durable hub epoch, hub-minted bridge identity, endpoint-pairing
|
|
176
|
+
journal, or split-brain fence. It also does not transfer gateway worker file
|
|
177
|
+
descriptors, recover an outbound gateway process, reopen a lost target socket,
|
|
178
|
+
or replay a Docker request after gateway/container/host loss. A newly constructed
|
|
179
|
+
gateway owns no inherited targets; if the hub reports retained bridges, it sends
|
|
180
|
+
an empty resume manifest so omitted bridges reset once, without calling the
|
|
181
|
+
target factory or emitting `OPEN`. Loss outside the supported boundary fails the
|
|
182
|
+
logical stream and leaves higher-level reconciliation to the application.
|
|
156
183
|
|
|
157
184
|
Target bytes already observed by the gateway are ordered before a subsequent
|
|
158
185
|
target-error reset. Stream `END` remains a directional half-close, while protocol
|
|
@@ -171,6 +198,9 @@ SOCKETDUCT_TLS_KEY_FILE=/certs/hub.key \
|
|
|
171
198
|
SOCKETDUCT_AUTH_TOKENS='server-a-starlink:<secret>,server-b-starlink:<secret>,tensorbuzz:<secret>' \
|
|
172
199
|
SOCKETDUCT_REVERSE_GATEWAY_BINDINGS='server-a-starlink:server-a-starlink,server-b-starlink:server-b-starlink' \
|
|
173
200
|
SOCKETDUCT_REVERSE_ROUTE_CLASS_ROUTES='tensorbuzz:starlink:server-a-starlink:docker-a:2375,tensorbuzz:starlink:server-b-starlink:docker-b:2375' \
|
|
201
|
+
SOCKETDUCT_LOG_DIRECTORY=/var/log/socketduct \
|
|
202
|
+
SOCKETDUCT_CONNECTION_FINGERPRINT_KEY='<unpadded-base64url-secret>' \
|
|
203
|
+
SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH='2026-09' \
|
|
174
204
|
node src/index.js
|
|
175
205
|
```
|
|
176
206
|
|
|
@@ -182,6 +212,37 @@ Route-class rows use `username:routeClass:gatewayId:host:port`. If two rows make
|
|
|
182
212
|
different gateways eligible for the same authenticated class and target, that
|
|
183
213
|
`OPEN` is rejected as ambiguous rather than choosing one implicitly.
|
|
184
214
|
|
|
215
|
+
When hub logging is enabled, the fingerprint key is mandatory, hub-only, at
|
|
216
|
+
least 32 random bytes encoded as canonical unpadded base64url, and never transmitted or
|
|
217
|
+
logged. Rotate it at least every 30 days. The public epoch label identifies the
|
|
218
|
+
key generation used for a 128-bit role-separated HMAC correlation value.
|
|
219
|
+
|
|
220
|
+
## Disconnect attribution
|
|
221
|
+
|
|
222
|
+
Every reverse physical loss has one 128-bit random `disconnectIncidentId` and
|
|
223
|
+
one first local `disconnectReason`. The initial socket error, EOF, peer GOAWAY,
|
|
224
|
+
protocol violation, output limit, replacement, shutdown, or send-after-loss
|
|
225
|
+
cause wins over later close callbacks. The incident records its originating
|
|
226
|
+
component, physical client/gateway role, explicitly named connection
|
|
227
|
+
generations, and fan-out/resume/terminal stream counts. A public RESET code such
|
|
228
|
+
as `RESUME_WINDOW_EXPIRED` classifies the stream outcome; it does not replace
|
|
229
|
+
the physical cause.
|
|
230
|
+
|
|
231
|
+
The hub propagates its validated incident in RESET/GOAWAY controls and typed
|
|
232
|
+
`SocketductStreamError` / `SocketductSessionLostError` instances. Direct client
|
|
233
|
+
or gateway loss creates and caches an equivalent local incident so concurrent
|
|
234
|
+
stream errors share it, including writes racing the close callback. A new
|
|
235
|
+
authenticated physical clears that cache. Peer-provided reserved attribution
|
|
236
|
+
fields are stripped; peers cannot claim hub origin, inject correlation values,
|
|
237
|
+
or alter named generations.
|
|
238
|
+
|
|
239
|
+
Terminal, non-resumable hub paths assign the incident before removing their
|
|
240
|
+
bridges, including protocol violations, output overflow, replacement, peer
|
|
241
|
+
GOAWAY, and shutdown. A safe peer GOAWAY code survives as terminal
|
|
242
|
+
classification while the first hub-observed `peer_goaway` cause and hub-created
|
|
243
|
+
incident identity remain authoritative; fatal client and gateway connection
|
|
244
|
+
errors retain that validated attribution.
|
|
245
|
+
|
|
185
246
|
## One outbound member per target and route class
|
|
186
247
|
|
|
187
248
|
Run this once in each target server's route-pinned network namespace, changing
|
|
@@ -238,6 +299,11 @@ Callers cannot override either selection per open. Reverse streams remain
|
|
|
238
299
|
sticky to their selected gateway/target but resume transparently after a
|
|
239
300
|
transient physical transport replacement.
|
|
240
301
|
|
|
302
|
+
For process replacement, call `recoverConnection()` before `start()` with
|
|
303
|
+
exactly the socket's immutable original selector: `gatewayId` for an exact set
|
|
304
|
+
or `routeClass` for a route-class set. Recovery derives the original member
|
|
305
|
+
identity and spool namespace and never falls back to or probes a sibling.
|
|
306
|
+
|
|
241
307
|
The socket is returned immediately and waits inside Socketduct until the set
|
|
242
308
|
reaches minimum readiness. New sockets rotate equally across ready members and
|
|
243
309
|
stay on their assigned member. If all members are reconnecting, new sockets wait
|
package/docs/running.md
CHANGED
|
@@ -58,16 +58,18 @@ roles and always uses verified TLS plus adaptive zstd. See
|
|
|
58
58
|
[Native reverse gateway](reverse-gateway.md) for the environment contract and
|
|
59
59
|
three-member TensorBuzz API.
|
|
60
60
|
|
|
61
|
-
##
|
|
62
|
-
|
|
63
|
-
Set `SOCKETDUCT_LOG_DIRECTORY` to opt into
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
61
|
+
## Durable compressed diagnostic logs
|
|
62
|
+
|
|
63
|
+
Set `SOCKETDUCT_LOG_DIRECTORY` to opt into private, bounded `.jsonl.zst`
|
|
64
|
+
segments and checksummed sidecar indexes. A dedicated worker writes level-19
|
|
65
|
+
checksummed frames; orderly shutdown awaits data and index sync. Defaults are
|
|
66
|
+
1 MiB frames, 128 MiB segments, 30 days/4 GiB retention, and a 64 MiB queue
|
|
67
|
+
with an 8 MiB critical reserve. A logging reverse hub must also configure its
|
|
68
|
+
hub-only connection-fingerprint key and public key epoch. Existing stdout/stderr
|
|
69
|
+
output is preserved. Use `socketduct logs tail` or `socketduct logs search`, not
|
|
70
|
+
`tail -f` on compressed bytes. See [durable compressed diagnostic
|
|
71
|
+
logs](logging.md) for all settings, lifecycle semantics, CLI exits, embedded
|
|
72
|
+
API, and the strict privacy contract.
|
|
71
73
|
|
|
72
74
|
## Minimal programmatic relay/edge shape
|
|
73
75
|
|
|
@@ -107,7 +109,8 @@ Do not hard-code real passwords or token values. Feed credentials through a secr
|
|
|
107
109
|
|
|
108
110
|
## MVP limitations
|
|
109
111
|
|
|
110
|
-
-
|
|
112
|
+
- The stable CLI surface currently covers owned compressed-log tail/search;
|
|
113
|
+
tunnel configuration remains environment/programmatic.
|
|
111
114
|
- Relay targets are requested per session and must be explicitly authorized by the injected target policy.
|
|
112
115
|
- User/session stores are in-memory in the current MVP skeleton.
|
|
113
116
|
- Spool files are temporary local files and may contain raw application payload.
|
|
@@ -210,6 +213,6 @@ inspect its ordered aggregate causes or privacy-safe per-member status codes.
|
|
|
210
213
|
No extra physical member is created. Configure the set-wide buffer/spool/queue
|
|
211
214
|
limits for the whole process rather than multiplying a desired budget by
|
|
212
215
|
gateway count.
|
|
213
|
-
For process replacement, call `recoverConnection()` with each stream's
|
|
214
|
-
gateway ID before `start()`; see the
|
|
216
|
+
For process replacement, call `recoverConnection()` with exactly each stream's
|
|
217
|
+
original gateway ID or reverse route class before `start()`; see the sequence in
|
|
215
218
|
[Node-compatible socket API design](socket-api.md#multiplexedsocketducttransportset-multi-gateway-v3-api).
|
package/docs/security.md
CHANGED
|
@@ -100,7 +100,7 @@ Optional later hardening:
|
|
|
100
100
|
|
|
101
101
|
## Logging
|
|
102
102
|
|
|
103
|
-
The optional
|
|
103
|
+
The optional durable compressed logger uses a closed event/field allowlist. It
|
|
104
104
|
records process/role identity, lifecycle state, bounded counters, validated
|
|
105
105
|
gateway IDs, and normalized error codes/classes. Unknown event names are
|
|
106
106
|
dropped and unknown error codes become `UNKNOWN`.
|
|
@@ -112,8 +112,12 @@ diagnostics retain bounded error names, messages, and stacks so operational
|
|
|
112
112
|
failures can be investigated while credential-shaped values are redacted. Each
|
|
113
113
|
JSON line is capped at 1024 bytes. Logger I/O and custom-writer failures are
|
|
114
114
|
isolated from network behavior and produce only one bounded stderr
|
|
115
|
-
classification.
|
|
116
|
-
|
|
115
|
+
classification. Managed files are private checksummed zstd frames with a
|
|
116
|
+
checksummed structural index and no plaintext fallback. Reverse-hub connection
|
|
117
|
+
correlation uses only a hub-held, role-separated HMAC truncated to 128 bits;
|
|
118
|
+
raw connection IDs, the key, unkeyed digests, targets, and payloads are never
|
|
119
|
+
persisted. See [durable compressed diagnostic logs](logging.md) for
|
|
120
|
+
configuration, key rotation, recovery, and retention.
|
|
117
121
|
|
|
118
122
|
V3 status is payload-free and credential-free: physical counts/generation,
|
|
119
123
|
virtual stream state counts, active local targets, bounded queued/spool bytes,
|
|
@@ -176,8 +180,15 @@ only above it. This preserves exact-once delivery without reopening or
|
|
|
176
180
|
reissuing a privileged Docker request. Ownership mismatches, missing targets,
|
|
177
181
|
invalid or incomplete manifests, resource-bound failures, and expired resume
|
|
178
182
|
windows fail the affected logical stream once instead of falling back to a new
|
|
179
|
-
stream.
|
|
180
|
-
|
|
183
|
+
stream. Reverse-client process replacement is allowed only from caller-selected
|
|
184
|
+
private persistent storage with the same stable connection/stream identities
|
|
185
|
+
and exactly one original gateway or route-class selector. Recovery validates
|
|
186
|
+
ownership, age, identity, lifecycle, contiguous chunks, and bounds before
|
|
187
|
+
network activity; spool files contain no bearer credential or session token.
|
|
188
|
+
The same in-memory hub bridge, gateway process, and live target socket must
|
|
189
|
+
survive. Hub reconstruction, gateway FD transfer, and target reopen/retry remain
|
|
190
|
+
unsupported, and a replacement gateway resets retained bridges with an empty
|
|
191
|
+
manifest rather than recreating a privileged operation.
|
|
181
192
|
|
|
182
193
|
## Network exposure
|
|
183
194
|
|
|
@@ -27,9 +27,20 @@ without opening a second target or retrying an application operation.
|
|
|
27
27
|
|
|
28
28
|
If the original target socket disappears, an owner/identifier or sequence does
|
|
29
29
|
not match, a buffer/spool bound is exceeded, or the resume window expires, that
|
|
30
|
-
logical stream fails once with the causal error and releases both sides.
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
logical stream fails once with the causal error and releases both sides. A dead
|
|
31
|
+
reverse-client process may be replaced: the new object creates a new
|
|
32
|
+
`VirtualSocket`, restores the durable generation/lifecycle and replay state, and
|
|
33
|
+
registers every stream before connecting. The same hub bridge, outbound gateway
|
|
34
|
+
process, and original target socket must still exist. The dead app-facing socket
|
|
35
|
+
is not transferred; target bytes above its durable receive ACK replay to the new
|
|
36
|
+
socket. That ACK covers bytes returned to the application, not bytes merely
|
|
37
|
+
buffered in the dead process. An orderly `detach()` drains already-queued frame
|
|
38
|
+
handling before its final durable snapshot and handoff.
|
|
39
|
+
|
|
40
|
+
Hub process reconstruction, gateway process or file-descriptor transfer, and
|
|
41
|
+
target reopen/retry are unsupported. A replacement gateway has no live target
|
|
42
|
+
sockets and therefore completes retained-state registration with an empty
|
|
43
|
+
manifest, resetting omitted bridges rather than issuing a second `OPEN`.
|
|
33
44
|
|
|
34
45
|
The bounded, paginated resume manifest carries stable identifiers and both
|
|
35
46
|
directional acknowledgement positions. The relay returns an independent result
|
package/docs/socket-api.md
CHANGED
|
@@ -219,24 +219,23 @@ the callback, which runs exactly once only after `secureConnect` or failure.
|
|
|
219
219
|
Both methods count only completed member assignments and preserve stream
|
|
220
220
|
stickiness. Every returned raw or TLS socket exposes enumerable `gatewayId` and
|
|
221
221
|
`routeClass` allocation metadata. Only the configured identity is set. Persist
|
|
222
|
-
an exact set's immutable
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
socket remain alive.
|
|
222
|
+
an exact set's immutable original selector after connection: `gatewayId` for a
|
|
223
|
+
gateway member or `routeClass` for a reverse member. A replacement uses that
|
|
224
|
+
selector to recover the stream on its original member without running selection
|
|
225
|
+
again.
|
|
227
226
|
|
|
228
227
|
Process-replacement recovery is deliberately separate from ordinary selection.
|
|
229
228
|
First detach the old set when an orderly handoff is possible. Construct the
|
|
230
|
-
replacement with the same base `connectionId`, spool directory,
|
|
231
|
-
|
|
232
|
-
`gatewayId` before calling `start()`:
|
|
229
|
+
replacement with the same base `connectionId`, spool directory, and configured
|
|
230
|
+
member endpoints. Then register each durable stream with exactly its original
|
|
231
|
+
`gatewayId` or `routeClass` before calling `start()`:
|
|
233
232
|
|
|
234
233
|
```js
|
|
235
234
|
await set.detach()
|
|
236
235
|
|
|
237
236
|
const replacement = new MultiplexedSocketductTransportSet(sameSetOptions)
|
|
238
237
|
const recovered = replacement.recoverConnection({
|
|
239
|
-
|
|
238
|
+
routeClass: durableRecord.routeClass, // exact sets use gatewayId instead
|
|
240
239
|
host: "api.internal",
|
|
241
240
|
port: 443,
|
|
242
241
|
identifier: "build_stream_01",
|
|
@@ -245,10 +244,11 @@ const recovered = replacement.recoverConnection({
|
|
|
245
244
|
await replacement.start()
|
|
246
245
|
```
|
|
247
246
|
|
|
248
|
-
`recoverConnection()` implies durable recovery and rejects a missing
|
|
249
|
-
|
|
250
|
-
begun. It bypasses
|
|
251
|
-
migrates or replays the stream on a sibling.
|
|
247
|
+
`recoverConnection()` implies durable recovery and rejects a missing,
|
|
248
|
+
ambiguous, invalid, or unknown original selector, an explicit `recover` option,
|
|
249
|
+
or registration after `start()` has begun. It bypasses selection only to reach
|
|
250
|
+
that configured member; it never migrates or replays the stream on a sibling.
|
|
251
|
+
Ordinary `createConnection()` and
|
|
252
252
|
`createTlsConnection()` reject `recover: true`, `gatewayId`, and `routeClass`,
|
|
253
253
|
so new opens cannot force a path. `detach()` coalesces with concurrent shutdown calls,
|
|
254
254
|
preserves active member spools, and retires empty warmed lanes; `close()` remains
|
|
@@ -401,13 +401,47 @@ Target `ca`, `cert`, and `key` accept Node TLS certificate material. `cert` and
|
|
|
401
401
|
It selects exactly one registered gateway during `connect()`. The existing
|
|
402
402
|
`MultiplexedSocketductTransportSet` accepts `endpoint.reverseHub`, allowing
|
|
403
403
|
three ready members to retain deterministic health-aware round-robin behavior.
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
replacement
|
|
409
|
-
|
|
410
|
-
|
|
404
|
+
An established reverse stream can be cold-registered in a replacement client
|
|
405
|
+
process before its first physical connection:
|
|
406
|
+
|
|
407
|
+
```js
|
|
408
|
+
const replacement = new ReverseGatewayConnection({
|
|
409
|
+
routeClass: durableRecord.routeClass,
|
|
410
|
+
connectionId: durableRecord.connectionId,
|
|
411
|
+
spoolDirectory: "/srv/socketduct/recovery",
|
|
412
|
+
hub,
|
|
413
|
+
tls
|
|
414
|
+
})
|
|
415
|
+
const socket = replacement.registerRecoveredConnection({
|
|
416
|
+
recover: true,
|
|
417
|
+
host: durableRecord.host,
|
|
418
|
+
port: durableRecord.port,
|
|
419
|
+
identifier: durableRecord.identifier,
|
|
420
|
+
sessionName: durableRecord.sessionName
|
|
421
|
+
})
|
|
422
|
+
await replacement.connect()
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
Recovery restores the durable stream lifecycle, sequences, acknowledgements,
|
|
426
|
+
and generation. It never emits a new `OPEN` or recreates the target; target data
|
|
427
|
+
above the durable receive acknowledgement is replayed by the surviving gateway.
|
|
428
|
+
Reverse receive ACKs cover bytes returned to the application, not unread bytes
|
|
429
|
+
held only in the old process. `detach()` resolves only after queued physical
|
|
430
|
+
frame handling is quiescent and the resulting recovery state is durable, so a
|
|
431
|
+
replacement cannot race a late write from the retired object.
|
|
432
|
+
The caller must supply stable `connectionId` and private persistent
|
|
433
|
+
`spoolDirectory` values. The default `/tmp` spool and the sample gateway
|
|
434
|
+
Compose tmpfs do not survive container or host replacement. Spool directories
|
|
435
|
+
remain mode 0700, files remain mode 0600, and credentials or session tokens are
|
|
436
|
+
not persisted.
|
|
437
|
+
|
|
438
|
+
This boundary requires the same hub bridge, reverse-gateway process, and target
|
|
439
|
+
TCP socket to survive. It does not reconstruct a restarted hub, keep or transfer
|
|
440
|
+
gateway sockets between processes, reopen a lost target, or replay an ambiguous
|
|
441
|
+
target request after gateway, container, or host loss. If a newly constructed
|
|
442
|
+
outbound gateway finds retained hub bridges but owns no live target sockets, it
|
|
443
|
+
sends an empty resume manifest so the hub resets those omitted streams instead
|
|
444
|
+
of creating targets.
|
|
411
445
|
|
|
412
446
|
Reverse virtual sockets accept individual writes larger than their configured
|
|
413
447
|
retained-byte limit. The connection progresses those writes in slices bounded by
|
package/package.json
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "socketduct",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.20",
|
|
4
4
|
"description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.js",
|
|
7
|
+
"bin": {
|
|
8
|
+
"socketduct": "./src/cli/socketduct.js"
|
|
9
|
+
},
|
|
7
10
|
"exports": {
|
|
8
11
|
".": "./src/index.js",
|
|
9
12
|
"./http": "./src/client/http-agent.js",
|
|
@@ -21,6 +24,7 @@
|
|
|
21
24
|
"release:patch": "npm run check && release-patch",
|
|
22
25
|
"start": "node src/index.js",
|
|
23
26
|
"test": "node scripts/test.js",
|
|
27
|
+
"verify:zstd-logs": "node scripts/verify-zstd-interoperability.mjs",
|
|
24
28
|
"typecheck": "tsc --noEmit",
|
|
25
29
|
"lint": "eslint . --max-warnings 0"
|
|
26
30
|
},
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import assert from "node:assert/strict"
|
|
2
|
+
import {execFile} from "node:child_process"
|
|
3
|
+
import {mkdtemp, readdir, rm} from "node:fs/promises"
|
|
4
|
+
import {tmpdir} from "node:os"
|
|
5
|
+
import {join} from "node:path"
|
|
6
|
+
import {promisify} from "node:util"
|
|
7
|
+
|
|
8
|
+
import {readCompressedLogDirectory} from "../src/logging/compressed-log-reader.js"
|
|
9
|
+
import {createCompressedJsonlFileLogger} from "../src/logging/compressed-log-writer.js"
|
|
10
|
+
|
|
11
|
+
const run = promisify(execFile)
|
|
12
|
+
const root = await mkdtemp(join(tmpdir(), "socketduct-zstd-reference-"))
|
|
13
|
+
const directory = join(root, "logs")
|
|
14
|
+
|
|
15
|
+
try {
|
|
16
|
+
let tick = 0
|
|
17
|
+
const logger = createCompressedJsonlFileLogger({
|
|
18
|
+
directory,
|
|
19
|
+
role: "reverse-hub",
|
|
20
|
+
now: () => new Date(1_795_000_000_000 + tick++),
|
|
21
|
+
frameMaxBytes: 256,
|
|
22
|
+
maxSegmentBytes: 1024 * 1024
|
|
23
|
+
})
|
|
24
|
+
logger.log("process.started", {runtimeMode: "reverse-hub"})
|
|
25
|
+
for (let value = 1; value <= 4; value += 1) logger.log("runtime.status", {virtualStreams: value})
|
|
26
|
+
await logger.close()
|
|
27
|
+
|
|
28
|
+
const segmentNames = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
|
|
29
|
+
assert.equal(segmentNames.length, 1)
|
|
30
|
+
const segment = join(directory, /** @type {string} */ (segmentNames[0]))
|
|
31
|
+
const owned = await readCompressedLogDirectory(directory)
|
|
32
|
+
assert.equal(owned.incomplete, false)
|
|
33
|
+
const expected = Buffer.from(owned.records.map((record) => `${JSON.stringify(record)}\n`).join(""))
|
|
34
|
+
|
|
35
|
+
const decoded = await run("zstdcat", [segment], {encoding: "buffer", maxBuffer: 4 * 1024 * 1024})
|
|
36
|
+
assert.deepEqual(Buffer.from(decoded.stdout), expected)
|
|
37
|
+
const searched = await run("zstdgrep", ["-c", '"event":"runtime.status"', segment], {encoding: "utf8"})
|
|
38
|
+
assert.equal(searched.stdout.trim(), "4")
|
|
39
|
+
} finally {
|
|
40
|
+
await rm(root, {recursive: true, force: true})
|
|
41
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/* eslint-disable jsdoc/escape-inline-tags, jsdoc/no-undefined-types, jsdoc/require-param-type, jsdoc/require-returns */
|
|
3
|
+
import {once} from "node:events"
|
|
4
|
+
|
|
5
|
+
import {CompressedLogTailReader, readCompressedLogDirectory} from "../logging/compressed-log-reader.js"
|
|
6
|
+
|
|
7
|
+
const FILTER_OPTIONS = new Map([
|
|
8
|
+
["--event", "event"],
|
|
9
|
+
["--incident", "incident"],
|
|
10
|
+
["--gateway", "gateway"],
|
|
11
|
+
["--route-class", "routeClass"],
|
|
12
|
+
["--client-generation", "clientGeneration"],
|
|
13
|
+
["--gateway-generation", "gatewayGeneration"],
|
|
14
|
+
["--reason", "reason"],
|
|
15
|
+
["--since", "since"],
|
|
16
|
+
["--until", "until"]
|
|
17
|
+
])
|
|
18
|
+
|
|
19
|
+
/** @typedef {{kind: 'search' | 'tail', directory: string, filters: Record<string, unknown>, follow: boolean, pollMs: number}} LogCommand */
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
*
|
|
23
|
+
* @param argv
|
|
24
|
+
* @param streams
|
|
25
|
+
*/
|
|
26
|
+
export async function runSocketductCli(/** @type {string[]} */ argv, /** @type {{stdout?: NodeJS.WriteStream, stderr?: NodeJS.WriteStream}} */ streams = {}) {
|
|
27
|
+
const stdout = streams.stdout ?? process.stdout
|
|
28
|
+
const stderr = streams.stderr ?? process.stderr
|
|
29
|
+
try {
|
|
30
|
+
const command = parseArguments(argv)
|
|
31
|
+
if (command.kind === "search") {
|
|
32
|
+
const result = await readCompressedLogDirectory(command.directory, {filters: command.filters})
|
|
33
|
+
if (result.incomplete) {
|
|
34
|
+
await writeOutput(stderr, `Socketduct log evidence error (${result.errors[0]?.code ?? "LOG_EVIDENCE_INVALID"})\n`)
|
|
35
|
+
return 2
|
|
36
|
+
}
|
|
37
|
+
for (const record of result.records) await writeOutput(stdout, `${JSON.stringify(record)}\n`)
|
|
38
|
+
return result.records.length > 0 ? 0 : 1
|
|
39
|
+
}
|
|
40
|
+
const reader = new CompressedLogTailReader(command.directory, {filters: command.filters})
|
|
41
|
+
let stopping = false
|
|
42
|
+
const stop = () => { stopping = true }
|
|
43
|
+
if (command.follow) {
|
|
44
|
+
process.once("SIGINT", stop)
|
|
45
|
+
process.once("SIGTERM", stop)
|
|
46
|
+
}
|
|
47
|
+
try {
|
|
48
|
+
do {
|
|
49
|
+
const result = await reader.poll()
|
|
50
|
+
if (result.incomplete) {
|
|
51
|
+
await writeOutput(stderr, `Socketduct log evidence error (${result.errors[0]?.code ?? "LOG_EVIDENCE_INVALID"})\n`)
|
|
52
|
+
return 2
|
|
53
|
+
}
|
|
54
|
+
for (const record of result.records) {
|
|
55
|
+
await writeOutput(stdout, `${JSON.stringify(record)}\n`)
|
|
56
|
+
}
|
|
57
|
+
if (command.follow && !stopping) await waitForPoll(command.pollMs)
|
|
58
|
+
} while (command.follow && !stopping)
|
|
59
|
+
return 0
|
|
60
|
+
} finally {
|
|
61
|
+
if (command.follow) {
|
|
62
|
+
process.off("SIGINT", stop)
|
|
63
|
+
process.off("SIGTERM", stop)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
} catch {
|
|
67
|
+
await writeOutput(stderr, "Socketduct logs error (INVALID_ARGUMENT_OR_IO)\n")
|
|
68
|
+
return 2
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** @param {string[]} argv - CLI arguments. @returns {LogCommand} */
|
|
73
|
+
function parseArguments(argv) {
|
|
74
|
+
if (argv[0] !== "logs" || (argv[1] !== "search" && argv[1] !== "tail")) throw new Error("Expected logs search or logs tail")
|
|
75
|
+
const kind = argv[1]
|
|
76
|
+
let directory
|
|
77
|
+
let follow = false
|
|
78
|
+
let pollMs = 250
|
|
79
|
+
/** @type {Record<string, unknown>} */
|
|
80
|
+
const filters = {}
|
|
81
|
+
for (let index = 2; index < argv.length; index += 1) {
|
|
82
|
+
const option = argv[index]
|
|
83
|
+
if (option === undefined) throw new Error("Missing option")
|
|
84
|
+
if (option === "--follow" && kind === "tail") { follow = true; continue }
|
|
85
|
+
const value = argv[++index]
|
|
86
|
+
if (value === undefined || value.length === 0) throw new Error("Missing option value")
|
|
87
|
+
if (option === "--directory") { directory = value; continue }
|
|
88
|
+
if (option === "--poll-ms" && kind === "tail") {
|
|
89
|
+
pollMs = positiveInteger(value, "poll interval")
|
|
90
|
+
if (pollMs > 60_000) throw new Error("Poll interval exceeds 60000 milliseconds")
|
|
91
|
+
continue
|
|
92
|
+
}
|
|
93
|
+
const filter = FILTER_OPTIONS.get(option)
|
|
94
|
+
if (filter === undefined) throw new Error("Unknown logs option")
|
|
95
|
+
filters[filter] = validateFilter(filter, value)
|
|
96
|
+
}
|
|
97
|
+
if (directory === undefined) throw new Error("Log directory is required")
|
|
98
|
+
return {kind, directory, filters, follow, pollMs}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* @param {string} filter - Internal filter name. @param {string} value - User input. @returns {string | number}
|
|
103
|
+
* @param value
|
|
104
|
+
*/
|
|
105
|
+
function validateFilter(filter, value) {
|
|
106
|
+
if (filter === "clientGeneration" || filter === "gatewayGeneration") return positiveInteger(value, "connection generation", true)
|
|
107
|
+
if (filter === "since" || filter === "until") {
|
|
108
|
+
const timestamp = Date.parse(value)
|
|
109
|
+
if (!Number.isSafeInteger(timestamp)) throw new Error("Invalid time filter")
|
|
110
|
+
return timestamp
|
|
111
|
+
}
|
|
112
|
+
if (filter === "incident" && !/^[A-Za-z0-9_-]{22}$/.test(value)) throw new Error("Invalid incident filter")
|
|
113
|
+
if ((filter === "gateway" || filter === "routeClass") && !/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(value)) throw new Error("Invalid route filter")
|
|
114
|
+
if ((filter === "event" || filter === "reason") && !/^[a-zA-Z][a-zA-Z0-9_.-]{0,95}$/.test(value)) throw new Error("Invalid diagnostic filter")
|
|
115
|
+
return value
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* @param {string} value - User input. @param {string} name - Setting name. @param {boolean} [allowZero] - Whether zero is valid.
|
|
120
|
+
* @param name
|
|
121
|
+
* @param allowZero
|
|
122
|
+
*/
|
|
123
|
+
function positiveInteger(value, name, allowZero = false) {
|
|
124
|
+
if (!/^(?:0|[1-9][0-9]*)$/.test(value)) throw new Error(`Invalid ${name}`)
|
|
125
|
+
const parsed = Number(value)
|
|
126
|
+
if (!Number.isSafeInteger(parsed) || (allowZero ? parsed < 0 : parsed < 1)) throw new Error(`Invalid ${name}`)
|
|
127
|
+
return parsed
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* @param {NodeJS.WriteStream} stream - Output stream. @param {string} text - Text to write.
|
|
132
|
+
* @param text
|
|
133
|
+
*/
|
|
134
|
+
async function writeOutput(stream, text) {
|
|
135
|
+
if (!stream.write(text)) await once(stream, "drain")
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** @param {number} milliseconds - Poll interval. */
|
|
139
|
+
async function waitForPoll(milliseconds) {
|
|
140
|
+
await new Promise((resolve) => setTimeout(resolve, milliseconds))
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (import.meta.url === `file://${process.argv[1]}`) process.exitCode = await runSocketductCli(process.argv.slice(2))
|
|
@@ -399,20 +399,27 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
|
|
|
399
399
|
}
|
|
400
400
|
|
|
401
401
|
/**
|
|
402
|
-
* Registers a durable stream on its original
|
|
403
|
-
* @param {import('./multiplexed-connection.js').VirtualConnectionOptions & {gatewayId: string}} options - Original gateway
|
|
402
|
+
* Registers a durable stream on its one original member before any member starts.
|
|
403
|
+
* @param {import('./multiplexed-connection.js').VirtualConnectionOptions & ({gatewayId: string, routeClass?: never} | {routeClass: string, gatewayId?: never})} options - Original gateway or route-class recovery identity.
|
|
404
404
|
* @param {(error: Error | null, socket: GatewayVirtualSocket) => void} [callback] - Existing callback convention.
|
|
405
405
|
* @returns {GatewayVirtualSocket} Original member-owned virtual socket with stable gateway metadata.
|
|
406
406
|
*/
|
|
407
407
|
recoverConnection(options, callback) {
|
|
408
408
|
if (this.#closed) throw closedError()
|
|
409
409
|
if (this.#started) throw new Error("Recovered streams must be registered before start()")
|
|
410
|
-
if (typeof options !== "object" || options === null
|
|
410
|
+
if (typeof options !== "object" || options === null) throw new TypeError("recoverConnection requires exactly one configured gatewayId or routeClass")
|
|
411
|
+
const hasGatewayId = Object.hasOwn(options, "gatewayId")
|
|
412
|
+
const hasRouteClass = Object.hasOwn(options, "routeClass")
|
|
413
|
+
if (hasGatewayId === hasRouteClass) throw new TypeError("recoverConnection requires exactly one configured gatewayId or routeClass")
|
|
414
|
+
if (hasGatewayId && (typeof options.gatewayId !== "string" || !GATEWAY_ID.test(options.gatewayId))) throw new TypeError("recoverConnection requires a valid configured gatewayId")
|
|
415
|
+
if (hasRouteClass && (typeof options.routeClass !== "string" || !ROUTE_CLASS.test(options.routeClass))) throw new TypeError("recoverConnection requires a valid configured routeClass")
|
|
411
416
|
if (Object.hasOwn(options, "recover")) throw new TypeError("recoverConnection owns the recover option")
|
|
412
|
-
const
|
|
413
|
-
|
|
417
|
+
const selector = hasGatewayId ? options.gatewayId : options.routeClass
|
|
418
|
+
const member = this.#members.find((candidate) => hasGatewayId ? candidate.gatewayId === selector : candidate.routeClass === selector)
|
|
419
|
+
if (member === undefined) throw new Error(`Unknown ${hasGatewayId ? "gatewayId" : "routeClass"}: ${selector}`)
|
|
414
420
|
const virtualOptions = {...options}
|
|
415
421
|
delete /** @type {{gatewayId?: string}} */ (virtualOptions).gatewayId
|
|
422
|
+
delete /** @type {{routeClass?: string}} */ (virtualOptions).routeClass
|
|
416
423
|
const allocation = memberAllocation(member)
|
|
417
424
|
/** @type {((error: Error | null, socket: import('./virtual-socket.js').VirtualSocket) => void) | undefined} */
|
|
418
425
|
const memberCallback = callback === undefined ? undefined : (error, socket) => callback(error, exposeAllocation(socket, allocation))
|