socketduct 0.1.18 → 0.1.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/.env.example +17 -5
  2. package/README.md +5 -5
  3. package/changelog.d/20260925143000-reverse-client-process-recovery.md +1 -0
  4. package/changelog.d/20260925170000-disconnect-attribution-compressed-logs.md +1 -0
  5. package/docs/architecture.md +11 -9
  6. package/docs/docker.md +5 -4
  7. package/docs/logging.md +131 -159
  8. package/docs/protocol.md +43 -7
  9. package/docs/reverse-gateway.md +75 -9
  10. package/docs/running.md +16 -13
  11. package/docs/security.md +16 -5
  12. package/docs/session-lifecycle.md +14 -3
  13. package/docs/socket-api.md +54 -20
  14. package/package.json +5 -1
  15. package/scripts/verify-zstd-interoperability.mjs +41 -0
  16. package/src/cli/socketduct.js +143 -0
  17. package/src/client/multiplexed-transport-set.js +12 -5
  18. package/src/client/virtual-socket.js +52 -0
  19. package/src/config/environment.js +56 -7
  20. package/src/index.js +64 -26
  21. package/src/logging/compressed-log-reader.js +256 -0
  22. package/src/logging/compressed-log-worker.js +430 -0
  23. package/src/logging/compressed-log-writer.js +538 -0
  24. package/src/logging/log-batch.js +72 -0
  25. package/src/logging/log-retention.js +241 -0
  26. package/src/logging/log-segment-index.js +176 -0
  27. package/src/logging/structured-logger.js +136 -30
  28. package/src/logging/zstd-frame-codec.js +73 -0
  29. package/src/reverse/client.js +372 -82
  30. package/src/reverse/disconnect-attribution.js +191 -0
  31. package/src/reverse/gateway.js +68 -18
  32. package/src/reverse/hub.js +215 -32
  33. package/src/reverse/runtime.js +5 -3
  34. package/src/spool/spool-file.js +136 -7
  35. package/tensorbuzz.yml +4 -2
  36. package/test/e2e/logger-shutdown-process.test.js +86 -0
  37. package/test/e2e/reverse-client-recovery-process.test.js +170 -0
  38. package/test/e2e/reverse-gateway-process.test.js +15 -14
  39. package/test/fixtures/reverse-client-recovery.mjs +123 -0
  40. package/test/integration/logging-cli.test.js +168 -0
  41. package/test/integration/logging-compressed-recovery.test.js +186 -0
  42. package/test/integration/reverse-gateway.test.js +597 -1
  43. package/test/integration/reverse-route-class.test.js +6 -1
  44. package/test/unit/client/multiplexed-transport-set.test.js +12 -1
  45. package/test/unit/config/docker-gateway-files.test.js +2 -0
  46. package/test/unit/config/environment.test.js +54 -14
  47. package/test/unit/index.test.js +3 -0
  48. package/test/unit/logging/compressed-log-worker.test.js +29 -0
  49. package/test/unit/logging/compressed-log-writer.test.js +183 -0
  50. package/test/unit/logging/log-retention.test.js +86 -0
  51. package/test/unit/logging/log-segment-index.test.js +104 -0
  52. package/test/unit/logging/structured-logger.test.js +14 -1
  53. package/test/unit/logging/zstd-frame-codec.test.js +24 -0
  54. package/test/unit/reverse/disconnect-attribution.test.js +82 -0
  55. package/test/unit/spool/spool-file.test.js +133 -0
@@ -125,8 +125,10 @@ in-memory gateway registry across independent cluster workers.
125
125
 
126
126
  Gateway registration reconnects with bounded exponential delay and a higher
127
127
  generation. An established Docker stream survives a transient physical loss
128
- when the hub process and its retained state, live client object, outbound gateway process, and
128
+ when the hub process and its retained state, outbound gateway process, and
129
129
  original target socket remain available through the configured resume window.
130
+ The original client object may reconnect in place, or a replacement reverse
131
+ client process may reconstruct a new `VirtualSocket` from durable spool state.
130
132
  Socketduct keeps
131
133
  the app-facing socket open, durably spools each direction's unacknowledged raw
132
134
  chunks before sending them, reconciles the two receiver-committed ACK positions,
@@ -145,14 +147,39 @@ mismatches, invalid manifests, spool/buffer limits, and authorization failures
145
147
  fail closed.
146
148
 
147
149
  Reverse replay state is bounded per stream and connection and stored in private
148
- temporary spool directories. ACKed chunks are truncated, and terminal
149
- close/reset removes the spool. `status().spoolBytes` reports retained durable
150
- payload. `registerRecoveredConnection()` remains unsupported for reverse mode:
151
- a process replacement cannot inherit the live target socket or the prior
152
- `VirtualSocket`. If either endpoint process disappears, the target socket is
153
- lost, or the hub resume window expires, the logical stream fails once with the
154
- causal session-loss/reset code and the application must reconcile at its own
155
- protocol layer.
150
+ spool directories. ACKed chunks are truncated, and terminal close/reset removes
151
+ the spool. `status().spoolBytes` reports retained durable payload. Reverse-client
152
+ replacement requires the same explicit `connectionId`, an explicit persistent
153
+ `spoolDirectory`, and the same stream identifier, name, target, and one original
154
+ selector (`gatewayId` or `routeClass`). Register every stream with
155
+ `registerRecoveredConnection({...options, recover: true})` before `connect()`;
156
+ registration synchronously rejects missing, expired, corrupt, wrong-owner,
157
+ mismatched, noncontiguous, or oversized state before any network attempt. The
158
+ stored generation is advanced and persisted before AUTH, recovered outbound
159
+ replay starts above the gateway high-water position, and inbound delivery starts
160
+ at the durable receive ACK because unread bytes in the dead `VirtualSocket` no
161
+ longer exist. Receive ACKs advance only after the application consumes bytes,
162
+ not when `VirtualSocket` merely accepts them into its readable buffer. Recovery
163
+ uses the committed recovery record to discard an uncommitted chunk tail and
164
+ repair derivable usage accounting after an abrupt process loss; structural,
165
+ identity, sequence, quota, and non-tail corruption still fail closed. Recovery
166
+ never sends `OPEN`.
167
+
168
+ The default reverse spool is under `/tmp`, and the sample reverse-gateway
169
+ Compose mounts `/tmp` as tmpfs. Those defaults do not survive a container or
170
+ host replacement. A caller claiming process/container recovery must supply a
171
+ private persistent directory; Socketduct maintains `0700` directories and
172
+ `0600` files and never writes hub credentials or session tokens there.
173
+
174
+ This boundary does not reconstruct a hub after hub process loss: the current
175
+ manifests have no durable hub epoch, hub-minted bridge identity, endpoint-pairing
176
+ journal, or split-brain fence. It also does not transfer gateway worker file
177
+ descriptors, recover an outbound gateway process, reopen a lost target socket,
178
+ or replay a Docker request after gateway/container/host loss. A newly constructed
179
+ gateway owns no inherited targets; if the hub reports retained bridges, it sends
180
+ an empty resume manifest so omitted bridges reset once, without calling the
181
+ target factory or emitting `OPEN`. Loss outside the supported boundary fails the
182
+ logical stream and leaves higher-level reconciliation to the application.
156
183
 
157
184
  Target bytes already observed by the gateway are ordered before a subsequent
158
185
  target-error reset. Stream `END` remains a directional half-close, while protocol
@@ -171,6 +198,9 @@ SOCKETDUCT_TLS_KEY_FILE=/certs/hub.key \
171
198
  SOCKETDUCT_AUTH_TOKENS='server-a-starlink:<secret>,server-b-starlink:<secret>,tensorbuzz:<secret>' \
172
199
  SOCKETDUCT_REVERSE_GATEWAY_BINDINGS='server-a-starlink:server-a-starlink,server-b-starlink:server-b-starlink' \
173
200
  SOCKETDUCT_REVERSE_ROUTE_CLASS_ROUTES='tensorbuzz:starlink:server-a-starlink:docker-a:2375,tensorbuzz:starlink:server-b-starlink:docker-b:2375' \
201
+ SOCKETDUCT_LOG_DIRECTORY=/var/log/socketduct \
202
+ SOCKETDUCT_CONNECTION_FINGERPRINT_KEY='<unpadded-base64url-secret>' \
203
+ SOCKETDUCT_CONNECTION_FINGERPRINT_KEY_EPOCH='2026-09' \
174
204
  node src/index.js
175
205
  ```
176
206
 
@@ -182,6 +212,37 @@ Route-class rows use `username:routeClass:gatewayId:host:port`. If two rows make
182
212
  different gateways eligible for the same authenticated class and target, that
183
213
  `OPEN` is rejected as ambiguous rather than choosing one implicitly.
184
214
 
215
+ When hub logging is enabled, the fingerprint key is mandatory, hub-only, at
216
+ least 32 random bytes encoded as canonical unpadded base64url, and never transmitted or
217
+ logged. Rotate it at least every 30 days. The public epoch label identifies the
218
+ key generation used for a 128-bit role-separated HMAC correlation value.
219
+
220
+ ## Disconnect attribution
221
+
222
+ Every reverse physical loss has one 128-bit random `disconnectIncidentId` and
223
+ one first local `disconnectReason`. The initial socket error, EOF, peer GOAWAY,
224
+ protocol violation, output limit, replacement, shutdown, or send-after-loss
225
+ cause wins over later close callbacks. The incident records its originating
226
+ component, physical client/gateway role, explicitly named connection
227
+ generations, and fan-out/resume/terminal stream counts. A public RESET code such
228
+ as `RESUME_WINDOW_EXPIRED` classifies the stream outcome; it does not replace
229
+ the physical cause.
230
+
231
+ The hub propagates its validated incident in RESET/GOAWAY controls and typed
232
+ `SocketductStreamError` / `SocketductSessionLostError` instances. Direct client
233
+ or gateway loss creates and caches an equivalent local incident so concurrent
234
+ stream errors share it, including writes racing the close callback. A new
235
+ authenticated physical clears that cache. Peer-provided reserved attribution
236
+ fields are stripped; peers cannot claim hub origin, inject correlation values,
237
+ or alter named generations.
238
+
239
+ Terminal, non-resumable hub paths assign the incident before removing their
240
+ bridges, including protocol violations, output overflow, replacement, peer
241
+ GOAWAY, and shutdown. A safe peer GOAWAY code survives as terminal
242
+ classification while the first hub-observed `peer_goaway` cause and hub-created
243
+ incident identity remain authoritative; fatal client and gateway connection
244
+ errors retain that validated attribution.
245
+
185
246
  ## One outbound member per target and route class
186
247
 
187
248
  Run this once in each target server's route-pinned network namespace, changing
@@ -238,6 +299,11 @@ Callers cannot override either selection per open. Reverse streams remain
238
299
  sticky to their selected gateway/target but resume transparently after a
239
300
  transient physical transport replacement.
240
301
 
302
+ For process replacement, call `recoverConnection()` before `start()` with
303
+ exactly the socket's immutable original selector: `gatewayId` for an exact set
304
+ or `routeClass` for a route-class set. Recovery derives the original member
305
+ identity and spool namespace and never falls back to or probes a sibling.
306
+
241
307
  The socket is returned immediately and waits inside Socketduct until the set
242
308
  reaches minimum readiness. New sockets rotate equally across ready members and
243
309
  stay on their assigned member. If all members are reconnecting, new sockets wait
package/docs/running.md CHANGED
@@ -58,16 +58,18 @@ roles and always uses verified TLS plus adaptive zstd. See
58
58
  [Native reverse gateway](reverse-gateway.md) for the environment contract and
59
59
  three-member TensorBuzz API.
60
60
 
61
- ## Persistent JSONL logs
62
-
63
- Set `SOCKETDUCT_LOG_DIRECTORY` to opt into bounded, per-process JSONL files. A
64
- Docker deployment should bind mount persistent host storage at that directory.
65
- `SOCKETDUCT_LOG_MAX_FILE_BYTES` defaults to 10 MiB and
66
- `SOCKETDUCT_LOG_MAX_FILES` defaults to five total files per process/role family.
67
- Existing stdout/stderr output is preserved. See
68
- [Persistent structured logging](logging.md) for the mount example, embedded
69
- client `logger` option, rotation layout, event list, and strict redaction
70
- contract.
61
+ ## Durable compressed diagnostic logs
62
+
63
+ Set `SOCKETDUCT_LOG_DIRECTORY` to opt into private, bounded `.jsonl.zst`
64
+ segments and checksummed sidecar indexes. A dedicated worker writes level-19
65
+ checksummed frames; orderly shutdown awaits data and index sync. Defaults are
66
+ 1 MiB frames, 128 MiB segments, 30 days/4 GiB retention, and a 64 MiB queue
67
+ with an 8 MiB critical reserve. A logging reverse hub must also configure its
68
+ hub-only connection-fingerprint key and public key epoch. Existing stdout/stderr
69
+ output is preserved. Use `socketduct logs tail` or `socketduct logs search`, not
70
+ `tail -f` on compressed bytes. See [durable compressed diagnostic
71
+ logs](logging.md) for all settings, lifecycle semantics, CLI exits, embedded
72
+ API, and the strict privacy contract.
71
73
 
72
74
  ## Minimal programmatic relay/edge shape
73
75
 
@@ -107,7 +109,8 @@ Do not hard-code real passwords or token values. Feed credentials through a secr
107
109
 
108
110
  ## MVP limitations
109
111
 
110
- - No stable CLI flags yet.
112
+ - The stable CLI surface currently covers owned compressed-log tail/search;
113
+ tunnel configuration remains environment/programmatic.
111
114
  - Relay targets are requested per session and must be explicitly authorized by the injected target policy.
112
115
  - User/session stores are in-memory in the current MVP skeleton.
113
116
  - Spool files are temporary local files and may contain raw application payload.
@@ -210,6 +213,6 @@ inspect its ordered aggregate causes or privacy-safe per-member status codes.
210
213
  No extra physical member is created. Configure the set-wide buffer/spool/queue
211
214
  limits for the whole process rather than multiplying a desired budget by
212
215
  gateway count.
213
- For process replacement, call `recoverConnection()` with each stream's original
214
- gateway ID before `start()`; see the exact sequence in
216
+ For process replacement, call `recoverConnection()` with exactly each stream's
217
+ original gateway ID or reverse route class before `start()`; see the sequence in
215
218
  [Node-compatible socket API design](socket-api.md#multiplexedsocketducttransportset-multi-gateway-v3-api).
package/docs/security.md CHANGED
@@ -100,7 +100,7 @@ Optional later hardening:
100
100
 
101
101
  ## Logging
102
102
 
103
- The optional persistent JSONL logger uses a closed event/field allowlist. It
103
+ The optional durable compressed logger uses a closed event/field allowlist. It
104
104
  records process/role identity, lifecycle state, bounded counters, validated
105
105
  gateway IDs, and normalized error codes/classes. Unknown event names are
106
106
  dropped and unknown error codes become `UNKNOWN`.
@@ -112,8 +112,12 @@ diagnostics retain bounded error names, messages, and stacks so operational
112
112
  failures can be investigated while credential-shaped values are redacted. Each
113
113
  JSON line is capped at 1024 bytes. Logger I/O and custom-writer failures are
114
114
  isolated from network behavior and produce only one bounded stderr
115
- classification. See [Persistent structured logging](logging.md) for
116
- configuration and rotation.
115
+ classification. Managed files are private checksummed zstd frames with a
116
+ checksummed structural index and no plaintext fallback. Reverse-hub connection
117
+ correlation uses only a hub-held, role-separated HMAC truncated to 128 bits;
118
+ raw connection IDs, the key, unkeyed digests, targets, and payloads are never
119
+ persisted. See [durable compressed diagnostic logs](logging.md) for
120
+ configuration, key rotation, recovery, and retention.
117
121
 
118
122
  V3 status is payload-free and credential-free: physical counts/generation,
119
123
  virtual stream state counts, active local targets, bounded queued/spool bytes,
@@ -176,8 +180,15 @@ only above it. This preserves exact-once delivery without reopening or
176
180
  reissuing a privileged Docker request. Ownership mismatches, missing targets,
177
181
  invalid or incomplete manifests, resource-bound failures, and expired resume
178
182
  windows fail the affected logical stream once instead of falling back to a new
179
- stream. Process replacement remains terminal because the live app and target
180
- sockets cannot be inherited safely.
183
+ stream. Reverse-client process replacement is allowed only from caller-selected
184
+ private persistent storage with the same stable connection/stream identities
185
+ and exactly one original gateway or route-class selector. Recovery validates
186
+ ownership, age, identity, lifecycle, contiguous chunks, and bounds before
187
+ network activity; spool files contain no bearer credential or session token.
188
+ The same in-memory hub bridge, gateway process, and live target socket must
189
+ survive. Hub reconstruction, gateway FD transfer, and target reopen/retry remain
190
+ unsupported, and a replacement gateway resets retained bridges with an empty
191
+ manifest rather than recreating a privileged operation.
181
192
 
182
193
  ## Network exposure
183
194
 
@@ -27,9 +27,20 @@ without opening a second target or retrying an application operation.
27
27
 
28
28
  If the original target socket disappears, an owner/identifier or sequence does
29
29
  not match, a buffer/spool bound is exceeded, or the resume window expires, that
30
- logical stream fails once with the causal error and releases both sides. Reverse
31
- process-replacement recovery remains unsupported because the live target and
32
- app-facing socket cannot be transferred to replacement JavaScript objects.
30
+ logical stream fails once with the causal error and releases both sides. A dead
31
+ reverse-client process may be replaced: the new object creates a new
32
+ `VirtualSocket`, restores the durable generation/lifecycle and replay state, and
33
+ registers every stream before connecting. The same hub bridge, outbound gateway
34
+ process, and original target socket must still exist. The dead app-facing socket
35
+ is not transferred; target bytes above its durable receive ACK replay to the new
36
+ socket. That ACK covers bytes returned to the application, not bytes merely
37
+ buffered in the dead process. An orderly `detach()` drains already-queued frame
38
+ handling before its final durable snapshot and handoff.
39
+
40
+ Hub process reconstruction, gateway process or file-descriptor transfer, and
41
+ target reopen/retry are unsupported. A replacement gateway has no live target
42
+ sockets and therefore completes retained-state registration with an empty
43
+ manifest, resetting omitted bridges rather than issuing a second `OPEN`.
33
44
 
34
45
  The bounded, paginated resume manifest carries stable identifiers and both
35
46
  directional acknowledgement positions. The relay returns an independent result
@@ -219,24 +219,23 @@ the callback, which runs exactly once only after `secureConnect` or failure.
219
219
  Both methods count only completed member assignments and preserve stream
220
220
  stickiness. Every returned raw or TLS socket exposes enumerable `gatewayId` and
221
221
  `routeClass` allocation metadata. Only the configured identity is set. Persist
222
- an exact set's immutable `gatewayId` after connection so a replacement can
223
- recover the stream on its original member. Reverse route-class streams are not
224
- recoverable across process replacement, but they do recover transparently from
225
- a transient physical transport loss while the original objects and target
226
- socket remain alive.
222
+ an exact set's immutable original selector after connection: `gatewayId` for a
223
+ gateway member or `routeClass` for a reverse member. A replacement uses that
224
+ selector to recover the stream on its original member without running selection
225
+ again.
227
226
 
228
227
  Process-replacement recovery is deliberately separate from ordinary selection.
229
228
  First detach the old set when an orderly handoff is possible. Construct the
230
- replacement with the same base `connectionId`, spool directory, gateway IDs and
231
- pinned endpoints. Then register each durable stream with its original
232
- `gatewayId` before calling `start()`:
229
+ replacement with the same base `connectionId`, spool directory, and configured
230
+ member endpoints. Then register each durable stream with exactly its original
231
+ `gatewayId` or `routeClass` before calling `start()`:
233
232
 
234
233
  ```js
235
234
  await set.detach()
236
235
 
237
236
  const replacement = new MultiplexedSocketductTransportSet(sameSetOptions)
238
237
  const recovered = replacement.recoverConnection({
239
- gatewayId: durableRecord.gatewayId, // captured from the original socket
238
+ routeClass: durableRecord.routeClass, // exact sets use gatewayId instead
240
239
  host: "api.internal",
241
240
  port: 443,
242
241
  identifier: "build_stream_01",
@@ -245,10 +244,11 @@ const recovered = replacement.recoverConnection({
245
244
  await replacement.start()
246
245
  ```
247
246
 
248
- `recoverConnection()` implies durable recovery and rejects a missing or unknown
249
- gateway ID, an explicit `recover` option, or registration after `start()` has
250
- begun. It bypasses round-robin only to reach that configured member; it never
251
- migrates or replays the stream on a sibling. Ordinary `createConnection()` and
247
+ `recoverConnection()` implies durable recovery and rejects a missing,
248
+ ambiguous, invalid, or unknown original selector, an explicit `recover` option,
249
+ or registration after `start()` has begun. It bypasses selection only to reach
250
+ that configured member; it never migrates or replays the stream on a sibling.
251
+ Ordinary `createConnection()` and
252
252
  `createTlsConnection()` reject `recover: true`, `gatewayId`, and `routeClass`,
253
253
  so new opens cannot force a path. `detach()` coalesces with concurrent shutdown calls,
254
254
  preserves active member spools, and retires empty warmed lanes; `close()` remains
@@ -401,13 +401,47 @@ Target `ca`, `cert`, and `key` accept Node TLS certificate material. `cert` and
401
401
  It selects exactly one registered gateway during `connect()`. The existing
402
402
  `MultiplexedSocketductTransportSet` accepts `endpoint.reverseHub`, allowing
403
403
  three ready members to retain deterministic health-aware round-robin behavior.
404
- Reverse established operations deliberately do not support
405
- `registerRecoveredConnection()` across process replacement. They do durably
406
- spool unacknowledged send chunks during the lifetime of the client and gateway
407
- objects, then use paired v3 resume manifests to survive physical transport
408
- replacement without reopening the target or replaying an application request
409
- as a new logical stream. Target loss, resume expiry, identity mismatch, or a
410
- configured bound remains terminal and requires application reconciliation.
404
+ An established reverse stream can be cold-registered in a replacement client
405
+ process before its first physical connection:
406
+
407
+ ```js
408
+ const replacement = new ReverseGatewayConnection({
409
+ routeClass: durableRecord.routeClass,
410
+ connectionId: durableRecord.connectionId,
411
+ spoolDirectory: "/srv/socketduct/recovery",
412
+ hub,
413
+ tls
414
+ })
415
+ const socket = replacement.registerRecoveredConnection({
416
+ recover: true,
417
+ host: durableRecord.host,
418
+ port: durableRecord.port,
419
+ identifier: durableRecord.identifier,
420
+ sessionName: durableRecord.sessionName
421
+ })
422
+ await replacement.connect()
423
+ ```
424
+
425
+ Recovery restores the durable stream lifecycle, sequences, acknowledgements,
426
+ and generation. It never emits a new `OPEN` or recreates the target; target data
427
+ above the durable receive acknowledgement is replayed by the surviving gateway.
428
+ Reverse receive ACKs cover bytes returned to the application, not unread bytes
429
+ held only in the old process. `detach()` resolves only after queued physical
430
+ frame handling is quiescent and the resulting recovery state is durable, so a
431
+ replacement cannot race a late write from the retired object.
432
+ The caller must supply stable `connectionId` and private persistent
433
+ `spoolDirectory` values. The default `/tmp` spool and the sample gateway
434
+ Compose tmpfs do not survive container or host replacement. Spool directories
435
+ remain mode 0700, files remain mode 0600, and credentials or session tokens are
436
+ not persisted.
437
+
438
+ This boundary requires the same hub bridge, reverse-gateway process, and target
439
+ TCP socket to survive. It does not reconstruct a restarted hub, keep or transfer
440
+ gateway sockets between processes, reopen a lost target, or replay an ambiguous
441
+ target request after gateway, container, or host loss. If a newly constructed
442
+ outbound gateway finds retained hub bridges but owns no live target sockets, it
443
+ sends an empty resume manifest so the hub resets those omitted streams instead
444
+ of creating targets.
411
445
 
412
446
  Reverse virtual sockets accept individual writes larger than their configured
413
447
  retained-byte limit. The connection progresses those writes in slices bounded by
package/package.json CHANGED
@@ -1,9 +1,12 @@
1
1
  {
2
2
  "name": "socketduct",
3
- "version": "0.1.18",
3
+ "version": "0.1.20",
4
4
  "description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
5
5
  "type": "module",
6
6
  "main": "./src/index.js",
7
+ "bin": {
8
+ "socketduct": "./src/cli/socketduct.js"
9
+ },
7
10
  "exports": {
8
11
  ".": "./src/index.js",
9
12
  "./http": "./src/client/http-agent.js",
@@ -21,6 +24,7 @@
21
24
  "release:patch": "npm run check && release-patch",
22
25
  "start": "node src/index.js",
23
26
  "test": "node scripts/test.js",
27
+ "verify:zstd-logs": "node scripts/verify-zstd-interoperability.mjs",
24
28
  "typecheck": "tsc --noEmit",
25
29
  "lint": "eslint . --max-warnings 0"
26
30
  },
@@ -0,0 +1,41 @@
1
+ import assert from "node:assert/strict"
2
+ import {execFile} from "node:child_process"
3
+ import {mkdtemp, readdir, rm} from "node:fs/promises"
4
+ import {tmpdir} from "node:os"
5
+ import {join} from "node:path"
6
+ import {promisify} from "node:util"
7
+
8
+ import {readCompressedLogDirectory} from "../src/logging/compressed-log-reader.js"
9
+ import {createCompressedJsonlFileLogger} from "../src/logging/compressed-log-writer.js"
10
+
11
+ const run = promisify(execFile)
12
+ const root = await mkdtemp(join(tmpdir(), "socketduct-zstd-reference-"))
13
+ const directory = join(root, "logs")
14
+
15
+ try {
16
+ let tick = 0
17
+ const logger = createCompressedJsonlFileLogger({
18
+ directory,
19
+ role: "reverse-hub",
20
+ now: () => new Date(1_795_000_000_000 + tick++),
21
+ frameMaxBytes: 256,
22
+ maxSegmentBytes: 1024 * 1024
23
+ })
24
+ logger.log("process.started", {runtimeMode: "reverse-hub"})
25
+ for (let value = 1; value <= 4; value += 1) logger.log("runtime.status", {virtualStreams: value})
26
+ await logger.close()
27
+
28
+ const segmentNames = (await readdir(directory)).filter((name) => name.endsWith(".jsonl.zst"))
29
+ assert.equal(segmentNames.length, 1)
30
+ const segment = join(directory, /** @type {string} */ (segmentNames[0]))
31
+ const owned = await readCompressedLogDirectory(directory)
32
+ assert.equal(owned.incomplete, false)
33
+ const expected = Buffer.from(owned.records.map((record) => `${JSON.stringify(record)}\n`).join(""))
34
+
35
+ const decoded = await run("zstdcat", [segment], {encoding: "buffer", maxBuffer: 4 * 1024 * 1024})
36
+ assert.deepEqual(Buffer.from(decoded.stdout), expected)
37
+ const searched = await run("zstdgrep", ["-c", '"event":"runtime.status"', segment], {encoding: "utf8"})
38
+ assert.equal(searched.stdout.trim(), "4")
39
+ } finally {
40
+ await rm(root, {recursive: true, force: true})
41
+ }
@@ -0,0 +1,143 @@
1
+ #!/usr/bin/env node
2
+ /* eslint-disable jsdoc/escape-inline-tags, jsdoc/no-undefined-types, jsdoc/require-param-type, jsdoc/require-returns */
3
+ import {once} from "node:events"
4
+
5
+ import {CompressedLogTailReader, readCompressedLogDirectory} from "../logging/compressed-log-reader.js"
6
+
7
+ const FILTER_OPTIONS = new Map([
8
+ ["--event", "event"],
9
+ ["--incident", "incident"],
10
+ ["--gateway", "gateway"],
11
+ ["--route-class", "routeClass"],
12
+ ["--client-generation", "clientGeneration"],
13
+ ["--gateway-generation", "gatewayGeneration"],
14
+ ["--reason", "reason"],
15
+ ["--since", "since"],
16
+ ["--until", "until"]
17
+ ])
18
+
19
+ /** @typedef {{kind: 'search' | 'tail', directory: string, filters: Record<string, unknown>, follow: boolean, pollMs: number}} LogCommand */
20
+
21
+ /**
22
+ *
23
+ * @param argv
24
+ * @param streams
25
+ */
26
+ export async function runSocketductCli(/** @type {string[]} */ argv, /** @type {{stdout?: NodeJS.WriteStream, stderr?: NodeJS.WriteStream}} */ streams = {}) {
27
+ const stdout = streams.stdout ?? process.stdout
28
+ const stderr = streams.stderr ?? process.stderr
29
+ try {
30
+ const command = parseArguments(argv)
31
+ if (command.kind === "search") {
32
+ const result = await readCompressedLogDirectory(command.directory, {filters: command.filters})
33
+ if (result.incomplete) {
34
+ await writeOutput(stderr, `Socketduct log evidence error (${result.errors[0]?.code ?? "LOG_EVIDENCE_INVALID"})\n`)
35
+ return 2
36
+ }
37
+ for (const record of result.records) await writeOutput(stdout, `${JSON.stringify(record)}\n`)
38
+ return result.records.length > 0 ? 0 : 1
39
+ }
40
+ const reader = new CompressedLogTailReader(command.directory, {filters: command.filters})
41
+ let stopping = false
42
+ const stop = () => { stopping = true }
43
+ if (command.follow) {
44
+ process.once("SIGINT", stop)
45
+ process.once("SIGTERM", stop)
46
+ }
47
+ try {
48
+ do {
49
+ const result = await reader.poll()
50
+ if (result.incomplete) {
51
+ await writeOutput(stderr, `Socketduct log evidence error (${result.errors[0]?.code ?? "LOG_EVIDENCE_INVALID"})\n`)
52
+ return 2
53
+ }
54
+ for (const record of result.records) {
55
+ await writeOutput(stdout, `${JSON.stringify(record)}\n`)
56
+ }
57
+ if (command.follow && !stopping) await waitForPoll(command.pollMs)
58
+ } while (command.follow && !stopping)
59
+ return 0
60
+ } finally {
61
+ if (command.follow) {
62
+ process.off("SIGINT", stop)
63
+ process.off("SIGTERM", stop)
64
+ }
65
+ }
66
+ } catch {
67
+ await writeOutput(stderr, "Socketduct logs error (INVALID_ARGUMENT_OR_IO)\n")
68
+ return 2
69
+ }
70
+ }
71
+
72
+ /** @param {string[]} argv - CLI arguments. @returns {LogCommand} */
73
+ function parseArguments(argv) {
74
+ if (argv[0] !== "logs" || (argv[1] !== "search" && argv[1] !== "tail")) throw new Error("Expected logs search or logs tail")
75
+ const kind = argv[1]
76
+ let directory
77
+ let follow = false
78
+ let pollMs = 250
79
+ /** @type {Record<string, unknown>} */
80
+ const filters = {}
81
+ for (let index = 2; index < argv.length; index += 1) {
82
+ const option = argv[index]
83
+ if (option === undefined) throw new Error("Missing option")
84
+ if (option === "--follow" && kind === "tail") { follow = true; continue }
85
+ const value = argv[++index]
86
+ if (value === undefined || value.length === 0) throw new Error("Missing option value")
87
+ if (option === "--directory") { directory = value; continue }
88
+ if (option === "--poll-ms" && kind === "tail") {
89
+ pollMs = positiveInteger(value, "poll interval")
90
+ if (pollMs > 60_000) throw new Error("Poll interval exceeds 60000 milliseconds")
91
+ continue
92
+ }
93
+ const filter = FILTER_OPTIONS.get(option)
94
+ if (filter === undefined) throw new Error("Unknown logs option")
95
+ filters[filter] = validateFilter(filter, value)
96
+ }
97
+ if (directory === undefined) throw new Error("Log directory is required")
98
+ return {kind, directory, filters, follow, pollMs}
99
+ }
100
+
101
+ /**
102
+ * @param {string} filter - Internal filter name. @param {string} value - User input. @returns {string | number}
103
+ * @param value
104
+ */
105
+ function validateFilter(filter, value) {
106
+ if (filter === "clientGeneration" || filter === "gatewayGeneration") return positiveInteger(value, "connection generation", true)
107
+ if (filter === "since" || filter === "until") {
108
+ const timestamp = Date.parse(value)
109
+ if (!Number.isSafeInteger(timestamp)) throw new Error("Invalid time filter")
110
+ return timestamp
111
+ }
112
+ if (filter === "incident" && !/^[A-Za-z0-9_-]{22}$/.test(value)) throw new Error("Invalid incident filter")
113
+ if ((filter === "gateway" || filter === "routeClass") && !/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(value)) throw new Error("Invalid route filter")
114
+ if ((filter === "event" || filter === "reason") && !/^[a-zA-Z][a-zA-Z0-9_.-]{0,95}$/.test(value)) throw new Error("Invalid diagnostic filter")
115
+ return value
116
+ }
117
+
118
+ /**
119
+ * @param {string} value - User input. @param {string} name - Setting name. @param {boolean} [allowZero] - Whether zero is valid.
120
+ * @param name
121
+ * @param allowZero
122
+ */
123
+ function positiveInteger(value, name, allowZero = false) {
124
+ if (!/^(?:0|[1-9][0-9]*)$/.test(value)) throw new Error(`Invalid ${name}`)
125
+ const parsed = Number(value)
126
+ if (!Number.isSafeInteger(parsed) || (allowZero ? parsed < 0 : parsed < 1)) throw new Error(`Invalid ${name}`)
127
+ return parsed
128
+ }
129
+
130
+ /**
131
+ * @param {NodeJS.WriteStream} stream - Output stream. @param {string} text - Text to write.
132
+ * @param text
133
+ */
134
+ async function writeOutput(stream, text) {
135
+ if (!stream.write(text)) await once(stream, "drain")
136
+ }
137
+
138
+ /** @param {number} milliseconds - Poll interval. */
139
+ async function waitForPoll(milliseconds) {
140
+ await new Promise((resolve) => setTimeout(resolve, milliseconds))
141
+ }
142
+
143
+ if (import.meta.url === `file://${process.argv[1]}`) process.exitCode = await runSocketductCli(process.argv.slice(2))
@@ -399,20 +399,27 @@ export class MultiplexedSocketductTransportSet extends EventEmitter {
399
399
  }
400
400
 
401
401
  /**
402
- * Registers a durable stream on its original gateway before any member starts.
403
- * @param {import('./multiplexed-connection.js').VirtualConnectionOptions & {gatewayId: string}} options - Original gateway and recovery identity.
402
+ * Registers a durable stream on its one original member before any member starts.
403
+ * @param {import('./multiplexed-connection.js').VirtualConnectionOptions & ({gatewayId: string, routeClass?: never} | {routeClass: string, gatewayId?: never})} options - Original gateway or route-class recovery identity.
404
404
  * @param {(error: Error | null, socket: GatewayVirtualSocket) => void} [callback] - Existing callback convention.
405
405
  * @returns {GatewayVirtualSocket} Original member-owned virtual socket with stable gateway metadata.
406
406
  */
407
407
  recoverConnection(options, callback) {
408
408
  if (this.#closed) throw closedError()
409
409
  if (this.#started) throw new Error("Recovered streams must be registered before start()")
410
- if (typeof options !== "object" || options === null || typeof options.gatewayId !== "string" || !GATEWAY_ID.test(options.gatewayId)) throw new TypeError("recoverConnection requires a valid configured gatewayId")
410
+ if (typeof options !== "object" || options === null) throw new TypeError("recoverConnection requires exactly one configured gatewayId or routeClass")
411
+ const hasGatewayId = Object.hasOwn(options, "gatewayId")
412
+ const hasRouteClass = Object.hasOwn(options, "routeClass")
413
+ if (hasGatewayId === hasRouteClass) throw new TypeError("recoverConnection requires exactly one configured gatewayId or routeClass")
414
+ if (hasGatewayId && (typeof options.gatewayId !== "string" || !GATEWAY_ID.test(options.gatewayId))) throw new TypeError("recoverConnection requires a valid configured gatewayId")
415
+ if (hasRouteClass && (typeof options.routeClass !== "string" || !ROUTE_CLASS.test(options.routeClass))) throw new TypeError("recoverConnection requires a valid configured routeClass")
411
416
  if (Object.hasOwn(options, "recover")) throw new TypeError("recoverConnection owns the recover option")
412
- const member = this.#members.find((candidate) => candidate.gatewayId === options.gatewayId)
413
- if (member === undefined) throw new Error(`Unknown gatewayId: ${options.gatewayId}`)
417
+ const selector = hasGatewayId ? options.gatewayId : options.routeClass
418
+ const member = this.#members.find((candidate) => hasGatewayId ? candidate.gatewayId === selector : candidate.routeClass === selector)
419
+ if (member === undefined) throw new Error(`Unknown ${hasGatewayId ? "gatewayId" : "routeClass"}: ${selector}`)
414
420
  const virtualOptions = {...options}
415
421
  delete /** @type {{gatewayId?: string}} */ (virtualOptions).gatewayId
422
+ delete /** @type {{routeClass?: string}} */ (virtualOptions).routeClass
416
423
  const allocation = memberAllocation(member)
417
424
  /** @type {((error: Error | null, socket: import('./virtual-socket.js').VirtualSocket) => void) | undefined} */
418
425
  const memberCallback = callback === undefined ? undefined : (error, socket) => callback(error, exposeAllocation(socket, allocation))