socketduct 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +11 -0
- package/.env.example +131 -0
- package/AGENTS.md +168 -0
- package/Dockerfile +30 -0
- package/README.md +134 -0
- package/changelog.d/20260716083144-v2-ack-wire-format.md +3 -0
- package/changelog.d/20260716092044-v2-only-stream-framing.md +1 -0
- package/changelog.d/20260727113000-relay-connect-timeout-destroy.md +1 -0
- package/changelog.d/20260813093000-private-relay-hardening.md +1 -0
- package/changelog.d/20260824183412-initial-target-recovery-https.md +1 -0
- package/changelog.d/20260824230000-tls-relay-session-close.md +1 -0
- package/changelog.d/20260825120000-reconnect-close-target-timeout.md +1 -0
- package/changelog.d/20260829120000-protocol-v3-multiplexing.md +1 -0
- package/changelog.d/20260829160000-v3-review-corrections.md +1 -0
- package/changelog.d/20260830000000-fix-socket-idle-timeout.md +1 -0
- package/changelog.d/20260830090000-multi-gateway-transport-set.md +1 -0
- package/changelog.d/20260830093000-relay-close-settlement.md +1 -0
- package/changelog.d/20260830120000-immutable-gateway-image.md +1 -0
- package/changelog.d/20260902210000-native-reverse-gateway.md +1 -0
- package/changelog.d/20260902230000-relay-resume-terminal-order.md +1 -0
- package/changelog.d/20260902230708-multiplexed-open-ownership.md +1 -0
- package/changelog.d/20260902231541-multiplexed-terminal-close.md +1 -0
- package/changelog.d/20260902231542-reverse-close-position.md +1 -0
- package/changelog.d/20260902231543-reverse-terminal-connect.md +1 -0
- package/changelog.d/20260902233000-reverse-gateway-bounds.md +1 -0
- package/changelog.d/20260903001000-reverse-data-ordering.md +1 -0
- package/changelog.d/20260903003000-reverse-hub-restart-window.md +1 -0
- package/changelog.d/20260903010000-reverse-terminal-lifecycle.md +1 -0
- package/changelog.d/20260904142743-transport-set-readiness-wait.md +1 -0
- package/changelog.d/20260904152855-deferred-open-snapshots-binary-views.md +1 -0
- package/changelog.d/20260904163041-tensorbuzz-container-workflow.md +1 -0
- package/changelog.d/20260904174955-transport-set-terminal-causes.md +1 -0
- package/changelog.d/20260905081000-contain-reverse-auth-write-errors.md +1 -0
- package/changelog.d/20260905100520-reverse-hub-close-race.md +1 -0
- package/changelog.d/20260905120000-persistent-jsonl-logging.md +1 -0
- package/changelog.d/20260908071009-jsonl-partial-append-recovery.md +1 -0
- package/changelog.d/20260908074916-private-log-directory-setup.md +1 -0
- package/changelog.d/20260909093057-reverse-large-write-backpressure.md +1 -0
- package/changelog.d/20260909135830-actionable-error-diagnostics.md +1 -0
- package/changelog.d/20260909150000-transport-set-error-redaction.md +1 -0
- package/docker-compose.dev.yml +19 -0
- package/docker-compose.reverse-gateway.yml +36 -0
- package/docker-compose.yml +73 -0
- package/docs/architecture.md +264 -0
- package/docs/docker-topology.md +164 -0
- package/docs/docker.md +281 -0
- package/docs/logging.md +185 -0
- package/docs/plans/initial-implementation.md +205 -0
- package/docs/plans/socket-docker-integration.md +221 -0
- package/docs/protocol.md +558 -0
- package/docs/releasing.md +18 -0
- package/docs/reverse-gateway.md +186 -0
- package/docs/runbooks/jsonl-integrity-rollout.md +144 -0
- package/docs/running.md +215 -0
- package/docs/security.md +258 -0
- package/docs/session-lifecycle.md +127 -0
- package/docs/socket-api.md +448 -0
- package/eslint.config.js +36 -0
- package/package.json +44 -0
- package/scripts/deploy-gateway.js +194 -0
- package/scripts/test.js +28 -0
- package/src/auth/passwords.js +38 -0
- package/src/auth/static-token-store.js +41 -0
- package/src/auth/user-store.js +139 -0
- package/src/client/connect.js +76 -0
- package/src/client/deferred-virtual-socket.js +312 -0
- package/src/client/http-agent.js +101 -0
- package/src/client/https-agent.js +366 -0
- package/src/client/multiplexed-connection.js +1504 -0
- package/src/client/multiplexed-transport-set.js +917 -0
- package/src/client/multiplexed.js +3 -0
- package/src/client/socketduct-socket.js +1174 -0
- package/src/client/virtual-socket.js +256 -0
- package/src/config/environment.js +400 -0
- package/src/edge/tcp-edge.js +82 -0
- package/src/index.js +101 -0
- package/src/logging/structured-logger.js +745 -0
- package/src/protocol/compression.js +325 -0
- package/src/protocol/frame.js +341 -0
- package/src/protocol/v3-compression.js +44 -0
- package/src/protocol/v3-frame.js +209 -0
- package/src/proxy/multiplexed-relay.js +798 -0
- package/src/proxy/tcp-relay.js +1191 -0
- package/src/relay.js +3 -0
- package/src/reverse/client.js +721 -0
- package/src/reverse/common.js +88 -0
- package/src/reverse/gateway.js +663 -0
- package/src/reverse/hub.js +563 -0
- package/src/reverse/index.js +3 -0
- package/src/reverse/runtime.js +78 -0
- package/src/server/control-handler.js +170 -0
- package/src/server/primary.js +64 -0
- package/src/server/relay-runtime.js +124 -0
- package/src/server/worker.js +39 -0
- package/src/sessions/session.js +212 -0
- package/src/spool/spool-file.js +451 -0
- package/tensorbuzz.yml +36 -0
- package/test/e2e/gateway-process.test.js +262 -0
- package/test/e2e/reverse-gateway-process.test.js +285 -0
- package/test/fixtures/logging/enospc-matrix.mjs +238 -0
- package/test/fixtures/logging/enospc-recovery.mjs +146 -0
- package/test/fixtures/logging/process-writer.mjs +20 -0
- package/test/helpers/tls-certificates.js +115 -0
- package/test/integration/initial-target-recovery.test.js +518 -0
- package/test/integration/logging-recovery.test.js +163 -0
- package/test/integration/multiplexed-compatibility.test.js +254 -0
- package/test/integration/multiplexed-corrections.test.js +551 -0
- package/test/integration/multiplexed-failures.test.js +339 -0
- package/test/integration/multiplexed-transport-set.test.js +456 -0
- package/test/integration/multiplexed-transport.test.js +208 -0
- package/test/integration/reverse-gateway.test.js +1293 -0
- package/test/integration/socketduct-http-agent.test.js +435 -0
- package/test/integration/socketduct-https-agent.test.js +669 -0
- package/test/integration/stream-compression.test.js +1267 -0
- package/test/integration/tcp-forwarding.test.js +2878 -0
- package/test/integration/tls-forwarding.test.js +748 -0
- package/test/unit/auth/auth.test.js +64 -0
- package/test/unit/auth/static-token-store.test.js +14 -0
- package/test/unit/client/connect.test.js +58 -0
- package/test/unit/client/multiplexed-spool-budget.test.js +309 -0
- package/test/unit/client/multiplexed-transport-set.test.js +906 -0
- package/test/unit/client/socketduct-socket.test.js +1124 -0
- package/test/unit/client/virtual-socket.test.js +98 -0
- package/test/unit/config/docker-gateway-files.test.js +274 -0
- package/test/unit/config/environment.test.js +507 -0
- package/test/unit/index.test.js +78 -0
- package/test/unit/logging/structured-logger.test.js +674 -0
- package/test/unit/protocol/compression.test.js +421 -0
- package/test/unit/protocol/frame.test.js +336 -0
- package/test/unit/protocol/v3-compression.test.js +31 -0
- package/test/unit/protocol/v3-frame.test.js +57 -0
- package/test/unit/release-configuration.test.js +21 -0
- package/test/unit/server/control-handler.test.js +291 -0
- package/test/unit/server/primary.test.js +59 -0
- package/test/unit/server/relay-runtime.test.js +434 -0
- package/test/unit/sessions/session.test.js +84 -0
- package/test/unit/spool/spool-file.test.js +295 -0
- package/tsconfig.json +20 -0
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
# Node-Compatible Socket API Design
|
|
2
|
+
|
|
3
|
+
Socketduct should be usable by code that already expects Node's socket-shaped APIs. The goal is not to subclass or perfectly emulate every `net.Socket` internal detail. The goal is a practical duck-typed client object that behaves like a normal `Duplex`/`net.Socket` for common consumers such as HTTP clients and Docker client libraries.
|
|
4
|
+
|
|
5
|
+
## Goals
|
|
6
|
+
|
|
7
|
+
- Expose a `SocketductSocket` that is `Duplex`-compatible and close enough to `net.Socket` for normal stream consumers.
|
|
8
|
+
- Provide a `connect()` factory with an option shape that mirrors `net.connect()` where it makes sense.
|
|
9
|
+
- Provide HTTP and HTTPS agent adapters so clients can route individual requests over Socketduct without opening a local listener.
|
|
10
|
+
- Keep the existing relay security model: authenticate before session operations and authorize every requested target with deny-by-default policy.
|
|
11
|
+
- Make Docker API clients a first-class target without baking Docker-specific behavior into the core socket.
|
|
12
|
+
|
|
13
|
+
## Non-goals
|
|
14
|
+
|
|
15
|
+
- Do not monkey-patch `node:net` globally.
|
|
16
|
+
- Do not promise transparent recovery after the remote target has closed its own connection.
|
|
17
|
+
- Do not expose an unauthenticated generic open proxy.
|
|
18
|
+
- Do not require consumers to use TypeScript. Public docs and source stay JavaScript with JSDoc.
|
|
19
|
+
|
|
20
|
+
## API shape
|
|
21
|
+
|
|
22
|
+
### `MultiplexedSocketductConnection` (primary v3 API)
|
|
23
|
+
|
|
24
|
+
```js
|
|
25
|
+
import {MultiplexedSocketductConnection} from "socketduct/multiplexed"
|
|
26
|
+
|
|
27
|
+
const transport = new MultiplexedSocketductConnection({
|
|
28
|
+
relay: {host: "relay.example.net", port: 3100, token: process.env.SOCKETDUCT_TOKEN},
|
|
29
|
+
tls: {caFile: "/certs/relay-ca.crt", servername: "relay.example.net"},
|
|
30
|
+
spoolDirectory: "/tmp/socketduct-v3",
|
|
31
|
+
maxAgeMs: 86_400_000,
|
|
32
|
+
connectionId: "runner_01_stable_identity"
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
// The method is pre-bound and has normal createConnection(options, callback)
|
|
36
|
+
// semantics, so it can be passed directly to a generic TCP/HTTP client.
|
|
37
|
+
const socket = transport.createConnection({
|
|
38
|
+
host: "api.internal",
|
|
39
|
+
port: 443,
|
|
40
|
+
sessionName: "api-1"
|
|
41
|
+
}, (error, connectedSocket) => {
|
|
42
|
+
if (error) throw error
|
|
43
|
+
connectedSocket.write("...")
|
|
44
|
+
})
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
The first call opens and authenticates one physical TCP/TLS connection. Every
|
|
48
|
+
later call allocates only a `VirtualSocket`; it does not create another WAN
|
|
49
|
+
transport. `VirtualSocket` extends `Duplex` and implements the ordinary socket
|
|
50
|
+
events, half-close, timeout, destruction, pause/resume, address metadata,
|
|
51
|
+
`ref`/`unref`, and compatible `setNoDelay`/`setKeepAlive` behavior. Standard
|
|
52
|
+
Node HTTP can use `agent.createConnection = transport.createConnection`; no
|
|
53
|
+
Socketduct-specific HTTP agent is required.
|
|
54
|
+
|
|
55
|
+
Inner TLS remains supported:
|
|
56
|
+
|
|
57
|
+
```js
|
|
58
|
+
const raw = transport.createConnection({host: "api.internal", port: 443})
|
|
59
|
+
const secure = tls.connect({socket: raw, servername: "api.internal", ca})
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`SocketductStreamError` is stream-local. `SocketductSessionLostError` means a
|
|
63
|
+
resumed stream's target/session could not be retained and callers must reconcile
|
|
64
|
+
any uncertain higher-level operation. Closing one virtual socket never closes
|
|
65
|
+
the shared transport; call `await transport.close()` to close the owner.
|
|
66
|
+
|
|
67
|
+
Durable recovery across a client-object or process replacement is explicit.
|
|
68
|
+
Supply the same caller-owned `connectionId`, `spoolDirectory`, stream
|
|
69
|
+
`identifier`, target and name, register every recovered stream synchronously
|
|
70
|
+
before the replacement transport starts, and set `recover: true`:
|
|
71
|
+
|
|
72
|
+
```js
|
|
73
|
+
await oldTransport.detach() // optional orderly handoff; a process crash also detaches
|
|
74
|
+
const replacement = new MultiplexedSocketductConnection({
|
|
75
|
+
relay,
|
|
76
|
+
connectionId: "runner_01_stable_identity",
|
|
77
|
+
spoolDirectory: "/tmp/socketduct-v3"
|
|
78
|
+
})
|
|
79
|
+
const socket = replacement.createConnection({
|
|
80
|
+
host: "api.internal",
|
|
81
|
+
port: 443,
|
|
82
|
+
identifier: "build_stream_01",
|
|
83
|
+
sessionName: "build",
|
|
84
|
+
recover: true
|
|
85
|
+
})
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Fresh opens use exclusive spool creation and never truncate an existing stable
|
|
89
|
+
identifier. Recovery validates bounded metadata, connection/stream identity,
|
|
90
|
+
target, generation, sequence positions and contiguous replay chunks; missing,
|
|
91
|
+
corrupt, expired or inconsistent state fails closed. V3 spools default to a
|
|
92
|
+
24-hour `maxAgeMs`; callers may set a shorter positive bound. Constructing a
|
|
93
|
+
spool owner removes well-formed abandoned entries older than that bound so
|
|
94
|
+
expired stable identifiers and disk space are reclaimed. Client-to-target bytes are durable
|
|
95
|
+
before transmission, so acknowledged bytes are not replayed and an existing
|
|
96
|
+
relay target is not reopened. The directional acknowledgement positions are
|
|
97
|
+
also durable. A replacement object resumes target-to-client delivery from its
|
|
98
|
+
durable receive acknowledgement rather than from unread bytes buffered only by
|
|
99
|
+
the old `VirtualSocket`; same-object reconnects retain that readable-buffer
|
|
100
|
+
optimization. An opening stream whose `OPEN` may already have reached the relay
|
|
101
|
+
is reconciled through the resume result and is never opened fresh speculatively.
|
|
102
|
+
Callers recovering higher-level request/response protocols remain
|
|
103
|
+
responsible for durably reconciling application state after uncertain results.
|
|
104
|
+
`detach()` requires an explicit `connectionId`, preserves active spools and
|
|
105
|
+
does not send stream resets. `close()` remains terminal and removes spools.
|
|
106
|
+
|
|
107
|
+
### `MultiplexedSocketductTransportSet` (multi-gateway v3 API)
|
|
108
|
+
|
|
109
|
+
Use one transport set per Node process when the process has several genuinely
|
|
110
|
+
distinct gateway paths:
|
|
111
|
+
|
|
112
|
+
```js
|
|
113
|
+
import {MultiplexedSocketductTransportSet} from "socketduct/multiplexed"
|
|
114
|
+
|
|
115
|
+
const set = new MultiplexedSocketductTransportSet({
|
|
116
|
+
connectionId: "runner_01_stable_identity",
|
|
117
|
+
spoolDirectory: "/tmp/socketduct-v3-set",
|
|
118
|
+
minimumReadyMembers: 1,
|
|
119
|
+
maxTotalBufferBytes: 32 * 1024 * 1024,
|
|
120
|
+
maxTotalSpoolBytes: 32 * 1024 * 1024,
|
|
121
|
+
maxTotalQueuedBytes: 33 * 1024 * 1024,
|
|
122
|
+
maxPendingOpens: 1024,
|
|
123
|
+
memberOptions: {reconnectDelayMs: 100},
|
|
124
|
+
gateways: [
|
|
125
|
+
{gatewayId: "starlink", endpoint: {relay: {host: "192.0.2.10", port: 3100, token}}},
|
|
126
|
+
{gatewayId: "zyxel", endpoint: {relay: {host: "192.0.2.11", port: 3100, token}}},
|
|
127
|
+
{gatewayId: "o2gateway", endpoint: {relay: {host: "192.0.2.12", port: 3100, token}}}
|
|
128
|
+
]
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
const starting = set.start()
|
|
132
|
+
agent.createConnection = set.createConnection
|
|
133
|
+
await starting
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Every entry must have a stable 2-64 character `gatewayId` and an explicit
|
|
137
|
+
`endpoint.relay`. An optional, gateway-specific `endpoint.createPhysicalSocket`
|
|
138
|
+
can pin an interface, routing table or external transport. Repeating the same
|
|
139
|
+
relay address without distinct pinned factories is rejected: assigning three
|
|
140
|
+
names to one load-balanced endpoint is not path diversity. Socketduct cannot
|
|
141
|
+
inspect arbitrary DNS or factory routing, so callers remain responsible for
|
|
142
|
+
making each declared endpoint/factory genuinely gateway-pinned. Blank,
|
|
143
|
+
duplicate and path-unsafe IDs and malformed endpoint entries fail synchronously.
|
|
144
|
+
|
|
145
|
+
`start()` constructs no new members: the constructor creates exactly one
|
|
146
|
+
`MultiplexedSocketductConnection` per gateway and `start()` warms all of them
|
|
147
|
+
concurrently. Concurrent starts share one promise. It resolves when
|
|
148
|
+
`minimumReadyMembers` (default 1) are authenticated; slower/unavailable members
|
|
149
|
+
keep reconnecting and rejoin automatically. New virtual sockets are assigned in
|
|
150
|
+
configured-order round-robin across members whose physical state is `ready`.
|
|
151
|
+
Fairness is per open, and a stream remains absolutely pinned to its selected
|
|
152
|
+
member, connection identity, spool namespace and reconnect/resume owner.
|
|
153
|
+
|
|
154
|
+
`createConnection()` always returns an app-facing `VirtualSocket` immediately.
|
|
155
|
+
Calling it before `start()` lazily starts the set; calling it while initial
|
|
156
|
+
startup is pending or while every member is reconnecting leaves the socket in
|
|
157
|
+
its normal `connecting`/`pending` state. Opens wait FIFO until initial
|
|
158
|
+
`minimumReadyMembers` has been reached, and after that any one recovered ready
|
|
159
|
+
member may resume the queue. New calls never wait behind a reconnecting route
|
|
160
|
+
when a healthy sibling is ready.
|
|
161
|
+
|
|
162
|
+
Waiting does not assign an anonymous or merely connecting member. `gatewayId`
|
|
163
|
+
is `undefined` until one ready member accepts the open, then becomes the stable,
|
|
164
|
+
immutable configured ID for that socket. Destroying the socket first removes
|
|
165
|
+
the pending open, so later readiness cannot leak a member stream. A member
|
|
166
|
+
selected immediately before its path drops retains the stream and performs
|
|
167
|
+
member-local v3 recovery; the set never speculatively migrates it.
|
|
168
|
+
|
|
169
|
+
`createConnection(options, callback)` is pre-bound and invokes its callback once
|
|
170
|
+
on `connect` or the terminal pre-connect error; the socket emits `connect` before
|
|
171
|
+
`ready`. A pending open snapshots its top-level virtual options and nested
|
|
172
|
+
`target` when called, so later caller mutations cannot change the eventual
|
|
173
|
+
member open. Pending writes accept every TypedArray and `DataView` supported by
|
|
174
|
+
Node Writable, retain normal backpressure and timeouts, and reserve their exact
|
|
175
|
+
bytes in the shared in-memory/queued budget. Pending opens are also capped by
|
|
176
|
+
`maxPendingOpens` (default 1024), with
|
|
177
|
+
`SOCKETDUCT_SET_PENDING_OPEN_LIMIT` reported synchronously when the count is
|
|
178
|
+
full. Terminal inability to meet initial readiness and terminal loss of every
|
|
179
|
+
member fail waiting sockets normally. Post-readiness loss of every member uses
|
|
180
|
+
`SocketductTransportSetTerminalError` with code
|
|
181
|
+
`SOCKETDUCT_TRANSPORT_SET_TERMINAL`; its `errors` array preserves one terminal
|
|
182
|
+
cause per configured gateway in configured order. Privacy-safe
|
|
183
|
+
`status().members[].terminalErrorCode` exposes only each stable cause code, not
|
|
184
|
+
its message, stack, endpoint, credentials or payload. `close()` fails pending
|
|
185
|
+
sockets and later opens reject synchronously as closed.
|
|
186
|
+
|
|
187
|
+
`createTlsConnection(options, callback)` immediately wraps that raw virtual
|
|
188
|
+
socket with ordinary `tls.connect` target options such as `servername`, `ca`,
|
|
189
|
+
`cert` and `key`. Node TLS owns the handshake, `secureConnect`, TLS errors and
|
|
190
|
+
the callback, which runs exactly once only after `secureConnect` or failure.
|
|
191
|
+
Both methods count only completed member assignments and preserve stream
|
|
192
|
+
stickiness. Every returned raw or TLS socket exposes enumerable `gatewayId`;
|
|
193
|
+
persist its immutable value after connection so a replacement can recover the
|
|
194
|
+
stream on its original member.
|
|
195
|
+
|
|
196
|
+
Process-replacement recovery is deliberately separate from ordinary selection.
|
|
197
|
+
First detach the old set when an orderly handoff is possible. Construct the
|
|
198
|
+
replacement with the same base `connectionId`, spool directory, gateway IDs and
|
|
199
|
+
pinned endpoints. Then register each durable stream with its original
|
|
200
|
+
`gatewayId` before calling `start()`:
|
|
201
|
+
|
|
202
|
+
```js
|
|
203
|
+
await set.detach()
|
|
204
|
+
|
|
205
|
+
const replacement = new MultiplexedSocketductTransportSet(sameSetOptions)
|
|
206
|
+
const recovered = replacement.recoverConnection({
|
|
207
|
+
gatewayId: durableRecord.gatewayId, // captured from the original socket
|
|
208
|
+
host: "api.internal",
|
|
209
|
+
port: 443,
|
|
210
|
+
identifier: "build_stream_01",
|
|
211
|
+
sessionName: "build"
|
|
212
|
+
})
|
|
213
|
+
await replacement.start()
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
`recoverConnection()` implies durable recovery and rejects a missing or unknown
|
|
217
|
+
gateway ID, an explicit `recover` option, or registration after `start()` has
|
|
218
|
+
begun. It bypasses round-robin only to reach that configured member; it never
|
|
219
|
+
migrates or replays the stream on a sibling. Ordinary `createConnection()` and
|
|
220
|
+
`createTlsConnection()` reject `recover: true` and `gatewayId`, so new opens
|
|
221
|
+
cannot force a path. `detach()` coalesces with concurrent shutdown calls,
|
|
222
|
+
preserves active member spools, and retires empty warmed lanes; `close()` remains
|
|
223
|
+
terminal and removes their spools.
|
|
224
|
+
|
|
225
|
+
The caller-owned set `connectionId` is required. Socketduct derives a stable,
|
|
226
|
+
opaque member connection identity and a separate safe spool subdirectory from
|
|
227
|
+
each `{connectionId, gatewayId}` pair; gateway IDs are never interpolated into
|
|
228
|
+
filesystem paths. The three set limits default to 32 MiB retained data, 32 MiB
|
|
229
|
+
spool data and 33 MiB total queued data/control respectively. They are shared
|
|
230
|
+
reservations across all members, in addition to each member's existing limits,
|
|
231
|
+
so adding paths does not multiply the process budget. Invalid types and
|
|
232
|
+
non-positive values fail rather than being coerced. Pending pre-assignment bytes
|
|
233
|
+
use those same memory and queued-byte budgets but never consume spool capacity;
|
|
234
|
+
they transfer into the selected member's normal accounting on assignment.
|
|
235
|
+
Memory is released when its
|
|
236
|
+
buffer is discarded, but shared spool capacity remains reserved until the
|
|
237
|
+
corresponding durable truncate or remove succeeds. Failed cleanup keeps that
|
|
238
|
+
capacity occupied and fails the affected member visibly.
|
|
239
|
+
|
|
240
|
+
`status()` returns only safe aggregate counts/bytes, `pendingOpens`, its maximum,
|
|
241
|
+
and an ordered member list:
|
|
242
|
+
gateway ID, lifecycle state, assignments, physical connects/reconnects, virtual
|
|
243
|
+
stream count and queued/spool bytes. It excludes relays, targets, tokens,
|
|
244
|
+
credentials, names, payloads and spool contents. Concurrent `close()` calls
|
|
245
|
+
share one promise. Every member is attempted once in configured order; multiple
|
|
246
|
+
failures are returned as an `AggregateError` with its `errors` in that order.
|
|
247
|
+
Per-member `spoolCleanupFailures` records durable cleanup failures without
|
|
248
|
+
exposing spool identity or contents.
|
|
249
|
+
|
|
250
|
+
The `socketduct/http` and `socketduct/https` adapters below remain supported for
|
|
251
|
+
legacy v2 compatibility. They create one v2 WAN tunnel per pooled HTTP socket
|
|
252
|
+
and are not the primary v3 API.
|
|
253
|
+
|
|
254
|
+
### `connectSocketduct(options)`
|
|
255
|
+
|
|
256
|
+
```js
|
|
257
|
+
import {connectSocketduct} from "socketduct"
|
|
258
|
+
|
|
259
|
+
const socket = connectSocketduct({
|
|
260
|
+
relay: {
|
|
261
|
+
host: "gateway.example.net",
|
|
262
|
+
port: 3100,
|
|
263
|
+
token: process.env.SOCKETDUCT_TOKEN
|
|
264
|
+
},
|
|
265
|
+
target: {
|
|
266
|
+
host: "127.0.0.1",
|
|
267
|
+
port: 2375
|
|
268
|
+
},
|
|
269
|
+
sessionName: "docker-api",
|
|
270
|
+
spoolDirectory: "/tmp/socketduct-docker",
|
|
271
|
+
maxBytes: 5_000_000,
|
|
272
|
+
maxAgeMs: 300_000,
|
|
273
|
+
reconnect: {
|
|
274
|
+
initialDelayMs: 250,
|
|
275
|
+
maxDelayMs: 5_000,
|
|
276
|
+
maxInterruptedMs: 60_000
|
|
277
|
+
}
|
|
278
|
+
})
|
|
279
|
+
|
|
280
|
+
socket.write(Buffer.from("GET /_ping HTTP/1.1\r\nHost: docker\r\n\r\n"))
|
|
281
|
+
socket.on("data", (chunk) => process.stdout.write(chunk))
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
`connectSocketduct()` returns immediately like `net.connect()`. It emits `connect` only after the relay has connected the requested target, transitioned the session to active, and sent `session.created` (or after a successful resume). Writes made earlier remain queued locally and do not reach the target before readiness.
|
|
285
|
+
|
|
286
|
+
### `new SocketductSocket(options)`
|
|
287
|
+
|
|
288
|
+
`SocketductSocket` should extend `Duplex` and expose the common socket fields/methods that libraries usually probe:
|
|
289
|
+
|
|
290
|
+
- events: `connect`, `ready`, `data`, `end`, `close`, `error`, `timeout`, `drain`
|
|
291
|
+
- methods: `connect`, `write`, `end`, `destroy`, `pause`, `resume`, `setTimeout`, `setNoDelay`, `setKeepAlive`, `address`, `ref`, `unref`
|
|
292
|
+
- properties: `connecting`, `destroyed`, `pending`, `remoteAddress`, `remotePort`, `localAddress`, `localPort`, `bytesRead`, `bytesWritten`
|
|
293
|
+
|
|
294
|
+
The constructor should configure the socket without opening the relay connection. The public `.connect()` method starts relay connection/session create or resume and returns `this`, matching Node's chaining style. `connectSocketduct(options)` is the convenience wrapper that constructs `new SocketductSocket(options)`, schedules `.connect()`, and returns the socket immediately.
|
|
295
|
+
|
|
296
|
+
No-op compatibility methods are acceptable only when the behavior is harmless and documented. For example, `setNoDelay()` can return `this` because Socketduct frames data over its own tunnel and does not expose the target TCP socket directly to the caller.
|
|
297
|
+
|
|
298
|
+
### `createSocketductConnectionFactory(options)`
|
|
299
|
+
|
|
300
|
+
```js
|
|
301
|
+
const createConnection = createSocketductConnectionFactory({
|
|
302
|
+
relay: {host, port, token},
|
|
303
|
+
spoolDirectory: "/tmp/socketduct-docker"
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
const socket = createConnection({
|
|
307
|
+
host: "127.0.0.1",
|
|
308
|
+
port: 2375,
|
|
309
|
+
sessionName: "docker-api"
|
|
310
|
+
})
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
The factory lets libraries accept a `createConnection` hook without needing to know about all Socketduct relay settings on every call. The relay token is required in `relay.token`; callers must treat it as a secret and Socketduct client APIs must not log it.
|
|
314
|
+
|
|
315
|
+
### `SocketductHttpAgent`
|
|
316
|
+
|
|
317
|
+
```js
|
|
318
|
+
import {SocketductHttpAgent} from "socketduct/http"
|
|
319
|
+
|
|
320
|
+
const agent = new SocketductHttpAgent({
|
|
321
|
+
relay: {host, port, token},
|
|
322
|
+
target: {host: "127.0.0.1", port: 2375},
|
|
323
|
+
sessionNamePrefix: "docker-http"
|
|
324
|
+
})
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
The agent should override `createConnection(options, callback)` and return a `SocketductSocket`. It should support one session per HTTP connection so keep-alive pooling can reuse sockets normally.
|
|
328
|
+
|
|
329
|
+
The callback is invoked exactly once with either the connected socket or the pre-connect error. Relay protocol failures are `SocketductRelayError` instances with stable `code` and `retryable` fields, so integrations should use those fields rather than matching human-readable messages.
|
|
330
|
+
|
|
331
|
+
### `SocketductHttpsAgent`
|
|
332
|
+
|
|
333
|
+
The HTTPS adapter is a dedicated package subpath and is intentionally not exported from the main barrel:
|
|
334
|
+
|
|
335
|
+
```js
|
|
336
|
+
import {readFileSync} from "node:fs"
|
|
337
|
+
import {SocketductHttpsAgent} from "socketduct/https"
|
|
338
|
+
|
|
339
|
+
const agent = new SocketductHttpsAgent({
|
|
340
|
+
relay: {host: "relay.example.net", port: 3100, token: process.env.SOCKETDUCT_TOKEN},
|
|
341
|
+
relayTls: {
|
|
342
|
+
caFile: "/certs/relay-ca.crt",
|
|
343
|
+
servername: "relay.example.net"
|
|
344
|
+
},
|
|
345
|
+
target: {host: "10.0.0.15", port: 2376},
|
|
346
|
+
targetTls: {
|
|
347
|
+
ca: readFileSync("/certs/docker-ca.crt"),
|
|
348
|
+
cert: readFileSync("/certs/docker-client.crt"),
|
|
349
|
+
key: readFileSync("/certs/docker-client.key"),
|
|
350
|
+
rejectUnauthorized: true,
|
|
351
|
+
servername: "docker.example.internal"
|
|
352
|
+
},
|
|
353
|
+
spoolDirectory: "/tmp/socketduct-docker"
|
|
354
|
+
})
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
`relayTls` protects and authenticates the edge-to-relay connection. `targetTls` is independent and protects the logical Socketduct stream to the target HTTPS server. The agent creates a raw `SocketductSocket`, waits for real target readiness, then wraps it with `node:tls` and invokes Node's agent callback only after `secureConnect`.
|
|
358
|
+
|
|
359
|
+
`createConnection(options, callback)` is the asynchronous hook used by Node's `https.Agent` machinery and requires that callback ownership path. Calling it directly without a callback throws synchronously before allocating a tunnel or target connection; applications should pass the agent to `https.request()` instead.
|
|
360
|
+
|
|
361
|
+
Target `ca`, `cert`, and `key` accept Node TLS certificate material. `cert` and `key` are an atomic pair. `rejectUnauthorized` defaults to `true`. When supplied, `servername` must be non-empty and selects the certificate identity and SNI hostname. Otherwise the identity comes from the request host/hostname, falling back to the agent's fixed target host. For an IP identity, Socketduct omits SNI but still verifies the certificate against that IP.
|
|
362
|
+
|
|
363
|
+
`AbortSignal` cancellation, `ClientRequest.abort()`, connection timeout, TLS failure, and agent destruction close the logical session exactly once and preserve the original Socketduct or TLS error. A temporary edge-to-relay interruption resumes underneath the same target `TLSSocket`. A target TCP/TLS failure remains terminal.
|
|
364
|
+
|
|
365
|
+
## Reverse gateway sockets
|
|
366
|
+
|
|
367
|
+
`ReverseGatewayConnection` from `socketduct/reverse` exposes the same bound
|
|
368
|
+
`createConnection(options, callback?)` shape and returns `VirtualSocket`.
|
|
369
|
+
It selects exactly one registered gateway during `connect()`. The existing
|
|
370
|
+
`MultiplexedSocketductTransportSet` accepts `endpoint.reverseHub`, allowing
|
|
371
|
+
three ready members to retain deterministic health-aware round-robin behavior.
|
|
372
|
+
Reverse established operations deliberately do not support
|
|
373
|
+
`registerRecoveredConnection()` or durable replay; a transport loss is a typed
|
|
374
|
+
terminal result requiring application reconciliation.
|
|
375
|
+
|
|
376
|
+
Reverse virtual sockets accept individual writes larger than their configured
|
|
377
|
+
retained-byte limit. The connection progresses those writes in slices bounded by
|
|
378
|
+
the frame size and negotiated peer window as ACKs return flow-control capacity,
|
|
379
|
+
rotating released connection capacity among streams with pending writes. It
|
|
380
|
+
completes each Node Writable callback only after the full write has entered that
|
|
381
|
+
bounded progression. Per-stream and per-connection limits remain hard ceilings
|
|
382
|
+
for retained replay bytes.
|
|
383
|
+
|
|
384
|
+
See [Native reverse gateway](reverse-gateway.md) for the full constructor and
|
|
385
|
+
role contract.
|
|
386
|
+
|
|
387
|
+
## Spool limits
|
|
388
|
+
|
|
389
|
+
All public APIs (`connectSocketduct`, `createSocketductConnectionFactory`, `SocketductHttpAgent`, and `SocketductHttpsAgent`) accept optional `maxBytes` and `maxAgeMs` to bound per-session spool storage:
|
|
390
|
+
|
|
391
|
+
- `maxBytes` — Maximum total bytes stored on disk for a session's unacknowledged chunks. When a new append would exceed the limit, a `SpoolQuotaError` with code `"SPOOL_QUOTA_EXCEEDED"` is thrown and the chunk is not written. The quota is enforced against durable queued payload bytes tracked via a per-session sidecar file (`usage.json`), so reopening a spool directory with a new `SessionSpool` instance continues to respect the limit.
|
|
392
|
+
- `maxAgeMs` — Maximum age of a spool session since creation. When the session exceeds this age, appends throw a `SpoolQuotaError` with code `"SPOOL_AGE_EXCEEDED"`. Accepts an optional `now` function for deterministic testing.
|
|
393
|
+
|
|
394
|
+
Both options default to unbounded (no limit). These limits apply independently: a session can be rejected by age even when under the byte limit, and vice versa.
|
|
395
|
+
|
|
396
|
+
## State model
|
|
397
|
+
|
|
398
|
+
```text
|
|
399
|
+
idle -> connecting -> connected -> interrupted -> reconnecting -> connected
|
|
400
|
+
| | |
|
|
401
|
+
v v v
|
|
402
|
+
ending expired failed
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
Important behavior:
|
|
406
|
+
|
|
407
|
+
1. Before `connect`, writes are queued by the `Duplex` implementation and by the existing Socketduct spool path.
|
|
408
|
+
2. During `interrupted`, the app-facing socket remains open while limits allow.
|
|
409
|
+
3. If the relay resumes before `maxInterruptedMs`, queued/spooled data is replayed in sequence.
|
|
410
|
+
4. If the remote target closes or the resume window expires, the socket emits `error` and then `close` like a normal broken TCP socket.
|
|
411
|
+
5. Backpressure must be driven by Node stream return values and `drain`, not by unbounded in-memory arrays.
|
|
412
|
+
|
|
413
|
+
Destroying an announced socket during relay interruption records a logical-close intent. Socketduct may reconnect only far enough to authenticate and resume before sending `session.close`, so the relay can promptly remove the target and session instead of retaining them for the resume window. That close-only path never replays application data.
|
|
414
|
+
|
|
415
|
+
## Implementation mapping
|
|
416
|
+
|
|
417
|
+
The current `startTcpEdge()` already contains the key session logic but wraps it behind a local TCP server. The socket API should extract that per-connection behavior into reusable client-side pieces:
|
|
418
|
+
|
|
419
|
+
- `src/client/socketduct-socket.js` — `Duplex`-compatible socket facade.
|
|
420
|
+
- `src/client/connect.js` — public `connectSocketduct()` and factory helpers.
|
|
421
|
+
- `src/client/http-agent.js` — Node `http.Agent` adapter.
|
|
422
|
+
- `src/client/https-agent.js` — Node `https.Agent` adapter with target TLS ownership.
|
|
423
|
+
- `src/edge/tcp-edge.js` — keep as a small local-listener wrapper around `connectSocketduct()`.
|
|
424
|
+
- `src/index.js` or package exports — export the client APIs without making CLI startup run on import.
|
|
425
|
+
|
|
426
|
+
The extraction should keep protocol details private to the socket implementation. Callers should only think in terms of socket writes, reads, errors, and close semantics.
|
|
427
|
+
|
|
428
|
+
## Compatibility checks
|
|
429
|
+
|
|
430
|
+
Each implementation slice should include tests for:
|
|
431
|
+
|
|
432
|
+
- duck typing: object is a `Duplex`, has common socket methods, and supports chaining for no-op methods;
|
|
433
|
+
- connection lifecycle: `connect` emitted once after the initial `session.created`, and `ready` emitted after initial connect and each successful resume;
|
|
434
|
+
- streaming: bidirectional bytes flow through relay to a fake target;
|
|
435
|
+
- backpressure: large writes do not bypass stream pressure signals;
|
|
436
|
+
- interruption: relay tunnel drops, target stays alive, stale control frames are ignored, pending writes are replayed through the resumed relay, and the socket resumes;
|
|
437
|
+
- failure: unauthorized target and expired resume emit normal socket errors;
|
|
438
|
+
- HTTP: `http.request({agent})` can call a fake Docker-style endpoint.
|
|
439
|
+
|
|
440
|
+
## Docker-specific notes
|
|
441
|
+
|
|
442
|
+
Docker clients need ordinary HTTP request/response traffic plus long-lived streaming endpoints. The socket API must not special-case Docker, but the tests should include Docker-shaped flows:
|
|
443
|
+
|
|
444
|
+
- buffered JSON requests such as `GET /version`;
|
|
445
|
+
- chunked or streaming responses such as logs/events;
|
|
446
|
+
- HTTP connection upgrade/hijack-style streams used by Docker attach/exec.
|
|
447
|
+
|
|
448
|
+
These flows drive the need for a real `Duplex` facade and an `http.Agent` adapter instead of only a local port-forwarding process.
|
package/eslint.config.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import js from "@eslint/js"
|
|
4
|
+
import globals from "globals"
|
|
5
|
+
import {defineConfig, globalIgnores} from "eslint/config"
|
|
6
|
+
import {jsdoc} from "eslint-plugin-jsdoc"
|
|
7
|
+
|
|
8
|
+
export default defineConfig([
|
|
9
|
+
globalIgnores(["dist/**", "node_modules/**"]),
|
|
10
|
+
{
|
|
11
|
+
files: ["**/*.{js,mjs,cjs}"],
|
|
12
|
+
plugins: {js},
|
|
13
|
+
extends: ["js/recommended"],
|
|
14
|
+
languageOptions: {
|
|
15
|
+
ecmaVersion: "latest",
|
|
16
|
+
globals: globals.node,
|
|
17
|
+
sourceType: "module"
|
|
18
|
+
},
|
|
19
|
+
rules: {
|
|
20
|
+
"brace-style": ["error", "1tbs", {allowSingleLine: true}],
|
|
21
|
+
"comma-dangle": ["error", "never"],
|
|
22
|
+
"indent": ["error", 2],
|
|
23
|
+
"no-multiple-empty-lines": ["error", {max: 1, maxBOF: 0, maxEOF: 0}],
|
|
24
|
+
"object-curly-spacing": ["error", "never"],
|
|
25
|
+
"quotes": ["error", "double", {avoidEscape: true}],
|
|
26
|
+
"semi": ["error", "never"]
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
jsdoc({
|
|
30
|
+
config: "flat/recommended",
|
|
31
|
+
rules: {
|
|
32
|
+
"jsdoc/reject-any-type": "error",
|
|
33
|
+
"jsdoc/require-param-description": "off"
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
])
|
package/package.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "socketduct",
|
|
3
|
+
"version": "0.1.15",
|
|
4
|
+
"description": "Resilient authenticated tunnel/proxy for keeping socket streams alive across flaky links.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./src/index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./src/index.js",
|
|
9
|
+
"./http": "./src/client/http-agent.js",
|
|
10
|
+
"./https": "./src/client/https-agent.js",
|
|
11
|
+
"./multiplexed": "./src/client/multiplexed.js",
|
|
12
|
+
"./reverse": "./src/reverse/index.js",
|
|
13
|
+
"./relay": "./src/relay.js"
|
|
14
|
+
},
|
|
15
|
+
"engines": {
|
|
16
|
+
"node": ">=24.0.0"
|
|
17
|
+
},
|
|
18
|
+
"scripts": {
|
|
19
|
+
"gateway:deploy": "node scripts/deploy-gateway.js",
|
|
20
|
+
"check": "npm run lint && npm run typecheck && npm test",
|
|
21
|
+
"release:patch": "npm run check && release-patch",
|
|
22
|
+
"start": "node src/index.js",
|
|
23
|
+
"test": "node scripts/test.js",
|
|
24
|
+
"typecheck": "tsc --noEmit",
|
|
25
|
+
"lint": "eslint . --max-warnings 0"
|
|
26
|
+
},
|
|
27
|
+
"keywords": [
|
|
28
|
+
"resilient",
|
|
29
|
+
"tunnel",
|
|
30
|
+
"proxy",
|
|
31
|
+
"sessions"
|
|
32
|
+
],
|
|
33
|
+
"license": "UNLICENSED",
|
|
34
|
+
"devDependencies": {
|
|
35
|
+
"@eslint/js": "^10.0.1",
|
|
36
|
+
"@types/node": "^24.0.0",
|
|
37
|
+
"@velocious/testing": "0.0.10",
|
|
38
|
+
"eslint": "^10.0.3",
|
|
39
|
+
"eslint-plugin-jsdoc": "^62.7.1",
|
|
40
|
+
"globals": "^17.2.0",
|
|
41
|
+
"release-patch": "^1.0.8",
|
|
42
|
+
"typescript": "^5.7.2"
|
|
43
|
+
}
|
|
44
|
+
}
|