socketduct 0.1.19 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +17 -5
- package/README.md +3 -3
- package/changelog.d/20260925170000-disconnect-attribution-compressed-logs.md +1 -0
- package/docs/docker.md +5 -4
- package/docs/logging.md +131 -159
- package/docs/protocol.md +27 -4
- package/docs/reverse-gateway.md +34 -0
- package/docs/running.md +14 -11
- package/docs/security.md +7 -3
- package/package.json +5 -1
- package/scripts/verify-zstd-interoperability.mjs +41 -0
- package/src/cli/socketduct.js +143 -0
- package/src/client/virtual-socket.js +10 -0
- package/src/config/environment.js +56 -7
- package/src/index.js +64 -26
- package/src/logging/compressed-log-reader.js +256 -0
- package/src/logging/compressed-log-worker.js +430 -0
- package/src/logging/compressed-log-writer.js +538 -0
- package/src/logging/log-batch.js +72 -0
- package/src/logging/log-retention.js +241 -0
- package/src/logging/log-segment-index.js +176 -0
- package/src/logging/structured-logger.js +136 -30
- package/src/logging/zstd-frame-codec.js +73 -0
- package/src/reverse/client.js +71 -11
- package/src/reverse/disconnect-attribution.js +191 -0
- package/src/reverse/gateway.js +58 -8
- package/src/reverse/hub.js +215 -32
- package/src/reverse/runtime.js +5 -3
- package/tensorbuzz.yml +2 -1
- package/test/e2e/logger-shutdown-process.test.js +86 -0
- package/test/e2e/reverse-gateway-process.test.js +15 -14
- package/test/integration/logging-cli.test.js +168 -0
- package/test/integration/logging-compressed-recovery.test.js +186 -0
- package/test/integration/reverse-gateway.test.js +176 -0
- package/test/integration/reverse-route-class.test.js +6 -1
- package/test/unit/config/docker-gateway-files.test.js +2 -0
- package/test/unit/config/environment.test.js +54 -14
- package/test/unit/index.test.js +3 -0
- package/test/unit/logging/compressed-log-worker.test.js +29 -0
- package/test/unit/logging/compressed-log-writer.test.js +183 -0
- package/test/unit/logging/log-retention.test.js +86 -0
- package/test/unit/logging/log-segment-index.test.js +104 -0
- package/test/unit/logging/structured-logger.test.js +14 -1
- package/test/unit/logging/zstd-frame-codec.test.js +24 -0
- package/test/unit/reverse/disconnect-attribution.test.js +82 -0
|
@@ -14,6 +14,13 @@ const MAX_ERROR_STACK_BYTES = 512
|
|
|
14
14
|
const ERROR_DETAILS = Symbol("errorDetails")
|
|
15
15
|
const ROLE = /^[a-z][a-z0-9-]{0,63}$/
|
|
16
16
|
const GATEWAY_ID = /^[A-Za-z0-9_-]+$/
|
|
17
|
+
const ROUTE_CLASS = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/
|
|
18
|
+
const DISCONNECT_INCIDENT_ID = /^[A-Za-z0-9_-]{22}$/
|
|
19
|
+
const CONNECTION_FINGERPRINT = /^[a-f0-9]{32}$/
|
|
20
|
+
const FINGERPRINT_KEY_EPOCH = /^[A-Za-z0-9_-]{1,32}$/
|
|
21
|
+
const DISCONNECT_REASONS = new Set(["remote_socket_error", "remote_eof", "peer_goaway", "protocol_violation", "output_backpressure_limit", "heartbeat_timeout", "gateway_replaced", "gateway_shutdown", "hub_shutdown", "local_close", "send_after_transport_loss", "unknown_transport_close"])
|
|
22
|
+
const INCIDENT_ORIGINS = new Set(["reverse_hub", "reverse_client", "reverse_gateway"])
|
|
23
|
+
const PHYSICAL_ROLES = new Set(["client", "gateway"])
|
|
17
24
|
const RUNTIME_MODES = new Set(["summary", "relay", "reverse-hub", "reverse-gateway"])
|
|
18
25
|
const TRANSPORT_SECURITY = new Set(["plaintext", "TLS"])
|
|
19
26
|
const COMPRESSION = new Set(["auto", "off", "zstd", "none"])
|
|
@@ -85,22 +92,35 @@ const NUMBER_FIELDS = new Set([
|
|
|
85
92
|
"bufferLimitBytes",
|
|
86
93
|
"clientPhysicalConnections",
|
|
87
94
|
"closedMembers",
|
|
95
|
+
"compressionLevel",
|
|
88
96
|
"compressionBytesSaved",
|
|
97
|
+
"coalescedDrops",
|
|
89
98
|
"configuredMembers",
|
|
90
99
|
"connectingMembers",
|
|
91
100
|
"connectionGeneration",
|
|
101
|
+
"clientConnectionGeneration",
|
|
92
102
|
"connectionIdentities",
|
|
93
103
|
"controlQueuedBytes",
|
|
104
|
+
"criticalDrops",
|
|
105
|
+
"criticalFlushIntervalMs",
|
|
106
|
+
"criticalReserveBytes",
|
|
94
107
|
"dataQueuedBytes",
|
|
95
108
|
"degradedMembers",
|
|
96
109
|
"delayMs",
|
|
97
110
|
"detachedSessions",
|
|
98
111
|
"gatewayPhysicalConnections",
|
|
112
|
+
"gatewayConnectionGeneration",
|
|
99
113
|
"gatewayRegistrations",
|
|
100
114
|
"generation",
|
|
115
|
+
"fanOutCount",
|
|
116
|
+
"flushIntervalMs",
|
|
117
|
+
"frameMaxBytes",
|
|
101
118
|
"healthyGateways",
|
|
102
119
|
"maxPendingOpens",
|
|
120
|
+
"maxQueueBytes",
|
|
121
|
+
"maxSegmentBytes",
|
|
103
122
|
"minimumReadyMembers",
|
|
123
|
+
"noncriticalDrops",
|
|
104
124
|
"openFailures",
|
|
105
125
|
"pendingOpens",
|
|
106
126
|
"physicalAccepts",
|
|
@@ -115,7 +135,11 @@ const NUMBER_FIELDS = new Set([
|
|
|
115
135
|
"rejectedOpens",
|
|
116
136
|
"rejectedRegistrations",
|
|
117
137
|
"retainedClientIdentities",
|
|
138
|
+
"retentionBytes",
|
|
139
|
+
"retentionDays",
|
|
118
140
|
"resets",
|
|
141
|
+
"reservedFieldsStripped",
|
|
142
|
+
"resumeSuccessCount",
|
|
119
143
|
"resumeFailures",
|
|
120
144
|
"resumeSuccesses",
|
|
121
145
|
"sessions",
|
|
@@ -127,17 +151,19 @@ const NUMBER_FIELDS = new Set([
|
|
|
127
151
|
"targetOpenFailures",
|
|
128
152
|
"targetOpens",
|
|
129
153
|
"targetRetryTimers",
|
|
154
|
+
"terminalStreamCount",
|
|
130
155
|
"virtualOpens",
|
|
131
156
|
"virtualStreams",
|
|
132
157
|
"wireDataBytes"
|
|
133
158
|
])
|
|
134
159
|
const RUNTIME_STATUS_FIELDS = new Set([...NUMBER_FIELDS, "compression", "physicalState"])
|
|
135
160
|
const EVENT_FIELDS = new Map([
|
|
136
|
-
["process.started", new Set(["runtimeMode", "transportSecurity", "compression"])],
|
|
161
|
+
["process.started", new Set(["runtimeMode", "transportSecurity", "compression", "compressionLevel", "frameMaxBytes", "flushIntervalMs", "criticalFlushIntervalMs", "maxSegmentBytes", "retentionDays", "retentionBytes", "maxQueueBytes", "criticalReserveBytes"])],
|
|
162
|
+
["process.stopped", new Set(["runtimeMode"])],
|
|
137
163
|
["runtime.status", RUNTIME_STATUS_FIELDS],
|
|
138
164
|
["transport.connecting", new Set(["generation", "attempt", "gatewayId", "reconnect"])],
|
|
139
165
|
["transport.connected", new Set(["generation", "attempt", "gatewayId", "physicalConnects", "reconnect"])],
|
|
140
|
-
["transport.disconnected", new Set(["
|
|
166
|
+
["transport.disconnected", new Set(["gatewayId", "routeClass", "virtualStreams", "errorCode", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "safeSocketCode"])],
|
|
141
167
|
["transport.reconnect_scheduled", new Set(["generation", "attempt", "delayMs", "gatewayId", "errorCode"])],
|
|
142
168
|
["transport.reconnect_succeeded", new Set(["generation", "attempt", "gatewayId", "physicalConnects"])],
|
|
143
169
|
["transport.failed", new Set(["generation", "attempt", "gatewayId", "errorCode"])],
|
|
@@ -149,10 +175,28 @@ const EVENT_FIELDS = new Map([
|
|
|
149
175
|
["virtual_stream.opening", new Set(["streamId", "generation", "gatewayId", "recovered"])],
|
|
150
176
|
["virtual_stream.opened", new Set(["streamId", "generation", "gatewayId", "recovered"])],
|
|
151
177
|
["virtual_stream.closed", new Set(["streamId", "generation", "gatewayId"])],
|
|
152
|
-
["virtual_stream.error", new Set(["streamId", "generation", "gatewayId", "errorCode"])],
|
|
178
|
+
["virtual_stream.error", new Set(["streamId", "generation", "gatewayId", "routeClass", "errorCode", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount"])],
|
|
153
179
|
["reverse_gateway.target_opened", new Set(["streamId", "generation", "gatewayId"])],
|
|
154
180
|
["reverse_gateway.target_open_failed", new Set(["streamId", "generation", "gatewayId", "errorCode"])],
|
|
155
|
-
["reverse_gateway.target_closed", new Set(["streamId", "generation", "gatewayId", "errorCode"])]
|
|
181
|
+
["reverse_gateway.target_closed", new Set(["streamId", "generation", "gatewayId", "errorCode"])],
|
|
182
|
+
["reverse_hub.physical_detached", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "connectionFingerprint", "fingerprintKeyEpoch", "safeSocketCode"])],
|
|
183
|
+
["reverse_hub.streams_interrupted", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "gatewayId", "routeClass"])],
|
|
184
|
+
["reverse_hub.streams_resumed", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "gatewayId", "routeClass"])],
|
|
185
|
+
["reverse_hub.streams_reset", new Set(["disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "gatewayId", "routeClass", "errorCode"])],
|
|
186
|
+
["reverse_hub.reserved_control_field_stripped", new Set(["physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "reservedFieldsStripped"])],
|
|
187
|
+
["reverse_client.physical_detached", new Set(["gatewayId", "routeClass", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "safeSocketCode"])],
|
|
188
|
+
["reverse_client.send_after_transport_loss", new Set(["gatewayId", "routeClass", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount"])],
|
|
189
|
+
["reverse_gateway.physical_detached", new Set(["gatewayId", "disconnectIncidentId", "disconnectReason", "incidentOrigin", "physicalRole", "clientConnectionGeneration", "gatewayConnectionGeneration", "fanOutCount", "resumeSuccessCount", "terminalStreamCount", "safeSocketCode"])],
|
|
190
|
+
["log_batch_accepted", new Set(["queuedBytes"])],
|
|
191
|
+
["log_frame_committed", new Set(["queuedBytes"])],
|
|
192
|
+
["log_orphan_recovered", new Set([])],
|
|
193
|
+
["log_worker_restarted", new Set(["attempt"])],
|
|
194
|
+
["log_queue_pressure", new Set(["queuedBytes", "criticalDrops", "noncriticalDrops", "coalescedDrops"])],
|
|
195
|
+
["log_records_dropped", new Set(["criticalDrops", "noncriticalDrops", "coalescedDrops"])],
|
|
196
|
+
["log_flush_failed", new Set(["errorCode"])],
|
|
197
|
+
["log_sink_failed", new Set(["errorCode"])],
|
|
198
|
+
["log_sink_recovered", new Set(["attempt"])],
|
|
199
|
+
["log_closed", new Set(["criticalDrops", "noncriticalDrops", "coalescedDrops"])]
|
|
156
200
|
])
|
|
157
201
|
const FAILED_LOGGERS = new WeakSet()
|
|
158
202
|
|
|
@@ -185,6 +229,60 @@ export function createStructuredLogger(options) {
|
|
|
185
229
|
return createStructuredLoggerWithFailureCode(options, "LOGGER_WRITER_ERROR")
|
|
186
230
|
}
|
|
187
231
|
|
|
232
|
+
/**
|
|
233
|
+
* Produces one bounded, allowlisted JSONL record for a trusted logger envelope.
|
|
234
|
+
* Unknown events return undefined. Payload-like caller fields never survive the
|
|
235
|
+
* event allowlist.
|
|
236
|
+
* @param {{role: string, processId?: number, now?: () => Date, event: string, fields?: Record<string, unknown>, envelope?: {writerEpoch: string, eventSequence: string}}} options - Record inputs.
|
|
237
|
+
* @returns {string | undefined} Complete newline-terminated JSONL record.
|
|
238
|
+
*/
|
|
239
|
+
export function formatDiagnosticLogLine(options) {
|
|
240
|
+
validateRole(options?.role)
|
|
241
|
+
const processId = validateProcessId(options.processId ?? process.pid)
|
|
242
|
+
const now = options.now ?? (() => new Date())
|
|
243
|
+
if (typeof now !== "function") throw new TypeError("now must be a function")
|
|
244
|
+
const safeFields = sanitizeFields(options.event, options.fields ?? {})
|
|
245
|
+
if (safeFields === undefined) return undefined
|
|
246
|
+
const timestamp = now()
|
|
247
|
+
if (!(timestamp instanceof Date) || !Number.isFinite(timestamp.getTime())) throw new TypeError("Logger clock returned an invalid date")
|
|
248
|
+
const envelope = options.envelope === undefined ? {} : validateCompressedEnvelope(options.envelope)
|
|
249
|
+
const base = {timestamp: timestamp.toISOString(), event: options.event, processId, role: options.role, ...envelope}
|
|
250
|
+
let line = formatLogLine(base, safeFields)
|
|
251
|
+
if (Buffer.byteLength(line) > MAX_JSONL_BYTES && safeFields.errorCode !== undefined) {
|
|
252
|
+
/** @type {Record<string, string | number | boolean>} */
|
|
253
|
+
const boundedFields = {...safeFields}
|
|
254
|
+
delete boundedFields.errorStack
|
|
255
|
+
if (typeof boundedFields.errorMessage === "string") {
|
|
256
|
+
const emptyMessageLine = formatLogLine(base, {...boundedFields, errorMessage: ""})
|
|
257
|
+
const availableBytes = Math.max(0, MAX_JSONL_BYTES - Buffer.byteLength(emptyMessageLine))
|
|
258
|
+
boundedFields.errorMessage = truncateDiagnosticText(boundedFields.errorMessage, availableBytes)
|
|
259
|
+
}
|
|
260
|
+
line = formatLogLine(base, boundedFields)
|
|
261
|
+
if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
|
|
262
|
+
delete boundedFields.errorMessage
|
|
263
|
+
line = formatLogLine(base, boundedFields)
|
|
264
|
+
}
|
|
265
|
+
if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
|
|
266
|
+
delete boundedFields.errorName
|
|
267
|
+
line = formatLogLine(base, boundedFields)
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (Buffer.byteLength(line) > MAX_JSONL_BYTES) line = formatLogLine(base, {})
|
|
271
|
+
if (Buffer.byteLength(line) > MAX_JSONL_BYTES) throw new Error("Trusted logger envelope exceeds the JSONL record bound")
|
|
272
|
+
return line
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* @param {Record<string, unknown>} envelope - Producer-owned durable identity.
|
|
277
|
+
* @returns {{writerEpoch: string, eventSequence: string}} Validated fields.
|
|
278
|
+
*/
|
|
279
|
+
function validateCompressedEnvelope(envelope) {
|
|
280
|
+
if (Object.keys(envelope).length !== 2 || !Object.hasOwn(envelope, "writerEpoch") || !Object.hasOwn(envelope, "eventSequence")) throw new TypeError("compressed logger envelope must contain only writerEpoch and eventSequence")
|
|
281
|
+
if (typeof envelope.writerEpoch !== "string" || !/^[A-Za-z0-9_-]{22}$/.test(envelope.writerEpoch) || Buffer.from(envelope.writerEpoch, "base64url").toString("base64url") !== envelope.writerEpoch) throw new TypeError("compressed logger writerEpoch must be canonical 128-bit base64url")
|
|
282
|
+
if (typeof envelope.eventSequence !== "string" || !/^[1-9][0-9]*$/.test(envelope.eventSequence) || BigInt(envelope.eventSequence) > 0xffff_ffff_ffff_ffffn) throw new TypeError("compressed logger eventSequence must be an unsigned 64-bit decimal string")
|
|
283
|
+
return {writerEpoch: envelope.writerEpoch, eventSequence: envelope.eventSequence}
|
|
284
|
+
}
|
|
285
|
+
|
|
188
286
|
/**
|
|
189
287
|
* @param {StructuredLoggerOptions} options - Logger identity and writer.
|
|
190
288
|
* @param {'LOGGER_IO_ERROR' | 'LOGGER_WRITER_ERROR'} failureCode - Fixed failure classification.
|
|
@@ -209,32 +307,8 @@ function createStructuredLoggerWithFailureCode(options, failureCode) {
|
|
|
209
307
|
return {
|
|
210
308
|
log(event, fields = {}) {
|
|
211
309
|
try {
|
|
212
|
-
const
|
|
213
|
-
if (
|
|
214
|
-
const timestamp = now()
|
|
215
|
-
if (!(timestamp instanceof Date) || !Number.isFinite(timestamp.getTime())) throw new TypeError("Logger clock returned an invalid date")
|
|
216
|
-
const base = {timestamp: timestamp.toISOString(), event, processId, role}
|
|
217
|
-
let line = formatLogLine(base, safeFields)
|
|
218
|
-
if (Buffer.byteLength(line) > MAX_JSONL_BYTES && safeFields.errorCode !== undefined) {
|
|
219
|
-
/** @type {Record<string, string | number | boolean>} */
|
|
220
|
-
const boundedFields = {...safeFields}
|
|
221
|
-
delete boundedFields.errorStack
|
|
222
|
-
if (typeof boundedFields.errorMessage === "string") {
|
|
223
|
-
const emptyMessageLine = formatLogLine(base, {...boundedFields, errorMessage: ""})
|
|
224
|
-
const availableBytes = Math.max(0, MAX_JSONL_BYTES - Buffer.byteLength(emptyMessageLine))
|
|
225
|
-
boundedFields.errorMessage = truncateDiagnosticText(boundedFields.errorMessage, availableBytes)
|
|
226
|
-
}
|
|
227
|
-
line = formatLogLine(base, boundedFields)
|
|
228
|
-
if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
|
|
229
|
-
delete boundedFields.errorMessage
|
|
230
|
-
line = formatLogLine(base, boundedFields)
|
|
231
|
-
}
|
|
232
|
-
if (Buffer.byteLength(line) > MAX_JSONL_BYTES) {
|
|
233
|
-
delete boundedFields.errorName
|
|
234
|
-
line = formatLogLine(base, boundedFields)
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
if (Buffer.byteLength(line) > MAX_JSONL_BYTES) line = formatLogLine(base, {})
|
|
310
|
+
const line = formatDiagnosticLogLine({role, processId, now, event, fields})
|
|
311
|
+
if (line === undefined) return
|
|
238
312
|
observeRejection(writeLine(line), reportOnce)
|
|
239
313
|
} catch {
|
|
240
314
|
reportOnce()
|
|
@@ -541,6 +615,38 @@ function sanitizeFields(event, fields) {
|
|
|
541
615
|
}
|
|
542
616
|
continue
|
|
543
617
|
}
|
|
618
|
+
if (field === "routeClass") {
|
|
619
|
+
if (typeof value === "string" && ROUTE_CLASS.test(value)) safe.routeClass = value
|
|
620
|
+
continue
|
|
621
|
+
}
|
|
622
|
+
if (field === "disconnectIncidentId") {
|
|
623
|
+
if (typeof value === "string" && DISCONNECT_INCIDENT_ID.test(value)) safe.disconnectIncidentId = value
|
|
624
|
+
continue
|
|
625
|
+
}
|
|
626
|
+
if (field === "disconnectReason") {
|
|
627
|
+
if (typeof value === "string" && DISCONNECT_REASONS.has(value)) safe.disconnectReason = value
|
|
628
|
+
continue
|
|
629
|
+
}
|
|
630
|
+
if (field === "incidentOrigin") {
|
|
631
|
+
if (typeof value === "string" && INCIDENT_ORIGINS.has(value)) safe.incidentOrigin = value
|
|
632
|
+
continue
|
|
633
|
+
}
|
|
634
|
+
if (field === "physicalRole") {
|
|
635
|
+
if (typeof value === "string" && PHYSICAL_ROLES.has(value)) safe.physicalRole = value
|
|
636
|
+
continue
|
|
637
|
+
}
|
|
638
|
+
if (field === "connectionFingerprint") {
|
|
639
|
+
if (typeof value === "string" && CONNECTION_FINGERPRINT.test(value)) safe.connectionFingerprint = value
|
|
640
|
+
continue
|
|
641
|
+
}
|
|
642
|
+
if (field === "fingerprintKeyEpoch") {
|
|
643
|
+
if (typeof value === "string" && FINGERPRINT_KEY_EPOCH.test(value)) safe.fingerprintKeyEpoch = value
|
|
644
|
+
continue
|
|
645
|
+
}
|
|
646
|
+
if (field === "safeSocketCode") {
|
|
647
|
+
if (typeof value === "string" && /^(?:E[A-Z0-9_]{1,31}|ERR_(?:TLS|SSL)_[A-Z0-9_]{1,48})$/.test(value)) safe.safeSocketCode = value
|
|
648
|
+
continue
|
|
649
|
+
}
|
|
544
650
|
if (field === "runtimeMode" && typeof value === "string" && RUNTIME_MODES.has(value)) safe.runtimeMode = value
|
|
545
651
|
else if (field === "transportSecurity" && typeof value === "string" && TRANSPORT_SECURITY.has(value)) safe.transportSecurity = value
|
|
546
652
|
else if (field === "compression" && typeof value === "string" && COMPRESSION.has(value)) safe.compression = value
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/* eslint-disable jsdoc/require-param-type, jsdoc/require-returns */
|
|
2
|
+
import {promisify} from "node:util"
|
|
3
|
+
import {constants, zstdCompress, zstdDecompress} from "node:zlib"
|
|
4
|
+
|
|
5
|
+
export const DEFAULT_LOG_COMPRESSION_LEVEL = 19
|
|
6
|
+
|
|
7
|
+
const compress = promisify(zstdCompress)
|
|
8
|
+
const decompress = promisify(zstdDecompress)
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Compresses one complete JSONL batch as one standard checksummed zstd frame.
|
|
12
|
+
* @param input
|
|
13
|
+
* @param level
|
|
14
|
+
*/
|
|
15
|
+
export async function compressLogFrame(/** @type {Buffer | Uint8Array} */ input, /** @type {number} */ level = DEFAULT_LOG_COMPRESSION_LEVEL) {
|
|
16
|
+
if (!(input instanceof Uint8Array) || input.byteLength === 0) throw new TypeError("log frame input must be non-empty bytes")
|
|
17
|
+
if (!Number.isInteger(level) || level < -131_072 || level > 22) throw new RangeError("log compression level is outside the zstd range")
|
|
18
|
+
const frame = await compress(input, {
|
|
19
|
+
params: {
|
|
20
|
+
[constants.ZSTD_c_compressionLevel]: level,
|
|
21
|
+
[constants.ZSTD_c_checksumFlag]: 1,
|
|
22
|
+
[constants.ZSTD_c_contentSizeFlag]: 1
|
|
23
|
+
}
|
|
24
|
+
})
|
|
25
|
+
const encoded = Buffer.from(frame)
|
|
26
|
+
inspectZstdFrame(encoded)
|
|
27
|
+
return encoded
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Decodes one independently indexed zstd frame.
|
|
32
|
+
* @param frame
|
|
33
|
+
*/
|
|
34
|
+
export async function decompressLogFrame(/** @type {Buffer | Uint8Array} */ frame) {
|
|
35
|
+
if (!(frame instanceof Uint8Array) || frame.byteLength === 0) throw new TypeError("zstd frame must be non-empty bytes")
|
|
36
|
+
inspectZstdFrame(frame)
|
|
37
|
+
return Buffer.from(await decompress(frame))
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Requires exactly one complete standard zstd frame with a checksum trailer.
|
|
42
|
+
* @param value
|
|
43
|
+
*/
|
|
44
|
+
export function inspectZstdFrame(/** @type {Buffer | Uint8Array} */ value) {
|
|
45
|
+
const frame = Buffer.from(value)
|
|
46
|
+
if (frame.length < 9 || frame.readUInt32LE(0) !== 0xfd2f_b528) throw new Error("Invalid zstd frame magic")
|
|
47
|
+
const descriptor = /** @type {number} */ (frame[4])
|
|
48
|
+
const contentSizeFlag = descriptor >>> 6
|
|
49
|
+
const singleSegment = (descriptor & 0x20) !== 0
|
|
50
|
+
const checksum = (descriptor & 0x04) !== 0
|
|
51
|
+
const dictionaryFlag = descriptor & 0x03
|
|
52
|
+
if ((descriptor & 0x08) !== 0) throw new Error("Reserved zstd frame descriptor bit is set")
|
|
53
|
+
if (!checksum) throw new Error("Zstd log frame checksum flag is required")
|
|
54
|
+
const dictionaryBytes = /** @type {number} */ ([0, 1, 2, 4][dictionaryFlag])
|
|
55
|
+
const contentSizeBytes = contentSizeFlag === 0 ? singleSegment ? 1 : 0 : contentSizeFlag === 1 ? 2 : contentSizeFlag === 2 ? 4 : 8
|
|
56
|
+
let offset = 5 + (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes
|
|
57
|
+
if (offset > frame.length) throw new Error("Truncated zstd frame header")
|
|
58
|
+
let lastBlock = false
|
|
59
|
+
while (!lastBlock) {
|
|
60
|
+
if (offset + 3 > frame.length) throw new Error("Truncated zstd block header")
|
|
61
|
+
const header = /** @type {number} */ (frame[offset]) | (/** @type {number} */ (frame[offset + 1]) << 8) | (/** @type {number} */ (frame[offset + 2]) << 16)
|
|
62
|
+
offset += 3
|
|
63
|
+
lastBlock = (header & 1) !== 0
|
|
64
|
+
const blockType = (header >>> 1) & 0x03
|
|
65
|
+
const blockSize = header >>> 3
|
|
66
|
+
if (blockType === 3) throw new Error("Reserved zstd block type")
|
|
67
|
+
const encodedBytes = blockType === 1 ? 1 : blockSize
|
|
68
|
+
if (offset + encodedBytes > frame.length) throw new Error("Truncated zstd block")
|
|
69
|
+
offset += encodedBytes
|
|
70
|
+
}
|
|
71
|
+
if (offset + 4 !== frame.length) throw new Error(offset + 4 > frame.length ? "Truncated zstd checksum" : "Zstd frame contains trailing bytes")
|
|
72
|
+
return {bytes: frame.length, checksum: true}
|
|
73
|
+
}
|
package/src/reverse/client.js
CHANGED
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
STABLE_IDENTIFIER,
|
|
28
28
|
writeSocket
|
|
29
29
|
} from "./common.js"
|
|
30
|
+
import {attachDisconnectAttribution, createDisconnectAttribution, parseDisconnectAttribution, safeSocketCode} from "./disconnect-attribution.js"
|
|
30
31
|
|
|
31
32
|
const DEFAULT_STREAM_BUFFER_BYTES = 4 * 1024 * 1024
|
|
32
33
|
const DEFAULT_CONNECTION_BUFFER_BYTES = 32 * 1024 * 1024
|
|
@@ -159,6 +160,7 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
159
160
|
/** @type {number} */ #spoolBytes = 0
|
|
160
161
|
/** @type {number} */ #spoolCleanupFailures = 0
|
|
161
162
|
/** @type {Set<Promise<void>>} */ #spoolCleanupTasks = new Set()
|
|
163
|
+
/** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */ #disconnectAttribution
|
|
162
164
|
|
|
163
165
|
/** @param {ReverseGatewayConnectionOptions} options - Hub route and safety policy. */
|
|
164
166
|
constructor(options) {
|
|
@@ -411,7 +413,8 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
411
413
|
streamId,
|
|
412
414
|
generation: this.#generation,
|
|
413
415
|
...this.#routeIdentityFields(),
|
|
414
|
-
...(error === null ? {} : errorDiagnostics(error))
|
|
416
|
+
...(error === null ? {} : errorDiagnostics(error)),
|
|
417
|
+
...(error === null ? {} : publicDisconnectAttribution(error))
|
|
415
418
|
})
|
|
416
419
|
this.#retireStream(stream, error ?? new SocketductStreamError("Reverse stream is not writable", "ERR_STREAM_DESTROYED"))
|
|
417
420
|
if (!wasClosed && !this.#closing && this.#state === "ready") void this.#sendControl(V3FrameType.RESET, streamId, 0, {code: error instanceof SocketductStreamError ? error.code : "CLIENT_CANCELLED"}).catch(() => {})
|
|
@@ -483,9 +486,18 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
483
486
|
this.#physicalFrameQueue = queue
|
|
484
487
|
let authenticated = false
|
|
485
488
|
let selected = false
|
|
489
|
+
/** @type {import('./disconnect-attribution.js').DisconnectReason | undefined} */
|
|
490
|
+
let attemptDisconnectReason
|
|
491
|
+
/** @type {string | undefined} */
|
|
492
|
+
let attemptSafeSocketCode
|
|
493
|
+
/** @type {import('./disconnect-attribution.js').DisconnectAttribution | undefined} */
|
|
494
|
+
let attemptAttribution
|
|
486
495
|
/** @type {Error | undefined} */
|
|
487
496
|
let attemptError
|
|
488
|
-
const timer = setTimeout(() =>
|
|
497
|
+
const timer = setTimeout(() => {
|
|
498
|
+
attemptDisconnectReason ??= "heartbeat_timeout"
|
|
499
|
+
physical.destroy(Object.assign(new Error("Reverse client TLS/auth timeout"), {code: "SOCKETDUCT_AUTH_TIMEOUT"}))
|
|
500
|
+
}, this.#options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS)
|
|
489
501
|
timer.unref?.()
|
|
490
502
|
physical.once("secureConnect", () => {
|
|
491
503
|
// The attempted generation must survive a crash after AUTH reaches the
|
|
@@ -505,13 +517,16 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
505
517
|
if (this.#closing || this.#physical !== physical) return
|
|
506
518
|
if (frame.type === V3FrameType.AUTH_ERROR || frame.type === V3FrameType.GOAWAY) {
|
|
507
519
|
const detail = decodeV3Control(frame.payload)
|
|
508
|
-
|
|
520
|
+
if (frame.type === V3FrameType.GOAWAY) attemptDisconnectReason ??= "peer_goaway"
|
|
521
|
+
attemptAttribution = validHubAttribution(detail) ?? attemptAttribution
|
|
522
|
+
throw connectionError(typeof detail.code === "string" ? detail.code : "CONNECTION_REJECTED", attemptAttribution)
|
|
509
523
|
}
|
|
510
524
|
if (!authenticated) {
|
|
511
525
|
if (frame.type !== V3FrameType.AUTH_OK) throw fatalError("AUTH_NEGOTIATION_FAILED")
|
|
512
526
|
const detail = decodeV3Control(frame.payload)
|
|
513
527
|
if (detail.version !== 3 || detail.role !== "client" || detail.connectionId !== this.#connectionId || detail.generation !== generation || detail.compression !== ZSTD_CODEC || !validPeerWindow(detail.initialWindow)) throw fatalError("AUTH_NEGOTIATION_FAILED")
|
|
514
528
|
authenticated = true
|
|
529
|
+
this.#disconnectAttribution = undefined
|
|
515
530
|
this.#codec = ZSTD_CODEC
|
|
516
531
|
this.#sendWindowLimit = Math.min(this.#initialWindow(), detail.initialWindow)
|
|
517
532
|
for (const stream of this.#streams.values()) {
|
|
@@ -544,12 +559,19 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
544
559
|
}
|
|
545
560
|
}).catch((error) => {
|
|
546
561
|
attemptError = asError(error)
|
|
562
|
+
attemptDisconnectReason ??= "protocol_violation"
|
|
547
563
|
if (/** @type {{fatal?: unknown}} */ (attemptError).fatal === true) this.#fatal = true
|
|
548
564
|
physical.destroy(attemptError)
|
|
549
565
|
}).finally(() => { if (!physical.destroyed) physical.resume() })
|
|
550
566
|
this.#physicalFrameQueue = queue
|
|
551
567
|
})
|
|
552
|
-
physical.once("
|
|
568
|
+
physical.once("end", () => { attemptDisconnectReason ??= "remote_eof" })
|
|
569
|
+
physical.once("error", (error) => {
|
|
570
|
+
attemptError = error
|
|
571
|
+
attemptDisconnectReason ??= "remote_socket_error"
|
|
572
|
+
attemptSafeSocketCode ??= safeSocketCode(error)
|
|
573
|
+
if (isFatalTransportError(error)) this.#fatal = true
|
|
574
|
+
})
|
|
553
575
|
physical.once("close", () => {
|
|
554
576
|
clearTimeout(timer)
|
|
555
577
|
void queue.finally(() => {
|
|
@@ -561,11 +583,28 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
561
583
|
this.#resumeNextPage = 0
|
|
562
584
|
if (!this.#closing && !this.#fatal) this.#setState("reconnecting")
|
|
563
585
|
const lost = new SocketductSessionLostError("Reverse gateway physical transport was interrupted", "GATEWAY_DISCONNECTED")
|
|
586
|
+
const attribution = attemptAttribution ?? this.#disconnectAttribution ?? createDisconnectAttribution({
|
|
587
|
+
disconnectReason: attemptDisconnectReason ?? "unknown_transport_close",
|
|
588
|
+
incidentOrigin: "reverse_client",
|
|
589
|
+
physicalRole: "client",
|
|
590
|
+
clientConnectionGeneration: generation,
|
|
591
|
+
fanOutCount: this.#streams.size,
|
|
592
|
+
resumeSuccessCount: 0,
|
|
593
|
+
terminalStreamCount: 0
|
|
594
|
+
})
|
|
595
|
+
this.#disconnectAttribution = attribution
|
|
564
596
|
if (!this.#closing) {
|
|
597
|
+
logDiagnostic(this.#options.logger, "reverse_client.physical_detached", {
|
|
598
|
+
...this.#routeIdentityFields(),
|
|
599
|
+
...attribution,
|
|
600
|
+
...(attemptSafeSocketCode === undefined ? {} : {safeSocketCode: attemptSafeSocketCode})
|
|
601
|
+
})
|
|
565
602
|
logDiagnostic(this.#options.logger, "transport.disconnected", {
|
|
566
|
-
generation,
|
|
603
|
+
clientConnectionGeneration: generation,
|
|
567
604
|
...this.#routeIdentityFields(),
|
|
568
605
|
virtualStreams: this.#streams.size,
|
|
606
|
+
...attribution,
|
|
607
|
+
...(attemptSafeSocketCode === undefined ? {} : {safeSocketCode: attemptSafeSocketCode}),
|
|
569
608
|
...errorDiagnostics(attemptError ?? lost, "GATEWAY_DISCONNECTED")
|
|
570
609
|
})
|
|
571
610
|
}
|
|
@@ -681,7 +720,7 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
681
720
|
const detail = decodeV3Control(frame.payload)
|
|
682
721
|
const code = typeof detail.code === "string" ? detail.code : "STREAM_RESET"
|
|
683
722
|
stream.state = "reset"
|
|
684
|
-
const error = new SocketductStreamError("Reverse gateway stream was reset", code)
|
|
723
|
+
const error = attachDisconnectAttribution(new SocketductStreamError("Reverse gateway stream was reset", code), validHubAttribution(detail) ?? (code === "GATEWAY_DISCONNECTED" ? this.#disconnectAttribution : undefined))
|
|
685
724
|
stream.socket._receiveError(error)
|
|
686
725
|
return
|
|
687
726
|
}
|
|
@@ -725,7 +764,7 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
725
764
|
if (stream === undefined) return
|
|
726
765
|
if (result.status !== "resumed") {
|
|
727
766
|
const code = typeof result.code === "string" ? result.code : "RESUME_REJECTED"
|
|
728
|
-
stream.socket._receiveError(new SocketductSessionLostError("Reverse gateway stream could not be resumed", code))
|
|
767
|
+
stream.socket._receiveError(attachDisconnectAttribution(new SocketductSessionLostError("Reverse gateway stream could not be resumed", code), validHubAttribution(result) ?? this.#disconnectAttribution))
|
|
729
768
|
return
|
|
730
769
|
}
|
|
731
770
|
const clientToGatewaySequence = /** @type {number} */ (result.clientToGatewaySequence)
|
|
@@ -954,7 +993,19 @@ export class ReverseGatewayConnection extends EventEmitter {
|
|
|
954
993
|
/** @param {number} type - Type. @param {number} streamId - Stream. @param {number} sequence - Sequence. @param {Record<string, unknown>} body - Body. */
|
|
955
994
|
async #sendControl(type, streamId, sequence, body) {
|
|
956
995
|
const physical = this.#physical
|
|
957
|
-
if (physical === undefined || physical.destroyed)
|
|
996
|
+
if (physical === undefined || physical.destroyed) {
|
|
997
|
+
this.#disconnectAttribution ??= createDisconnectAttribution({
|
|
998
|
+
disconnectReason: "send_after_transport_loss",
|
|
999
|
+
incidentOrigin: "reverse_client",
|
|
1000
|
+
physicalRole: "client",
|
|
1001
|
+
clientConnectionGeneration: this.#generation,
|
|
1002
|
+
fanOutCount: this.#streams.size,
|
|
1003
|
+
resumeSuccessCount: 0,
|
|
1004
|
+
terminalStreamCount: 0
|
|
1005
|
+
})
|
|
1006
|
+
logDiagnostic(this.#options.logger, "reverse_client.send_after_transport_loss", {...this.#routeIdentityFields(), ...this.#disconnectAttribution})
|
|
1007
|
+
throw attachDisconnectAttribution(new SocketductSessionLostError("Reverse gateway transport is unavailable", "GATEWAY_DISCONNECTED"), this.#disconnectAttribution)
|
|
1008
|
+
}
|
|
958
1009
|
return await writeSocket(physical, encodeV3Frame({type, streamId, sequence}, encodeV3Control(body)))
|
|
959
1010
|
}
|
|
960
1011
|
|
|
@@ -1168,9 +1219,18 @@ function installCallback(socket, callback) {
|
|
|
1168
1219
|
}
|
|
1169
1220
|
|
|
1170
1221
|
function fatalError(/** @type {string} */ code) { return Object.assign(new Error("Reverse gateway client was rejected by the hub"), {code, fatal: true}) }
|
|
1171
|
-
function connectionError(/** @type {string} */ code) {
|
|
1172
|
-
if (code === "HUB_SHUTDOWN" || code === "GATEWAY_UNAVAILABLE" || code === "GATEWAY_DISCONNECTED" || code === "GATEWAY_REPLACED") return Object.assign(new Error("Selected reverse gateway is temporarily unavailable"), {code, fatal: false})
|
|
1173
|
-
return fatalError(code)
|
|
1222
|
+
function connectionError(/** @type {string} */ code, /** @type {unknown} */ attribution = undefined) {
|
|
1223
|
+
if (code === "HUB_SHUTDOWN" || code === "GATEWAY_UNAVAILABLE" || code === "GATEWAY_DISCONNECTED" || code === "GATEWAY_REPLACED") return attachDisconnectAttribution(Object.assign(new Error("Selected reverse gateway is temporarily unavailable"), {code, fatal: false}), attribution)
|
|
1224
|
+
return attachDisconnectAttribution(fatalError(code), attribution)
|
|
1225
|
+
}
|
|
1226
|
+
/** Accepts attribution only from the authenticated hub transport. */
|
|
1227
|
+
function validHubAttribution(/** @type {unknown} */ value) {
|
|
1228
|
+
const attribution = parseDisconnectAttribution(value)
|
|
1229
|
+
return attribution?.incidentOrigin === "reverse_hub" ? attribution : undefined
|
|
1230
|
+
}
|
|
1231
|
+
/** Extracts only the closed public attribution contract from an error. */
|
|
1232
|
+
function publicDisconnectAttribution(/** @type {unknown} */ error) {
|
|
1233
|
+
return parseDisconnectAttribution(error) ?? {}
|
|
1174
1234
|
}
|
|
1175
1235
|
/** @param {unknown} value - Advertised peer receive window. @returns {value is number} Whether valid. */
|
|
1176
1236
|
function validPeerWindow(value) { return Number.isSafeInteger(value) && /** @type {number} */ (value) > 0 }
|