teyyare 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/teyyare.js +37 -10
- package/package.json +1 -1
- package/src/cli/progress.js +4 -0
- package/src/index.js +196 -5
- package/src/receiver/pipeline.js +91 -12
- package/src/sender/pipeline.js +33 -4
- package/test/integration/multichannel.test.js +108 -0
package/bin/teyyare.js
CHANGED
|
@@ -13,15 +13,20 @@ function printUsage() {
|
|
|
13
13
|
|
|
14
14
|
\x1b[1mUSAGE:\x1b[0m
|
|
15
15
|
teyyare serve [--port 7421] [--dir <path>]
|
|
16
|
-
teyyare send <source> <destination>
|
|
17
|
-
teyyare pull <source> <
|
|
16
|
+
teyyare send <source> <destination> [-c <channels>]
|
|
17
|
+
teyyare pull <source> [destination] [-c <channels>]
|
|
18
18
|
teyyare status
|
|
19
19
|
|
|
20
|
+
\x1b[1mOPTIONS:\x1b[0m
|
|
21
|
+
-c, --channels <n> Parallel TCP connections (default: 4 for remote, 1 for local)
|
|
22
|
+
--port <port> Port number (default: 7421)
|
|
23
|
+
--dir <path> Serve root directory (default: current dir)
|
|
24
|
+
|
|
20
25
|
\x1b[1mEXAMPLES:\x1b[0m
|
|
21
26
|
teyyare serve --port 7421
|
|
22
27
|
teyyare send ./10gb.bin localhost:/tmp/10gb.bin
|
|
23
|
-
teyyare send ./dist 127.0.0.1:7421:/var/www/app
|
|
24
|
-
teyyare pull server:7421:/backup/db.tar.zst ./
|
|
28
|
+
teyyare send ./dist 127.0.0.1:7421:/var/www/app -c 4
|
|
29
|
+
teyyare pull server:7421:/backup/db.tar.zst ./ -c 4
|
|
25
30
|
`);
|
|
26
31
|
}
|
|
27
32
|
|
|
@@ -51,7 +56,7 @@ async function main() {
|
|
|
51
56
|
}
|
|
52
57
|
|
|
53
58
|
if (command === '--version' || command === '-v' || command === 'version') {
|
|
54
|
-
console.log('0.2.
|
|
59
|
+
console.log('0.2.7');
|
|
55
60
|
process.exit(0);
|
|
56
61
|
}
|
|
57
62
|
|
|
@@ -82,8 +87,18 @@ async function main() {
|
|
|
82
87
|
// Keep alive
|
|
83
88
|
await new Promise(() => {});
|
|
84
89
|
} else if (command === 'send') {
|
|
85
|
-
|
|
86
|
-
const
|
|
90
|
+
let channels = undefined;
|
|
91
|
+
const positional = [];
|
|
92
|
+
for (let i = 1; i < args.length; i++) {
|
|
93
|
+
if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
|
|
94
|
+
channels = Number(args[++i]);
|
|
95
|
+
} else {
|
|
96
|
+
positional.push(args[i]);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const source = positional[0];
|
|
101
|
+
const target = positional[1];
|
|
87
102
|
|
|
88
103
|
if (!source || !target) {
|
|
89
104
|
console.error('Error: teyyare send requires <source> and <destination>');
|
|
@@ -96,6 +111,7 @@ async function main() {
|
|
|
96
111
|
sourcePath: source,
|
|
97
112
|
host,
|
|
98
113
|
port,
|
|
114
|
+
channels,
|
|
99
115
|
showProgress: true
|
|
100
116
|
});
|
|
101
117
|
|
|
@@ -112,8 +128,18 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
|
112
128
|
`);
|
|
113
129
|
process.exit(0);
|
|
114
130
|
} else if (command === 'pull') {
|
|
115
|
-
|
|
116
|
-
const
|
|
131
|
+
let channels = undefined;
|
|
132
|
+
const positional = [];
|
|
133
|
+
for (let i = 1; i < args.length; i++) {
|
|
134
|
+
if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
|
|
135
|
+
channels = Number(args[++i]);
|
|
136
|
+
} else {
|
|
137
|
+
positional.push(args[i]);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const source = positional[0];
|
|
142
|
+
const target = positional[1] || './';
|
|
117
143
|
|
|
118
144
|
if (!source) {
|
|
119
145
|
console.error('Error: teyyare pull requires <source> and [destination]');
|
|
@@ -126,6 +152,7 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
|
126
152
|
sourceRemotePath: remotePath,
|
|
127
153
|
host,
|
|
128
154
|
port,
|
|
155
|
+
channels,
|
|
129
156
|
destinationDir: target,
|
|
130
157
|
showProgress: true
|
|
131
158
|
});
|
|
@@ -143,7 +170,7 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
|
143
170
|
`);
|
|
144
171
|
process.exit(0);
|
|
145
172
|
} else if (command === 'status') {
|
|
146
|
-
console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.
|
|
173
|
+
console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.7 status: ready');
|
|
147
174
|
} else {
|
|
148
175
|
console.error(`Unknown command: ${command}`);
|
|
149
176
|
printUsage();
|
package/package.json
CHANGED
package/src/cli/progress.js
CHANGED
|
@@ -28,6 +28,7 @@ export class ProgressReporter {
|
|
|
28
28
|
this.tcpPressure = 0;
|
|
29
29
|
this.resumeHits = 0;
|
|
30
30
|
this.diskSpeedMBs = 0;
|
|
31
|
+
this.channels = 1;
|
|
31
32
|
|
|
32
33
|
this.startTime = Date.now();
|
|
33
34
|
this.lastRender = 0;
|
|
@@ -68,6 +69,7 @@ export class ProgressReporter {
|
|
|
68
69
|
if (state.tcpPressure !== undefined) this.tcpPressure = state.tcpPressure;
|
|
69
70
|
if (state.resumeHits !== undefined) this.resumeHits = state.resumeHits;
|
|
70
71
|
if (state.diskSpeedMBs !== undefined) this.diskSpeedMBs = state.diskSpeedMBs;
|
|
72
|
+
if (state.channels !== undefined) this.channels = state.channels;
|
|
71
73
|
}
|
|
72
74
|
|
|
73
75
|
formatBytes(bytes) {
|
|
@@ -126,6 +128,7 @@ export class ProgressReporter {
|
|
|
126
128
|
'',
|
|
127
129
|
`network speed \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
|
|
128
130
|
`disk speed \x1b[1m${diskStr}\x1b[0m`,
|
|
131
|
+
`channels ${this.channels}`,
|
|
129
132
|
`ETA ${etaStr}`,
|
|
130
133
|
'',
|
|
131
134
|
`chunks ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
|
|
@@ -143,6 +146,7 @@ export class ProgressReporter {
|
|
|
143
146
|
`\x1b[32m${percent}%\x1b[0m`,
|
|
144
147
|
'',
|
|
145
148
|
`throughput \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
|
|
149
|
+
`channels ${this.channels}`,
|
|
146
150
|
`ETA ${etaStr}`,
|
|
147
151
|
'',
|
|
148
152
|
`chunks ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
|
package/src/index.js
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import path from 'node:path';
|
|
6
|
+
import net from 'node:net';
|
|
6
7
|
import { existsSync } from 'node:fs';
|
|
7
8
|
import { TCPTransport, ReplicationPipeline, BatchReceipt } from 'raptiye';
|
|
8
|
-
import { readHeader, decodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
|
|
9
|
+
import { readHeader, decodeMessage, encodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
|
|
9
10
|
import { MessageType } from 'raptiye/src/types.js';
|
|
10
11
|
import { crc32Buffers } from './core/crc32.js';
|
|
11
12
|
import { SenderPipeline } from './sender/pipeline.js';
|
|
@@ -25,8 +26,19 @@ import {
|
|
|
25
26
|
unpackPullError
|
|
26
27
|
} from './core/protocol.js';
|
|
27
28
|
|
|
28
|
-
// High-performance streaming TCP framer patch for TCPTransport
|
|
29
|
-
|
|
29
|
+
// High-performance streaming TCP framer & Multi-Channel socket pool patch for TCPTransport
|
|
30
|
+
TCPTransport.prototype._getPeerPool = function(peer) {
|
|
31
|
+
if (!this._peerSocketPool) {
|
|
32
|
+
this._peerSocketPool = new Map();
|
|
33
|
+
}
|
|
34
|
+
let pool = this._peerSocketPool.get(peer);
|
|
35
|
+
if (!pool) {
|
|
36
|
+
pool = [];
|
|
37
|
+
this._peerSocketPool.set(peer, pool);
|
|
38
|
+
}
|
|
39
|
+
return pool;
|
|
40
|
+
};
|
|
41
|
+
|
|
30
42
|
TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
31
43
|
socket.setNoDelay(true);
|
|
32
44
|
if (socket._readableState) socket._readableState.highWaterMark = 4 * 1024 * 1024;
|
|
@@ -102,6 +114,10 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
102
114
|
const msg = decodeMessage(frame, 0, false);
|
|
103
115
|
if (msg.sourceNode) {
|
|
104
116
|
this._outboundSockets.set(msg.sourceNode, socket);
|
|
117
|
+
const pool = this._getPeerPool(msg.sourceNode);
|
|
118
|
+
if (!pool.includes(socket)) {
|
|
119
|
+
pool.push(socket);
|
|
120
|
+
}
|
|
105
121
|
}
|
|
106
122
|
if (this._messageHandler) {
|
|
107
123
|
this._messageHandler(msg.sourceNode, msg);
|
|
@@ -119,6 +135,20 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
119
135
|
this._outboundSockets.delete(peer);
|
|
120
136
|
}
|
|
121
137
|
}
|
|
138
|
+
if (this._peerSocketPool) {
|
|
139
|
+
for (const [peer, pool] of this._peerSocketPool.entries()) {
|
|
140
|
+
const idx = pool.indexOf(socket);
|
|
141
|
+
if (idx !== -1) {
|
|
142
|
+
pool.splice(idx, 1);
|
|
143
|
+
}
|
|
144
|
+
if (pool.length === 0 && this.onPeerDisconnect) {
|
|
145
|
+
this.onPeerDisconnect(peer);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (this._inboundSockets.size === 0 && this.onPeerDisconnect) {
|
|
150
|
+
this.onPeerDisconnect(1);
|
|
151
|
+
}
|
|
122
152
|
});
|
|
123
153
|
|
|
124
154
|
socket.on('error', () => {
|
|
@@ -126,6 +156,136 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
126
156
|
});
|
|
127
157
|
};
|
|
128
158
|
|
|
159
|
+
TCPTransport.prototype.connectChannels = async function(peer, channelCount = 4) {
|
|
160
|
+
if (channelCount <= 1) return;
|
|
161
|
+
const addr = this.peerAddresses.get(peer);
|
|
162
|
+
if (!addr) {
|
|
163
|
+
throw new Error(`[TCPTransport ${this.id}] No address registered for peer ${peer}`);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const pool = this._getPeerPool(peer);
|
|
167
|
+
const activeSockets = pool.filter((s) => !s.destroyed && s.writable);
|
|
168
|
+
const needed = channelCount - activeSockets.length;
|
|
169
|
+
if (needed <= 0) return;
|
|
170
|
+
|
|
171
|
+
const connectPromises = [];
|
|
172
|
+
for (let c = 0; c < needed; c++) {
|
|
173
|
+
const p = new Promise((resolve, reject) => {
|
|
174
|
+
const socket = net.createConnection({ host: addr.host, port: addr.port }, () => {
|
|
175
|
+
this._attachSocketReader(socket);
|
|
176
|
+
if (!this._outboundSockets.has(peer)) {
|
|
177
|
+
this._outboundSockets.set(peer, socket);
|
|
178
|
+
}
|
|
179
|
+
const currentPool = this._getPeerPool(peer);
|
|
180
|
+
if (!currentPool.includes(socket)) {
|
|
181
|
+
currentPool.push(socket);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Send registration HEARTBEAT frame so remote peer associates this socket with this node
|
|
185
|
+
try {
|
|
186
|
+
const hbChunks = encodeMessage({
|
|
187
|
+
type: MessageType.HEARTBEAT,
|
|
188
|
+
sourceNode: this.id,
|
|
189
|
+
destNode: peer,
|
|
190
|
+
leaderId: this.id,
|
|
191
|
+
leaderCommit: 0n
|
|
192
|
+
});
|
|
193
|
+
socket.cork();
|
|
194
|
+
for (let i = 0; i < hbChunks.length; i++) {
|
|
195
|
+
socket.write(hbChunks[i]);
|
|
196
|
+
}
|
|
197
|
+
socket.uncork();
|
|
198
|
+
} catch (err) {
|
|
199
|
+
// Non-fatal
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
resolve(socket);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
socket.on('error', (err) => {
|
|
206
|
+
reject(err);
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
connectPromises.push(p);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
await Promise.all(connectPromises);
|
|
213
|
+
// Short pause to allow remote peer to process initial registration frames
|
|
214
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
TCPTransport.prototype.sendv = function(peer, buffers) {
|
|
218
|
+
if (!this._channelIndex) this._channelIndex = new Map();
|
|
219
|
+
|
|
220
|
+
const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
|
|
221
|
+
if (pool && pool.length > 0) {
|
|
222
|
+
const active = pool.filter((s) => !s.destroyed && s.writable);
|
|
223
|
+
if (active.length > 0) {
|
|
224
|
+
let idx = this._channelIndex.get(peer) || 0;
|
|
225
|
+
const socket = active[idx % active.length];
|
|
226
|
+
this._channelIndex.set(peer, idx + 1);
|
|
227
|
+
|
|
228
|
+
socket.cork();
|
|
229
|
+
for (let i = 0; i < buffers.length; i++) {
|
|
230
|
+
socket.write(buffers[i]);
|
|
231
|
+
}
|
|
232
|
+
socket.uncork();
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
this._getOrCreateSocket(peer)
|
|
238
|
+
.then((socket) => {
|
|
239
|
+
if (!socket.destroyed && socket.writable) {
|
|
240
|
+
socket.cork();
|
|
241
|
+
for (let i = 0; i < buffers.length; i++) {
|
|
242
|
+
socket.write(buffers[i]);
|
|
243
|
+
}
|
|
244
|
+
socket.uncork();
|
|
245
|
+
}
|
|
246
|
+
})
|
|
247
|
+
.catch(() => {});
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
TCPTransport.prototype.send = function(peer, buffer) {
|
|
251
|
+
if (!this._channelIndex) this._channelIndex = new Map();
|
|
252
|
+
|
|
253
|
+
const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
|
|
254
|
+
if (pool && pool.length > 0) {
|
|
255
|
+
const active = pool.filter((s) => !s.destroyed && s.writable);
|
|
256
|
+
if (active.length > 0) {
|
|
257
|
+
let idx = this._channelIndex.get(peer) || 0;
|
|
258
|
+
const socket = active[idx % active.length];
|
|
259
|
+
this._channelIndex.set(peer, idx + 1);
|
|
260
|
+
socket.write(buffer);
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
this._getOrCreateSocket(peer)
|
|
266
|
+
.then((socket) => {
|
|
267
|
+
if (!socket.destroyed && socket.writable) {
|
|
268
|
+
socket.write(buffer);
|
|
269
|
+
}
|
|
270
|
+
})
|
|
271
|
+
.catch(() => {});
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
const origClose = TCPTransport.prototype.close;
|
|
275
|
+
TCPTransport.prototype.close = async function() {
|
|
276
|
+
if (this._peerSocketPool) {
|
|
277
|
+
for (const pool of this._peerSocketPool.values()) {
|
|
278
|
+
for (const socket of pool) {
|
|
279
|
+
if (!socket.destroyed) {
|
|
280
|
+
socket.destroy();
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
this._peerSocketPool.clear();
|
|
285
|
+
}
|
|
286
|
+
return origClose.call(this);
|
|
287
|
+
};
|
|
288
|
+
|
|
129
289
|
// Fast SIMD-accelerated BATCH_DATA encoder
|
|
130
290
|
function fastEncodeBatchData(msg) {
|
|
131
291
|
const bufs = msg.buffers || [];
|
|
@@ -304,6 +464,15 @@ export async function serve({
|
|
|
304
464
|
|
|
305
465
|
let activeSender = null;
|
|
306
466
|
|
|
467
|
+
transport.onPeerDisconnect = async (peer) => {
|
|
468
|
+
if (activeSender) {
|
|
469
|
+
const s = activeSender;
|
|
470
|
+
activeSender = null;
|
|
471
|
+
await s.abort();
|
|
472
|
+
}
|
|
473
|
+
replication.reset();
|
|
474
|
+
};
|
|
475
|
+
|
|
307
476
|
replication.onBatch(async (msg) => {
|
|
308
477
|
if (!msg.buffers || msg.buffers.length < 2) return;
|
|
309
478
|
const [metaBuf, payloadBuf] = msg.buffers;
|
|
@@ -327,8 +496,9 @@ export async function serve({
|
|
|
327
496
|
|
|
328
497
|
// Abort any existing running sender before starting new transfer
|
|
329
498
|
if (activeSender) {
|
|
330
|
-
activeSender
|
|
499
|
+
const s = activeSender;
|
|
331
500
|
activeSender = null;
|
|
501
|
+
await s.abort();
|
|
332
502
|
}
|
|
333
503
|
|
|
334
504
|
const manifest = await TransferManifest.build(resolved);
|
|
@@ -380,6 +550,12 @@ export async function serve({
|
|
|
380
550
|
receiver,
|
|
381
551
|
metrics,
|
|
382
552
|
async close() {
|
|
553
|
+
if (activeSender) {
|
|
554
|
+
const s = activeSender;
|
|
555
|
+
activeSender = null;
|
|
556
|
+
await s.abort();
|
|
557
|
+
}
|
|
558
|
+
await receiver.close();
|
|
383
559
|
metrics.stop();
|
|
384
560
|
await transport.close();
|
|
385
561
|
}
|
|
@@ -401,12 +577,14 @@ export async function send({
|
|
|
401
577
|
sourcePath,
|
|
402
578
|
host = '127.0.0.1',
|
|
403
579
|
port = 7421,
|
|
580
|
+
channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
|
|
404
581
|
pipelineOptions = {},
|
|
405
582
|
showProgress = true,
|
|
406
583
|
knownResumeState = new Map()
|
|
407
584
|
}) {
|
|
408
585
|
const metrics = new MetricsCollector();
|
|
409
586
|
const progress = showProgress ? new ProgressReporter({ role: 'sender' }) : null;
|
|
587
|
+
if (progress) progress.update({ channels });
|
|
410
588
|
|
|
411
589
|
// Ephemeral sender port
|
|
412
590
|
const senderPort = 10000 + Math.floor(Math.random() * 20000);
|
|
@@ -418,6 +596,9 @@ export async function send({
|
|
|
418
596
|
});
|
|
419
597
|
|
|
420
598
|
await transport.start();
|
|
599
|
+
if (channels > 1) {
|
|
600
|
+
await transport.connectChannels(2, channels);
|
|
601
|
+
}
|
|
421
602
|
|
|
422
603
|
const replication = new ReplicationPipeline({
|
|
423
604
|
transport,
|
|
@@ -441,6 +622,7 @@ export async function send({
|
|
|
441
622
|
try {
|
|
442
623
|
await sender.transfer();
|
|
443
624
|
} finally {
|
|
625
|
+
await sender.abort();
|
|
444
626
|
metrics.stop();
|
|
445
627
|
await transport.close();
|
|
446
628
|
}
|
|
@@ -454,6 +636,7 @@ export async function send({
|
|
|
454
636
|
* @param {string} options.sourceRemotePath
|
|
455
637
|
* @param {string} [options.host='127.0.0.1']
|
|
456
638
|
* @param {number} [options.port=7421]
|
|
639
|
+
* @param {number} [options.channels]
|
|
457
640
|
* @param {string} [options.destinationDir='./']
|
|
458
641
|
* @param {object} [options.pipelineOptions]
|
|
459
642
|
* @param {boolean} [options.showProgress=true]
|
|
@@ -463,13 +646,17 @@ export async function pull({
|
|
|
463
646
|
sourceRemotePath,
|
|
464
647
|
host = '127.0.0.1',
|
|
465
648
|
port = 7421,
|
|
649
|
+
channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
|
|
466
650
|
destinationDir = './',
|
|
467
651
|
pipelineOptions = {},
|
|
468
652
|
showProgress = true
|
|
469
653
|
}) {
|
|
470
654
|
const metrics = new MetricsCollector();
|
|
471
655
|
const progress = showProgress ? new ProgressReporter({ role: 'receiver' }) : null;
|
|
472
|
-
if (progress)
|
|
656
|
+
if (progress) {
|
|
657
|
+
progress.update({ channels });
|
|
658
|
+
progress.start();
|
|
659
|
+
}
|
|
473
660
|
|
|
474
661
|
const clientPort = 10000 + Math.floor(Math.random() * 20000);
|
|
475
662
|
const transport = new TCPTransport({
|
|
@@ -480,6 +667,9 @@ export async function pull({
|
|
|
480
667
|
});
|
|
481
668
|
|
|
482
669
|
await transport.start();
|
|
670
|
+
if (channels > 1) {
|
|
671
|
+
await transport.connectChannels(2, channels);
|
|
672
|
+
}
|
|
483
673
|
|
|
484
674
|
const replication = new ReplicationPipeline({
|
|
485
675
|
transport,
|
|
@@ -520,6 +710,7 @@ export async function pull({
|
|
|
520
710
|
await receiver.waitForCompletion();
|
|
521
711
|
} finally {
|
|
522
712
|
if (progress) progress.stop();
|
|
713
|
+
await receiver.close();
|
|
523
714
|
metrics.stop();
|
|
524
715
|
await transport.close();
|
|
525
716
|
}
|
package/src/receiver/pipeline.js
CHANGED
|
@@ -37,6 +37,8 @@ export class ReceiverPipeline {
|
|
|
37
37
|
|
|
38
38
|
this.activeFiles = new Map(); // fileId -> { destPath, partPath, statePath, fileHandle, resumeState, ... }
|
|
39
39
|
this.completedFiles = new Set();
|
|
40
|
+
this._pendingChunks = new Map(); // fileId -> Array<{ key, val }>
|
|
41
|
+
this._pendingTransferEnd = false;
|
|
40
42
|
this.transferFinished = false;
|
|
41
43
|
this.transferError = null;
|
|
42
44
|
this._finishResolvers = [];
|
|
@@ -135,6 +137,7 @@ export class ReceiverPipeline {
|
|
|
135
137
|
case Opcodes.TRANSFER_BEGIN: {
|
|
136
138
|
this.transferFinished = false;
|
|
137
139
|
this.transferError = null;
|
|
140
|
+
this._pendingTransferEnd = false;
|
|
138
141
|
const tinfo = unpackTransferBegin(key);
|
|
139
142
|
break;
|
|
140
143
|
}
|
|
@@ -155,12 +158,11 @@ export class ReceiverPipeline {
|
|
|
155
158
|
}
|
|
156
159
|
|
|
157
160
|
case Opcodes.TRANSFER_END: {
|
|
158
|
-
this.
|
|
159
|
-
|
|
160
|
-
|
|
161
|
+
if (this.activeFiles.size === 0) {
|
|
162
|
+
this._finishTransfer();
|
|
163
|
+
} else {
|
|
164
|
+
this._pendingTransferEnd = true;
|
|
161
165
|
}
|
|
162
|
-
this._finishResolvers = [];
|
|
163
|
-
this._errorRejecters = [];
|
|
164
166
|
break;
|
|
165
167
|
}
|
|
166
168
|
|
|
@@ -179,6 +181,15 @@ export class ReceiverPipeline {
|
|
|
179
181
|
}
|
|
180
182
|
}
|
|
181
183
|
|
|
184
|
+
_finishTransfer() {
|
|
185
|
+
this.transferFinished = true;
|
|
186
|
+
for (const resolve of this._finishResolvers) {
|
|
187
|
+
resolve();
|
|
188
|
+
}
|
|
189
|
+
this._finishResolvers = [];
|
|
190
|
+
this._errorRejecters = [];
|
|
191
|
+
}
|
|
192
|
+
|
|
182
193
|
async _handleFileBegin(key) {
|
|
183
194
|
const info = unpackFileBegin(key);
|
|
184
195
|
const destPath = path.resolve(this.destinationDir, info.path);
|
|
@@ -200,7 +211,7 @@ export class ReceiverPipeline {
|
|
|
200
211
|
|
|
201
212
|
const fileHandle = await fs.open(partPath, 'a+');
|
|
202
213
|
|
|
203
|
-
|
|
214
|
+
const file = {
|
|
204
215
|
fileId: info.fileId,
|
|
205
216
|
relativePath: info.path,
|
|
206
217
|
destPath,
|
|
@@ -210,8 +221,12 @@ export class ReceiverPipeline {
|
|
|
210
221
|
resumeState,
|
|
211
222
|
fileSize: Number(info.fileSize),
|
|
212
223
|
sha256: info.sha256,
|
|
213
|
-
chunksWrittenSinceSave: 0
|
|
214
|
-
|
|
224
|
+
chunksWrittenSinceSave: 0,
|
|
225
|
+
pendingEnd: false,
|
|
226
|
+
endMeta: null
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
this.activeFiles.set(info.fileId, file);
|
|
215
230
|
|
|
216
231
|
if (this.progress) {
|
|
217
232
|
const fileSize = Number(info.fileSize);
|
|
@@ -226,16 +241,38 @@ export class ReceiverPipeline {
|
|
|
226
241
|
resumeHits: resumeState.verifiedCount
|
|
227
242
|
});
|
|
228
243
|
}
|
|
244
|
+
|
|
245
|
+
// Flush any pending chunks that arrived out-of-order before FILE_BEGIN
|
|
246
|
+
const pending = this._pendingChunks.get(info.fileId);
|
|
247
|
+
if (pending && pending.length > 0) {
|
|
248
|
+
this._pendingChunks.delete(info.fileId);
|
|
249
|
+
for (const item of pending) {
|
|
250
|
+
const meta = unpackChunkMeta(item.key);
|
|
251
|
+
await this._writeChunk(file, meta, item.val);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
229
254
|
}
|
|
230
255
|
|
|
231
256
|
async _handleChunk(key, val) {
|
|
232
257
|
const meta = unpackChunkMeta(key);
|
|
233
258
|
const file = this.activeFiles.get(meta.fileId);
|
|
234
259
|
if (!file) {
|
|
235
|
-
|
|
260
|
+
if (this.completedFiles.has(meta.fileId)) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
let pending = this._pendingChunks.get(meta.fileId);
|
|
264
|
+
if (!pending) {
|
|
265
|
+
pending = [];
|
|
266
|
+
this._pendingChunks.set(meta.fileId, pending);
|
|
267
|
+
}
|
|
268
|
+
pending.push({ key, val: new Uint8Array(val) });
|
|
236
269
|
return;
|
|
237
270
|
}
|
|
238
271
|
|
|
272
|
+
await this._writeChunk(file, meta, val);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async _writeChunk(file, meta, val) {
|
|
239
276
|
// Idempotency / resume hit check
|
|
240
277
|
if (file.resumeState.isVerified(meta.chunkIndex)) {
|
|
241
278
|
if (this.metrics) {
|
|
@@ -296,6 +333,14 @@ export class ReceiverPipeline {
|
|
|
296
333
|
file.chunksWrittenSinceSave = 0;
|
|
297
334
|
await file.resumeState.save(file.statePath);
|
|
298
335
|
}
|
|
336
|
+
|
|
337
|
+
// If FILE_END arrived earlier while chunks were still arriving
|
|
338
|
+
if (file.pendingEnd && (file.fileSize === 0 || file.resumeState.isComplete())) {
|
|
339
|
+
await this._finalizeFile(file);
|
|
340
|
+
if (this._pendingTransferEnd && this.activeFiles.size === 0) {
|
|
341
|
+
this._finishTransfer();
|
|
342
|
+
}
|
|
343
|
+
}
|
|
299
344
|
}
|
|
300
345
|
|
|
301
346
|
async _handleFileEnd(key) {
|
|
@@ -303,15 +348,32 @@ export class ReceiverPipeline {
|
|
|
303
348
|
const file = this.activeFiles.get(meta.fileId);
|
|
304
349
|
if (!file) return;
|
|
305
350
|
|
|
351
|
+
if (meta.sha256) {
|
|
352
|
+
file.sha256 = meta.sha256;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
if (file.fileSize === 0 || file.resumeState.isComplete()) {
|
|
356
|
+
await this._finalizeFile(file);
|
|
357
|
+
if (this._pendingTransferEnd && this.activeFiles.size === 0) {
|
|
358
|
+
this._finishTransfer();
|
|
359
|
+
}
|
|
360
|
+
} else {
|
|
361
|
+
file.pendingEnd = true;
|
|
362
|
+
file.endMeta = meta;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
async _finalizeFile(file) {
|
|
306
367
|
// Save final state before rename
|
|
307
368
|
await file.resumeState.save(file.statePath);
|
|
308
369
|
|
|
309
370
|
// Sync disk buffers
|
|
310
371
|
await file.fileHandle.sync();
|
|
311
372
|
await file.fileHandle.close();
|
|
373
|
+
file.fileHandle = null;
|
|
312
374
|
|
|
313
375
|
// Verify whole-file checksum if available
|
|
314
|
-
const expectedSha =
|
|
376
|
+
const expectedSha = file.sha256;
|
|
315
377
|
if (expectedSha) {
|
|
316
378
|
const actualSha = await this._computeFileSha256(file.partPath);
|
|
317
379
|
if (actualSha !== expectedSha) {
|
|
@@ -323,17 +385,34 @@ export class ReceiverPipeline {
|
|
|
323
385
|
await fs.rename(file.partPath, file.destPath);
|
|
324
386
|
await FileResumeState.cleanup(file.statePath);
|
|
325
387
|
|
|
326
|
-
this.activeFiles.delete(
|
|
388
|
+
this.activeFiles.delete(file.fileId);
|
|
389
|
+
this.completedFiles.add(file.fileId);
|
|
327
390
|
this.completedFiles.add(file.destPath);
|
|
328
391
|
}
|
|
329
392
|
|
|
393
|
+
async close() {
|
|
394
|
+
for (const [fileId, file] of this.activeFiles.entries()) {
|
|
395
|
+
if (file.fileHandle) {
|
|
396
|
+
try {
|
|
397
|
+
await file.fileHandle.close();
|
|
398
|
+
} catch {}
|
|
399
|
+
file.fileHandle = null;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
this.activeFiles.clear();
|
|
403
|
+
this._pendingChunks.clear();
|
|
404
|
+
}
|
|
405
|
+
|
|
330
406
|
async _computeFileSha256(filePath) {
|
|
331
407
|
return new Promise((resolve, reject) => {
|
|
332
408
|
const hash = crypto.createHash('sha256');
|
|
333
409
|
const stream = createReadStream(filePath);
|
|
334
410
|
stream.on('data', (d) => hash.update(d));
|
|
335
411
|
stream.on('end', () => resolve(hash.digest('hex')));
|
|
336
|
-
stream.on('error',
|
|
412
|
+
stream.on('error', (err) => {
|
|
413
|
+
stream.destroy(err);
|
|
414
|
+
reject(err);
|
|
415
|
+
});
|
|
337
416
|
});
|
|
338
417
|
}
|
|
339
418
|
}
|
package/src/sender/pipeline.js
CHANGED
|
@@ -141,14 +141,27 @@ export class SenderPipeline {
|
|
|
141
141
|
}
|
|
142
142
|
}
|
|
143
143
|
});
|
|
144
|
+
this._activeHandle = null;
|
|
145
|
+
this._abortWaiters = new Set();
|
|
144
146
|
this.aborted = false;
|
|
145
147
|
}
|
|
146
148
|
|
|
147
|
-
abort() {
|
|
149
|
+
async abort() {
|
|
148
150
|
this.aborted = true;
|
|
151
|
+
for (const waiter of this._abortWaiters) {
|
|
152
|
+
waiter();
|
|
153
|
+
}
|
|
154
|
+
this._abortWaiters.clear();
|
|
155
|
+
if (this._activeHandle) {
|
|
156
|
+
const h = this._activeHandle;
|
|
157
|
+
this._activeHandle = null;
|
|
158
|
+
try {
|
|
159
|
+
await h.close();
|
|
160
|
+
} catch {}
|
|
161
|
+
}
|
|
149
162
|
if (this.engine) {
|
|
150
163
|
try {
|
|
151
|
-
this.engine.
|
|
164
|
+
this.engine.reset();
|
|
152
165
|
} catch {}
|
|
153
166
|
}
|
|
154
167
|
}
|
|
@@ -157,7 +170,19 @@ export class SenderPipeline {
|
|
|
157
170
|
if (this.aborted) return;
|
|
158
171
|
while (this.engine.isBackpressured()) {
|
|
159
172
|
if (this.aborted) return;
|
|
160
|
-
await
|
|
173
|
+
await new Promise((resolve) => {
|
|
174
|
+
let done = false;
|
|
175
|
+
const waiter = () => {
|
|
176
|
+
if (!done) {
|
|
177
|
+
done = true;
|
|
178
|
+
this._abortWaiters.delete(waiter);
|
|
179
|
+
resolve();
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
this._abortWaiters.add(waiter);
|
|
183
|
+
this.engine.ready().then(waiter, waiter);
|
|
184
|
+
});
|
|
185
|
+
if (this.aborted) return;
|
|
161
186
|
await new Promise((r) => setImmediate(r));
|
|
162
187
|
}
|
|
163
188
|
if (this.aborted) return;
|
|
@@ -216,6 +241,7 @@ export class SenderPipeline {
|
|
|
216
241
|
|
|
217
242
|
if (file.size > 0) {
|
|
218
243
|
const handle = await fs.open(filePath, 'r');
|
|
244
|
+
this._activeHandle = handle;
|
|
219
245
|
try {
|
|
220
246
|
for (let i = 0; i < totalFileChunks; i++) {
|
|
221
247
|
if (this.aborted) break;
|
|
@@ -282,7 +308,10 @@ export class SenderPipeline {
|
|
|
282
308
|
}
|
|
283
309
|
}
|
|
284
310
|
} finally {
|
|
285
|
-
|
|
311
|
+
this._activeHandle = null;
|
|
312
|
+
try {
|
|
313
|
+
await handle.close();
|
|
314
|
+
} catch {}
|
|
286
315
|
}
|
|
287
316
|
}
|
|
288
317
|
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert';
|
|
3
|
+
import fs from 'node:fs/promises';
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import crypto from 'node:crypto';
|
|
7
|
+
import { serve, send, pull } from '../../src/index.js';
|
|
8
|
+
|
|
9
|
+
async function computeSha256(filePath) {
|
|
10
|
+
const content = await fs.readFile(filePath);
|
|
11
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
test('Integration - multi-channel parallel pull transfer (4 channels) with SHA-256 verification', async () => {
|
|
15
|
+
const testDir = path.resolve('tmp/multichannel-pull-test');
|
|
16
|
+
const serverDir = path.join(testDir, 'server');
|
|
17
|
+
const clientDir = path.join(testDir, 'client');
|
|
18
|
+
|
|
19
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
20
|
+
await fs.mkdir(serverDir, { recursive: true });
|
|
21
|
+
await fs.mkdir(clientDir, { recursive: true });
|
|
22
|
+
|
|
23
|
+
// Generate 16 MB test file
|
|
24
|
+
const testFile = path.join(serverDir, 'parallel-16mb.bin');
|
|
25
|
+
const size = 16 * 1024 * 1024;
|
|
26
|
+
const data = crypto.randomBytes(size);
|
|
27
|
+
await fs.writeFile(testFile, data);
|
|
28
|
+
const sourceSha = await computeSha256(testFile);
|
|
29
|
+
|
|
30
|
+
const port = 8921;
|
|
31
|
+
const server = await serve({ port, destinationDir: serverDir });
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
const metrics = await pull({
|
|
35
|
+
sourceRemotePath: 'parallel-16mb.bin',
|
|
36
|
+
host: '127.0.0.1',
|
|
37
|
+
port,
|
|
38
|
+
channels: 4,
|
|
39
|
+
destinationDir: clientDir,
|
|
40
|
+
showProgress: false
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const pulledFile = path.join(clientDir, 'parallel-16mb.bin');
|
|
44
|
+
assert.ok(existsSync(pulledFile), 'Pulled file must exist');
|
|
45
|
+
assert.strictEqual(await fs.stat(pulledFile).then((s) => s.size), size, 'Size must match');
|
|
46
|
+
|
|
47
|
+
const pulledSha = await computeSha256(pulledFile);
|
|
48
|
+
assert.strictEqual(pulledSha, sourceSha, 'SHA-256 checksum must match source exactly');
|
|
49
|
+
|
|
50
|
+
// Verify atomic rename and state cleanup
|
|
51
|
+
assert.strictEqual(existsSync(`${pulledFile}.teyyare-part`), false, 'Part file must be cleaned up');
|
|
52
|
+
assert.strictEqual(existsSync(`${pulledFile}.teyyare-state`), false, 'State file must be cleaned up');
|
|
53
|
+
|
|
54
|
+
const snap = metrics.snapshot();
|
|
55
|
+
assert.strictEqual(snap.fileBytesWritten, size);
|
|
56
|
+
assert.strictEqual(snap.chunksVerified, 16);
|
|
57
|
+
assert.ok(snap.throughputMBs > 0, 'Throughput must be positive');
|
|
58
|
+
} finally {
|
|
59
|
+
await server.close();
|
|
60
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test('Integration - multi-channel parallel send transfer (4 channels) with SHA-256 verification', async () => {
|
|
65
|
+
const testDir = path.resolve('tmp/multichannel-send-test');
|
|
66
|
+
const sourceDir = path.join(testDir, 'source');
|
|
67
|
+
const destDir = path.join(testDir, 'dest');
|
|
68
|
+
|
|
69
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
70
|
+
await fs.mkdir(sourceDir, { recursive: true });
|
|
71
|
+
await fs.mkdir(destDir, { recursive: true });
|
|
72
|
+
|
|
73
|
+
// Generate 12 MB test file
|
|
74
|
+
const testFile = path.join(sourceDir, 'upload-12mb.bin');
|
|
75
|
+
const size = 12 * 1024 * 1024;
|
|
76
|
+
const data = crypto.randomBytes(size);
|
|
77
|
+
await fs.writeFile(testFile, data);
|
|
78
|
+
const sourceSha = await computeSha256(testFile);
|
|
79
|
+
|
|
80
|
+
const port = 8922;
|
|
81
|
+
const server = await serve({ port, destinationDir: destDir });
|
|
82
|
+
|
|
83
|
+
try {
|
|
84
|
+
const metrics = await send({
|
|
85
|
+
sourcePath: testFile,
|
|
86
|
+
host: '127.0.0.1',
|
|
87
|
+
port,
|
|
88
|
+
channels: 4,
|
|
89
|
+
showProgress: false
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
await server.receiver.waitForCompletion();
|
|
93
|
+
|
|
94
|
+
const receivedFile = path.join(destDir, 'upload-12mb.bin');
|
|
95
|
+
assert.ok(existsSync(receivedFile), 'Received file must exist');
|
|
96
|
+
assert.strictEqual(await fs.stat(receivedFile).then((s) => s.size), size, 'Size must match');
|
|
97
|
+
|
|
98
|
+
const receivedSha = await computeSha256(receivedFile);
|
|
99
|
+
assert.strictEqual(receivedSha, sourceSha, 'SHA-256 checksum must match source');
|
|
100
|
+
|
|
101
|
+
const snap = metrics.snapshot();
|
|
102
|
+
assert.strictEqual(snap.fileBytesRead, size);
|
|
103
|
+
assert.strictEqual(snap.chunksCreated, 12);
|
|
104
|
+
} finally {
|
|
105
|
+
await server.close();
|
|
106
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
107
|
+
}
|
|
108
|
+
});
|