teyyare 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/teyyare.js +37 -10
- package/package.json +2 -2
- package/src/cli/progress.js +4 -0
- package/src/core/protocol.js +25 -4
- package/src/index.js +218 -12
- package/src/receiver/pipeline.js +79 -12
- package/src/sender/pipeline.js +18 -0
- package/test/integration/multichannel.test.js +108 -0
package/bin/teyyare.js
CHANGED
|
@@ -13,15 +13,20 @@ function printUsage() {
|
|
|
13
13
|
|
|
14
14
|
\x1b[1mUSAGE:\x1b[0m
|
|
15
15
|
teyyare serve [--port 7421] [--dir <path>]
|
|
16
|
-
teyyare send <source> <destination>
|
|
17
|
-
teyyare pull <source> <
|
|
16
|
+
teyyare send <source> <destination> [-c <channels>]
|
|
17
|
+
teyyare pull <source> [destination] [-c <channels>]
|
|
18
18
|
teyyare status
|
|
19
19
|
|
|
20
|
+
\x1b[1mOPTIONS:\x1b[0m
|
|
21
|
+
-c, --channels <n> Parallel TCP connections (default: 4 for remote, 1 for local)
|
|
22
|
+
--port <port> Port number (default: 7421)
|
|
23
|
+
--dir <path> Serve root directory (default: current dir)
|
|
24
|
+
|
|
20
25
|
\x1b[1mEXAMPLES:\x1b[0m
|
|
21
26
|
teyyare serve --port 7421
|
|
22
27
|
teyyare send ./10gb.bin localhost:/tmp/10gb.bin
|
|
23
|
-
teyyare send ./dist 127.0.0.1:7421:/var/www/app
|
|
24
|
-
teyyare pull server:7421:/backup/db.tar.zst ./
|
|
28
|
+
teyyare send ./dist 127.0.0.1:7421:/var/www/app -c 4
|
|
29
|
+
teyyare pull server:7421:/backup/db.tar.zst ./ -c 4
|
|
25
30
|
`);
|
|
26
31
|
}
|
|
27
32
|
|
|
@@ -51,7 +56,7 @@ async function main() {
|
|
|
51
56
|
}
|
|
52
57
|
|
|
53
58
|
if (command === '--version' || command === '-v' || command === 'version') {
|
|
54
|
-
console.log('0.2.
|
|
59
|
+
console.log('0.2.6');
|
|
55
60
|
process.exit(0);
|
|
56
61
|
}
|
|
57
62
|
|
|
@@ -82,8 +87,18 @@ async function main() {
|
|
|
82
87
|
// Keep alive
|
|
83
88
|
await new Promise(() => {});
|
|
84
89
|
} else if (command === 'send') {
|
|
85
|
-
|
|
86
|
-
const
|
|
90
|
+
let channels = undefined;
|
|
91
|
+
const positional = [];
|
|
92
|
+
for (let i = 1; i < args.length; i++) {
|
|
93
|
+
if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
|
|
94
|
+
channels = Number(args[++i]);
|
|
95
|
+
} else {
|
|
96
|
+
positional.push(args[i]);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const source = positional[0];
|
|
101
|
+
const target = positional[1];
|
|
87
102
|
|
|
88
103
|
if (!source || !target) {
|
|
89
104
|
console.error('Error: teyyare send requires <source> and <destination>');
|
|
@@ -96,6 +111,7 @@ async function main() {
|
|
|
96
111
|
sourcePath: source,
|
|
97
112
|
host,
|
|
98
113
|
port,
|
|
114
|
+
channels,
|
|
99
115
|
showProgress: true
|
|
100
116
|
});
|
|
101
117
|
|
|
@@ -112,8 +128,18 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
|
112
128
|
`);
|
|
113
129
|
process.exit(0);
|
|
114
130
|
} else if (command === 'pull') {
|
|
115
|
-
|
|
116
|
-
const
|
|
131
|
+
let channels = undefined;
|
|
132
|
+
const positional = [];
|
|
133
|
+
for (let i = 1; i < args.length; i++) {
|
|
134
|
+
if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
|
|
135
|
+
channels = Number(args[++i]);
|
|
136
|
+
} else {
|
|
137
|
+
positional.push(args[i]);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const source = positional[0];
|
|
142
|
+
const target = positional[1] || './';
|
|
117
143
|
|
|
118
144
|
if (!source) {
|
|
119
145
|
console.error('Error: teyyare pull requires <source> and [destination]');
|
|
@@ -126,6 +152,7 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
|
126
152
|
sourceRemotePath: remotePath,
|
|
127
153
|
host,
|
|
128
154
|
port,
|
|
155
|
+
channels,
|
|
129
156
|
destinationDir: target,
|
|
130
157
|
showProgress: true
|
|
131
158
|
});
|
|
@@ -143,7 +170,7 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
|
143
170
|
`);
|
|
144
171
|
process.exit(0);
|
|
145
172
|
} else if (command === 'status') {
|
|
146
|
-
console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.
|
|
173
|
+
console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.6 status: ready');
|
|
147
174
|
} else {
|
|
148
175
|
console.error(`Unknown command: ${command}`);
|
|
149
176
|
printUsage();
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "teyyare",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.6",
|
|
4
4
|
"description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
8
|
-
"teyyare": "
|
|
8
|
+
"teyyare": "bin/teyyare.js"
|
|
9
9
|
},
|
|
10
10
|
"scripts": {
|
|
11
11
|
"test": "node --test test/**/*.test.js",
|
package/src/cli/progress.js
CHANGED
|
@@ -28,6 +28,7 @@ export class ProgressReporter {
|
|
|
28
28
|
this.tcpPressure = 0;
|
|
29
29
|
this.resumeHits = 0;
|
|
30
30
|
this.diskSpeedMBs = 0;
|
|
31
|
+
this.channels = 1;
|
|
31
32
|
|
|
32
33
|
this.startTime = Date.now();
|
|
33
34
|
this.lastRender = 0;
|
|
@@ -68,6 +69,7 @@ export class ProgressReporter {
|
|
|
68
69
|
if (state.tcpPressure !== undefined) this.tcpPressure = state.tcpPressure;
|
|
69
70
|
if (state.resumeHits !== undefined) this.resumeHits = state.resumeHits;
|
|
70
71
|
if (state.diskSpeedMBs !== undefined) this.diskSpeedMBs = state.diskSpeedMBs;
|
|
72
|
+
if (state.channels !== undefined) this.channels = state.channels;
|
|
71
73
|
}
|
|
72
74
|
|
|
73
75
|
formatBytes(bytes) {
|
|
@@ -126,6 +128,7 @@ export class ProgressReporter {
|
|
|
126
128
|
'',
|
|
127
129
|
`network speed \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
|
|
128
130
|
`disk speed \x1b[1m${diskStr}\x1b[0m`,
|
|
131
|
+
`channels ${this.channels}`,
|
|
129
132
|
`ETA ${etaStr}`,
|
|
130
133
|
'',
|
|
131
134
|
`chunks ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
|
|
@@ -143,6 +146,7 @@ export class ProgressReporter {
|
|
|
143
146
|
`\x1b[32m${percent}%\x1b[0m`,
|
|
144
147
|
'',
|
|
145
148
|
`throughput \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
|
|
149
|
+
`channels ${this.channels}`,
|
|
146
150
|
`ETA ${etaStr}`,
|
|
147
151
|
'',
|
|
148
152
|
`chunks ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
|
package/src/core/protocol.js
CHANGED
|
@@ -165,14 +165,23 @@ export function unpackTransferEnd(buf) {
|
|
|
165
165
|
}
|
|
166
166
|
|
|
167
167
|
/**
|
|
168
|
-
* Pack PULL_REQUEST metadata.
|
|
168
|
+
* Pack PULL_REQUEST metadata with optional verifiedChunks bitmap/list for resume.
|
|
169
169
|
*/
|
|
170
|
-
export function packPullRequest({ remotePath }) {
|
|
170
|
+
export function packPullRequest({ remotePath, verifiedChunks = [] }) {
|
|
171
171
|
const pathBytes = encoder.encode(remotePath);
|
|
172
|
-
const
|
|
172
|
+
const count = verifiedChunks.length;
|
|
173
|
+
const buf = new Uint8Array(2 + pathBytes.byteLength + 4 + count * 4);
|
|
173
174
|
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
174
175
|
view.setUint16(0, pathBytes.byteLength, false);
|
|
175
176
|
buf.set(pathBytes, 2);
|
|
177
|
+
|
|
178
|
+
let off = 2 + pathBytes.byteLength;
|
|
179
|
+
view.setUint32(off, count, false);
|
|
180
|
+
off += 4;
|
|
181
|
+
for (let i = 0; i < count; i++) {
|
|
182
|
+
view.setUint32(off, verifiedChunks[i], false);
|
|
183
|
+
off += 4;
|
|
184
|
+
}
|
|
176
185
|
return buf;
|
|
177
186
|
}
|
|
178
187
|
|
|
@@ -180,7 +189,19 @@ export function unpackPullRequest(buf) {
|
|
|
180
189
|
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
181
190
|
const pathLen = view.getUint16(0, false);
|
|
182
191
|
const remotePath = decoder.decode(buf.subarray(2, 2 + pathLen));
|
|
183
|
-
|
|
192
|
+
|
|
193
|
+
const verifiedChunks = [];
|
|
194
|
+
let off = 2 + pathLen;
|
|
195
|
+
if (buf.byteLength >= off + 4) {
|
|
196
|
+
const count = view.getUint32(off, false);
|
|
197
|
+
off += 4;
|
|
198
|
+
for (let i = 0; i < count && off + 4 <= buf.byteLength; i++) {
|
|
199
|
+
verifiedChunks.push(view.getUint32(off, false));
|
|
200
|
+
off += 4;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
return { remotePath, verifiedChunks };
|
|
184
205
|
}
|
|
185
206
|
|
|
186
207
|
/**
|
package/src/index.js
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import path from 'node:path';
|
|
6
|
+
import net from 'node:net';
|
|
6
7
|
import { existsSync } from 'node:fs';
|
|
7
8
|
import { TCPTransport, ReplicationPipeline, BatchReceipt } from 'raptiye';
|
|
8
|
-
import { readHeader, decodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
|
|
9
|
+
import { readHeader, decodeMessage, encodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
|
|
9
10
|
import { MessageType } from 'raptiye/src/types.js';
|
|
10
11
|
import { crc32Buffers } from './core/crc32.js';
|
|
11
12
|
import { SenderPipeline } from './sender/pipeline.js';
|
|
@@ -25,8 +26,19 @@ import {
|
|
|
25
26
|
unpackPullError
|
|
26
27
|
} from './core/protocol.js';
|
|
27
28
|
|
|
28
|
-
// High-performance streaming TCP framer patch for TCPTransport
|
|
29
|
-
|
|
29
|
+
// High-performance streaming TCP framer & Multi-Channel socket pool patch for TCPTransport
|
|
30
|
+
TCPTransport.prototype._getPeerPool = function(peer) {
|
|
31
|
+
if (!this._peerSocketPool) {
|
|
32
|
+
this._peerSocketPool = new Map();
|
|
33
|
+
}
|
|
34
|
+
let pool = this._peerSocketPool.get(peer);
|
|
35
|
+
if (!pool) {
|
|
36
|
+
pool = [];
|
|
37
|
+
this._peerSocketPool.set(peer, pool);
|
|
38
|
+
}
|
|
39
|
+
return pool;
|
|
40
|
+
};
|
|
41
|
+
|
|
30
42
|
TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
31
43
|
socket.setNoDelay(true);
|
|
32
44
|
if (socket._readableState) socket._readableState.highWaterMark = 4 * 1024 * 1024;
|
|
@@ -102,6 +114,10 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
102
114
|
const msg = decodeMessage(frame, 0, false);
|
|
103
115
|
if (msg.sourceNode) {
|
|
104
116
|
this._outboundSockets.set(msg.sourceNode, socket);
|
|
117
|
+
const pool = this._getPeerPool(msg.sourceNode);
|
|
118
|
+
if (!pool.includes(socket)) {
|
|
119
|
+
pool.push(socket);
|
|
120
|
+
}
|
|
105
121
|
}
|
|
106
122
|
if (this._messageHandler) {
|
|
107
123
|
this._messageHandler(msg.sourceNode, msg);
|
|
@@ -119,6 +135,14 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
119
135
|
this._outboundSockets.delete(peer);
|
|
120
136
|
}
|
|
121
137
|
}
|
|
138
|
+
if (this._peerSocketPool) {
|
|
139
|
+
for (const [peer, pool] of this._peerSocketPool.entries()) {
|
|
140
|
+
const idx = pool.indexOf(socket);
|
|
141
|
+
if (idx !== -1) {
|
|
142
|
+
pool.splice(idx, 1);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
122
146
|
});
|
|
123
147
|
|
|
124
148
|
socket.on('error', () => {
|
|
@@ -126,6 +150,136 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
126
150
|
});
|
|
127
151
|
};
|
|
128
152
|
|
|
153
|
+
TCPTransport.prototype.connectChannels = async function(peer, channelCount = 4) {
|
|
154
|
+
if (channelCount <= 1) return;
|
|
155
|
+
const addr = this.peerAddresses.get(peer);
|
|
156
|
+
if (!addr) {
|
|
157
|
+
throw new Error(`[TCPTransport ${this.id}] No address registered for peer ${peer}`);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const pool = this._getPeerPool(peer);
|
|
161
|
+
const activeSockets = pool.filter((s) => !s.destroyed && s.writable);
|
|
162
|
+
const needed = channelCount - activeSockets.length;
|
|
163
|
+
if (needed <= 0) return;
|
|
164
|
+
|
|
165
|
+
const connectPromises = [];
|
|
166
|
+
for (let c = 0; c < needed; c++) {
|
|
167
|
+
const p = new Promise((resolve, reject) => {
|
|
168
|
+
const socket = net.createConnection({ host: addr.host, port: addr.port }, () => {
|
|
169
|
+
this._attachSocketReader(socket);
|
|
170
|
+
if (!this._outboundSockets.has(peer)) {
|
|
171
|
+
this._outboundSockets.set(peer, socket);
|
|
172
|
+
}
|
|
173
|
+
const currentPool = this._getPeerPool(peer);
|
|
174
|
+
if (!currentPool.includes(socket)) {
|
|
175
|
+
currentPool.push(socket);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Send registration HEARTBEAT frame so remote peer associates this socket with this node
|
|
179
|
+
try {
|
|
180
|
+
const hbChunks = encodeMessage({
|
|
181
|
+
type: MessageType.HEARTBEAT,
|
|
182
|
+
sourceNode: this.id,
|
|
183
|
+
destNode: peer,
|
|
184
|
+
leaderId: this.id,
|
|
185
|
+
leaderCommit: 0n
|
|
186
|
+
});
|
|
187
|
+
socket.cork();
|
|
188
|
+
for (let i = 0; i < hbChunks.length; i++) {
|
|
189
|
+
socket.write(hbChunks[i]);
|
|
190
|
+
}
|
|
191
|
+
socket.uncork();
|
|
192
|
+
} catch (err) {
|
|
193
|
+
// Non-fatal
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
resolve(socket);
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
socket.on('error', (err) => {
|
|
200
|
+
reject(err);
|
|
201
|
+
});
|
|
202
|
+
});
|
|
203
|
+
connectPromises.push(p);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
await Promise.all(connectPromises);
|
|
207
|
+
// Short pause to allow remote peer to process initial registration frames
|
|
208
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
TCPTransport.prototype.sendv = function(peer, buffers) {
|
|
212
|
+
if (!this._channelIndex) this._channelIndex = new Map();
|
|
213
|
+
|
|
214
|
+
const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
|
|
215
|
+
if (pool && pool.length > 0) {
|
|
216
|
+
const active = pool.filter((s) => !s.destroyed && s.writable);
|
|
217
|
+
if (active.length > 0) {
|
|
218
|
+
let idx = this._channelIndex.get(peer) || 0;
|
|
219
|
+
const socket = active[idx % active.length];
|
|
220
|
+
this._channelIndex.set(peer, idx + 1);
|
|
221
|
+
|
|
222
|
+
socket.cork();
|
|
223
|
+
for (let i = 0; i < buffers.length; i++) {
|
|
224
|
+
socket.write(buffers[i]);
|
|
225
|
+
}
|
|
226
|
+
socket.uncork();
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
this._getOrCreateSocket(peer)
|
|
232
|
+
.then((socket) => {
|
|
233
|
+
if (!socket.destroyed && socket.writable) {
|
|
234
|
+
socket.cork();
|
|
235
|
+
for (let i = 0; i < buffers.length; i++) {
|
|
236
|
+
socket.write(buffers[i]);
|
|
237
|
+
}
|
|
238
|
+
socket.uncork();
|
|
239
|
+
}
|
|
240
|
+
})
|
|
241
|
+
.catch(() => {});
|
|
242
|
+
};
|
|
243
|
+
|
|
244
|
+
TCPTransport.prototype.send = function(peer, buffer) {
|
|
245
|
+
if (!this._channelIndex) this._channelIndex = new Map();
|
|
246
|
+
|
|
247
|
+
const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
|
|
248
|
+
if (pool && pool.length > 0) {
|
|
249
|
+
const active = pool.filter((s) => !s.destroyed && s.writable);
|
|
250
|
+
if (active.length > 0) {
|
|
251
|
+
let idx = this._channelIndex.get(peer) || 0;
|
|
252
|
+
const socket = active[idx % active.length];
|
|
253
|
+
this._channelIndex.set(peer, idx + 1);
|
|
254
|
+
socket.write(buffer);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
this._getOrCreateSocket(peer)
|
|
260
|
+
.then((socket) => {
|
|
261
|
+
if (!socket.destroyed && socket.writable) {
|
|
262
|
+
socket.write(buffer);
|
|
263
|
+
}
|
|
264
|
+
})
|
|
265
|
+
.catch(() => {});
|
|
266
|
+
};
|
|
267
|
+
|
|
268
|
+
const origClose = TCPTransport.prototype.close;
|
|
269
|
+
TCPTransport.prototype.close = async function() {
|
|
270
|
+
if (this._peerSocketPool) {
|
|
271
|
+
for (const pool of this._peerSocketPool.values()) {
|
|
272
|
+
for (const socket of pool) {
|
|
273
|
+
if (!socket.destroyed) {
|
|
274
|
+
socket.destroy();
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
this._peerSocketPool.clear();
|
|
279
|
+
}
|
|
280
|
+
return origClose.call(this);
|
|
281
|
+
};
|
|
282
|
+
|
|
129
283
|
// Fast SIMD-accelerated BATCH_DATA encoder
|
|
130
284
|
function fastEncodeBatchData(msg) {
|
|
131
285
|
const bufs = msg.buffers || [];
|
|
@@ -302,12 +456,14 @@ export async function serve({
|
|
|
302
456
|
autoHookBatch: false
|
|
303
457
|
});
|
|
304
458
|
|
|
459
|
+
let activeSender = null;
|
|
460
|
+
|
|
305
461
|
replication.onBatch(async (msg) => {
|
|
306
462
|
if (!msg.buffers || msg.buffers.length < 2) return;
|
|
307
463
|
const [metaBuf, payloadBuf] = msg.buffers;
|
|
308
464
|
const entries = unpackGeneration(metaBuf, payloadBuf);
|
|
309
465
|
if (entries.length > 0 && entries[0].type === Opcodes.PULL_REQUEST) {
|
|
310
|
-
const { remotePath } = unpackPullRequest(entries[0].key);
|
|
466
|
+
const { remotePath, verifiedChunks = [] } = unpackPullRequest(entries[0].key);
|
|
311
467
|
let resolved = path.isAbsolute(remotePath) ? remotePath : path.resolve(destinationDir, remotePath);
|
|
312
468
|
if (!existsSync(resolved)) {
|
|
313
469
|
const alt = path.resolve(destinationDir, remotePath);
|
|
@@ -323,22 +479,47 @@ export async function serve({
|
|
|
323
479
|
return;
|
|
324
480
|
}
|
|
325
481
|
|
|
482
|
+
// Abort any existing running sender before starting new transfer
|
|
483
|
+
if (activeSender) {
|
|
484
|
+
activeSender.abort();
|
|
485
|
+
activeSender = null;
|
|
486
|
+
}
|
|
487
|
+
|
|
326
488
|
const manifest = await TransferManifest.build(resolved);
|
|
327
489
|
replication.reset();
|
|
490
|
+
|
|
491
|
+
// Seed sender with verified chunks sent by pull client for resume
|
|
492
|
+
const knownResumeState = new Map();
|
|
493
|
+
if (verifiedChunks && verifiedChunks.length > 0) {
|
|
494
|
+
knownResumeState.set(1, new FileResumeState({
|
|
495
|
+
fileId: 1,
|
|
496
|
+
verifiedChunks: new Set(verifiedChunks)
|
|
497
|
+
}));
|
|
498
|
+
}
|
|
499
|
+
|
|
328
500
|
const sender = new SenderPipeline({
|
|
329
501
|
replication,
|
|
330
502
|
manifest,
|
|
503
|
+
knownResumeState,
|
|
331
504
|
metrics
|
|
332
505
|
});
|
|
506
|
+
activeSender = sender;
|
|
507
|
+
|
|
333
508
|
setImmediate(async () => {
|
|
334
509
|
try {
|
|
335
510
|
await sender.transfer();
|
|
336
511
|
} catch (err) {
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
512
|
+
if (!sender.aborted) {
|
|
513
|
+
await submitSingleMutation(
|
|
514
|
+
replication,
|
|
515
|
+
Opcodes.PULL_ERROR,
|
|
516
|
+
packPullError({ errorCode: 500, message: err.message })
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
} finally {
|
|
520
|
+
if (activeSender === sender) {
|
|
521
|
+
activeSender = null;
|
|
522
|
+
}
|
|
342
523
|
}
|
|
343
524
|
});
|
|
344
525
|
return;
|
|
@@ -374,12 +555,14 @@ export async function send({
|
|
|
374
555
|
sourcePath,
|
|
375
556
|
host = '127.0.0.1',
|
|
376
557
|
port = 7421,
|
|
558
|
+
channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
|
|
377
559
|
pipelineOptions = {},
|
|
378
560
|
showProgress = true,
|
|
379
561
|
knownResumeState = new Map()
|
|
380
562
|
}) {
|
|
381
563
|
const metrics = new MetricsCollector();
|
|
382
564
|
const progress = showProgress ? new ProgressReporter({ role: 'sender' }) : null;
|
|
565
|
+
if (progress) progress.update({ channels });
|
|
383
566
|
|
|
384
567
|
// Ephemeral sender port
|
|
385
568
|
const senderPort = 10000 + Math.floor(Math.random() * 20000);
|
|
@@ -391,6 +574,9 @@ export async function send({
|
|
|
391
574
|
});
|
|
392
575
|
|
|
393
576
|
await transport.start();
|
|
577
|
+
if (channels > 1) {
|
|
578
|
+
await transport.connectChannels(2, channels);
|
|
579
|
+
}
|
|
394
580
|
|
|
395
581
|
const replication = new ReplicationPipeline({
|
|
396
582
|
transport,
|
|
@@ -427,6 +613,7 @@ export async function send({
|
|
|
427
613
|
* @param {string} options.sourceRemotePath
|
|
428
614
|
* @param {string} [options.host='127.0.0.1']
|
|
429
615
|
* @param {number} [options.port=7421]
|
|
616
|
+
* @param {number} [options.channels]
|
|
430
617
|
* @param {string} [options.destinationDir='./']
|
|
431
618
|
* @param {object} [options.pipelineOptions]
|
|
432
619
|
* @param {boolean} [options.showProgress=true]
|
|
@@ -436,13 +623,17 @@ export async function pull({
|
|
|
436
623
|
sourceRemotePath,
|
|
437
624
|
host = '127.0.0.1',
|
|
438
625
|
port = 7421,
|
|
626
|
+
channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
|
|
439
627
|
destinationDir = './',
|
|
440
628
|
pipelineOptions = {},
|
|
441
629
|
showProgress = true
|
|
442
630
|
}) {
|
|
443
631
|
const metrics = new MetricsCollector();
|
|
444
632
|
const progress = showProgress ? new ProgressReporter({ role: 'receiver' }) : null;
|
|
445
|
-
if (progress)
|
|
633
|
+
if (progress) {
|
|
634
|
+
progress.update({ channels });
|
|
635
|
+
progress.start();
|
|
636
|
+
}
|
|
446
637
|
|
|
447
638
|
const clientPort = 10000 + Math.floor(Math.random() * 20000);
|
|
448
639
|
const transport = new TCPTransport({
|
|
@@ -453,6 +644,9 @@ export async function pull({
|
|
|
453
644
|
});
|
|
454
645
|
|
|
455
646
|
await transport.start();
|
|
647
|
+
if (channels > 1) {
|
|
648
|
+
await transport.connectChannels(2, channels);
|
|
649
|
+
}
|
|
456
650
|
|
|
457
651
|
const replication = new ReplicationPipeline({
|
|
458
652
|
transport,
|
|
@@ -470,11 +664,23 @@ export async function pull({
|
|
|
470
664
|
});
|
|
471
665
|
|
|
472
666
|
try {
|
|
473
|
-
//
|
|
667
|
+
// Check if resume state exists for this file
|
|
668
|
+
const fileName = path.basename(sourceRemotePath);
|
|
669
|
+
const targetFile = path.resolve(destinationDir, fileName);
|
|
670
|
+
const statePath = `${targetFile}.teyyare-state`;
|
|
671
|
+
let verifiedChunks = [];
|
|
672
|
+
try {
|
|
673
|
+
const existingState = await FileResumeState.load(statePath);
|
|
674
|
+
if (existingState && existingState.verifiedChunks) {
|
|
675
|
+
verifiedChunks = Array.from(existingState.verifiedChunks);
|
|
676
|
+
}
|
|
677
|
+
} catch {}
|
|
678
|
+
|
|
679
|
+
// Submit PULL_REQUEST batch with verified chunks for resume
|
|
474
680
|
await submitSingleMutation(
|
|
475
681
|
replication,
|
|
476
682
|
Opcodes.PULL_REQUEST,
|
|
477
|
-
packPullRequest({ remotePath: sourceRemotePath })
|
|
683
|
+
packPullRequest({ remotePath: sourceRemotePath, verifiedChunks })
|
|
478
684
|
);
|
|
479
685
|
|
|
480
686
|
// Wait for receiver pipeline to receive and write all files
|
package/src/receiver/pipeline.js
CHANGED
|
@@ -37,6 +37,8 @@ export class ReceiverPipeline {
|
|
|
37
37
|
|
|
38
38
|
this.activeFiles = new Map(); // fileId -> { destPath, partPath, statePath, fileHandle, resumeState, ... }
|
|
39
39
|
this.completedFiles = new Set();
|
|
40
|
+
this._pendingChunks = new Map(); // fileId -> Array<{ key, val }>
|
|
41
|
+
this._pendingTransferEnd = false;
|
|
40
42
|
this.transferFinished = false;
|
|
41
43
|
this.transferError = null;
|
|
42
44
|
this._finishResolvers = [];
|
|
@@ -135,6 +137,7 @@ export class ReceiverPipeline {
|
|
|
135
137
|
case Opcodes.TRANSFER_BEGIN: {
|
|
136
138
|
this.transferFinished = false;
|
|
137
139
|
this.transferError = null;
|
|
140
|
+
this._pendingTransferEnd = false;
|
|
138
141
|
const tinfo = unpackTransferBegin(key);
|
|
139
142
|
break;
|
|
140
143
|
}
|
|
@@ -155,12 +158,11 @@ export class ReceiverPipeline {
|
|
|
155
158
|
}
|
|
156
159
|
|
|
157
160
|
case Opcodes.TRANSFER_END: {
|
|
158
|
-
this.
|
|
159
|
-
|
|
160
|
-
|
|
161
|
+
if (this.activeFiles.size === 0) {
|
|
162
|
+
this._finishTransfer();
|
|
163
|
+
} else {
|
|
164
|
+
this._pendingTransferEnd = true;
|
|
161
165
|
}
|
|
162
|
-
this._finishResolvers = [];
|
|
163
|
-
this._errorRejecters = [];
|
|
164
166
|
break;
|
|
165
167
|
}
|
|
166
168
|
|
|
@@ -179,6 +181,15 @@ export class ReceiverPipeline {
|
|
|
179
181
|
}
|
|
180
182
|
}
|
|
181
183
|
|
|
184
|
+
_finishTransfer() {
|
|
185
|
+
this.transferFinished = true;
|
|
186
|
+
for (const resolve of this._finishResolvers) {
|
|
187
|
+
resolve();
|
|
188
|
+
}
|
|
189
|
+
this._finishResolvers = [];
|
|
190
|
+
this._errorRejecters = [];
|
|
191
|
+
}
|
|
192
|
+
|
|
182
193
|
async _handleFileBegin(key) {
|
|
183
194
|
const info = unpackFileBegin(key);
|
|
184
195
|
const destPath = path.resolve(this.destinationDir, info.path);
|
|
@@ -200,7 +211,7 @@ export class ReceiverPipeline {
|
|
|
200
211
|
|
|
201
212
|
const fileHandle = await fs.open(partPath, 'a+');
|
|
202
213
|
|
|
203
|
-
|
|
214
|
+
const file = {
|
|
204
215
|
fileId: info.fileId,
|
|
205
216
|
relativePath: info.path,
|
|
206
217
|
destPath,
|
|
@@ -210,27 +221,58 @@ export class ReceiverPipeline {
|
|
|
210
221
|
resumeState,
|
|
211
222
|
fileSize: Number(info.fileSize),
|
|
212
223
|
sha256: info.sha256,
|
|
213
|
-
chunksWrittenSinceSave: 0
|
|
214
|
-
|
|
224
|
+
chunksWrittenSinceSave: 0,
|
|
225
|
+
pendingEnd: false,
|
|
226
|
+
endMeta: null
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
this.activeFiles.set(info.fileId, file);
|
|
215
230
|
|
|
216
231
|
if (this.progress) {
|
|
217
232
|
const fileSize = Number(info.fileSize);
|
|
218
233
|
const estimatedChunks = Math.max(1, Math.ceil(fileSize / (1024 * 1024)));
|
|
234
|
+
const verifiedBytes = resumeState.verifiedCount * (resumeState.chunkSize || 1024 * 1024);
|
|
219
235
|
this.progress.update({
|
|
220
236
|
fileName: info.path,
|
|
221
237
|
totalBytes: fileSize,
|
|
222
|
-
totalChunks: estimatedChunks
|
|
238
|
+
totalChunks: estimatedChunks,
|
|
239
|
+
bytesDone: verifiedBytes,
|
|
240
|
+
chunksDone: resumeState.verifiedCount,
|
|
241
|
+
resumeHits: resumeState.verifiedCount
|
|
223
242
|
});
|
|
224
243
|
}
|
|
244
|
+
|
|
245
|
+
// Flush any pending chunks that arrived out-of-order before FILE_BEGIN
|
|
246
|
+
const pending = this._pendingChunks.get(info.fileId);
|
|
247
|
+
if (pending && pending.length > 0) {
|
|
248
|
+
this._pendingChunks.delete(info.fileId);
|
|
249
|
+
for (const item of pending) {
|
|
250
|
+
const meta = unpackChunkMeta(item.key);
|
|
251
|
+
await this._writeChunk(file, meta, item.val);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
225
254
|
}
|
|
226
255
|
|
|
227
256
|
async _handleChunk(key, val) {
|
|
228
257
|
const meta = unpackChunkMeta(key);
|
|
229
258
|
const file = this.activeFiles.get(meta.fileId);
|
|
230
259
|
if (!file) {
|
|
231
|
-
|
|
260
|
+
if (this.completedFiles.has(meta.fileId)) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
let pending = this._pendingChunks.get(meta.fileId);
|
|
264
|
+
if (!pending) {
|
|
265
|
+
pending = [];
|
|
266
|
+
this._pendingChunks.set(meta.fileId, pending);
|
|
267
|
+
}
|
|
268
|
+
pending.push({ key, val: new Uint8Array(val) });
|
|
269
|
+
return;
|
|
232
270
|
}
|
|
233
271
|
|
|
272
|
+
await this._writeChunk(file, meta, val);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async _writeChunk(file, meta, val) {
|
|
234
276
|
// Idempotency / resume hit check
|
|
235
277
|
if (file.resumeState.isVerified(meta.chunkIndex)) {
|
|
236
278
|
if (this.metrics) {
|
|
@@ -291,6 +333,14 @@ export class ReceiverPipeline {
|
|
|
291
333
|
file.chunksWrittenSinceSave = 0;
|
|
292
334
|
await file.resumeState.save(file.statePath);
|
|
293
335
|
}
|
|
336
|
+
|
|
337
|
+
// If FILE_END arrived earlier while chunks were still arriving
|
|
338
|
+
if (file.pendingEnd && (file.fileSize === 0 || file.resumeState.isComplete())) {
|
|
339
|
+
await this._finalizeFile(file);
|
|
340
|
+
if (this._pendingTransferEnd && this.activeFiles.size === 0) {
|
|
341
|
+
this._finishTransfer();
|
|
342
|
+
}
|
|
343
|
+
}
|
|
294
344
|
}
|
|
295
345
|
|
|
296
346
|
async _handleFileEnd(key) {
|
|
@@ -298,6 +348,22 @@ export class ReceiverPipeline {
|
|
|
298
348
|
const file = this.activeFiles.get(meta.fileId);
|
|
299
349
|
if (!file) return;
|
|
300
350
|
|
|
351
|
+
if (meta.sha256) {
|
|
352
|
+
file.sha256 = meta.sha256;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
if (file.fileSize === 0 || file.resumeState.isComplete()) {
|
|
356
|
+
await this._finalizeFile(file);
|
|
357
|
+
if (this._pendingTransferEnd && this.activeFiles.size === 0) {
|
|
358
|
+
this._finishTransfer();
|
|
359
|
+
}
|
|
360
|
+
} else {
|
|
361
|
+
file.pendingEnd = true;
|
|
362
|
+
file.endMeta = meta;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
async _finalizeFile(file) {
|
|
301
367
|
// Save final state before rename
|
|
302
368
|
await file.resumeState.save(file.statePath);
|
|
303
369
|
|
|
@@ -306,7 +372,7 @@ export class ReceiverPipeline {
|
|
|
306
372
|
await file.fileHandle.close();
|
|
307
373
|
|
|
308
374
|
// Verify whole-file checksum if available
|
|
309
|
-
const expectedSha =
|
|
375
|
+
const expectedSha = file.sha256;
|
|
310
376
|
if (expectedSha) {
|
|
311
377
|
const actualSha = await this._computeFileSha256(file.partPath);
|
|
312
378
|
if (actualSha !== expectedSha) {
|
|
@@ -318,7 +384,8 @@ export class ReceiverPipeline {
|
|
|
318
384
|
await fs.rename(file.partPath, file.destPath);
|
|
319
385
|
await FileResumeState.cleanup(file.statePath);
|
|
320
386
|
|
|
321
|
-
this.activeFiles.delete(
|
|
387
|
+
this.activeFiles.delete(file.fileId);
|
|
388
|
+
this.completedFiles.add(file.fileId);
|
|
322
389
|
this.completedFiles.add(file.destPath);
|
|
323
390
|
}
|
|
324
391
|
|
package/src/sender/pipeline.js
CHANGED
|
@@ -141,13 +141,26 @@ export class SenderPipeline {
|
|
|
141
141
|
}
|
|
142
142
|
}
|
|
143
143
|
});
|
|
144
|
+
this.aborted = false;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
abort() {
|
|
148
|
+
this.aborted = true;
|
|
149
|
+
if (this.engine) {
|
|
150
|
+
try {
|
|
151
|
+
this.engine.halt();
|
|
152
|
+
} catch {}
|
|
153
|
+
}
|
|
144
154
|
}
|
|
145
155
|
|
|
146
156
|
async _append(opcode, key, value = undefined, keyBytes = 0, valueBytes = 0) {
|
|
157
|
+
if (this.aborted) return;
|
|
147
158
|
while (this.engine.isBackpressured()) {
|
|
159
|
+
if (this.aborted) return;
|
|
148
160
|
await this.engine.ready();
|
|
149
161
|
await new Promise((r) => setImmediate(r));
|
|
150
162
|
}
|
|
163
|
+
if (this.aborted) return;
|
|
151
164
|
const kLen = keyBytes || (key ? key.byteLength : 0);
|
|
152
165
|
const vLen = valueBytes || (value ? value.byteLength : 0);
|
|
153
166
|
return this.engine.custom(opcode, key, value, kLen, vLen);
|
|
@@ -205,6 +218,7 @@ export class SenderPipeline {
|
|
|
205
218
|
const handle = await fs.open(filePath, 'r');
|
|
206
219
|
try {
|
|
207
220
|
for (let i = 0; i < totalFileChunks; i++) {
|
|
221
|
+
if (this.aborted) break;
|
|
208
222
|
const offset = BigInt(i * this.chunkSize);
|
|
209
223
|
const remaining = file.size - (i * this.chunkSize);
|
|
210
224
|
const chunkLen = Math.min(this.chunkSize, remaining);
|
|
@@ -272,11 +286,15 @@ export class SenderPipeline {
|
|
|
272
286
|
}
|
|
273
287
|
}
|
|
274
288
|
|
|
289
|
+
if (this.aborted) return;
|
|
290
|
+
|
|
275
291
|
// OP_FILE_END
|
|
276
292
|
const fend = packFileEnd({ fileId: file.fileId, sha256: file.sha256 || '' });
|
|
277
293
|
await this._append(Opcodes.FILE_END, fend);
|
|
278
294
|
}
|
|
279
295
|
|
|
296
|
+
if (this.aborted) return;
|
|
297
|
+
|
|
280
298
|
// 3. OP_TRANSFER_END
|
|
281
299
|
const tend = packTransferEnd({ transferId: this.transferId, status: 0 });
|
|
282
300
|
await this._append(Opcodes.TRANSFER_END, tend);
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert';
|
|
3
|
+
import fs from 'node:fs/promises';
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import crypto from 'node:crypto';
|
|
7
|
+
import { serve, send, pull } from '../../src/index.js';
|
|
8
|
+
|
|
9
|
+
async function computeSha256(filePath) {
|
|
10
|
+
const content = await fs.readFile(filePath);
|
|
11
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
test('Integration - multi-channel parallel pull transfer (4 channels) with SHA-256 verification', async () => {
|
|
15
|
+
const testDir = path.resolve('tmp/multichannel-pull-test');
|
|
16
|
+
const serverDir = path.join(testDir, 'server');
|
|
17
|
+
const clientDir = path.join(testDir, 'client');
|
|
18
|
+
|
|
19
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
20
|
+
await fs.mkdir(serverDir, { recursive: true });
|
|
21
|
+
await fs.mkdir(clientDir, { recursive: true });
|
|
22
|
+
|
|
23
|
+
// Generate 16 MB test file
|
|
24
|
+
const testFile = path.join(serverDir, 'parallel-16mb.bin');
|
|
25
|
+
const size = 16 * 1024 * 1024;
|
|
26
|
+
const data = crypto.randomBytes(size);
|
|
27
|
+
await fs.writeFile(testFile, data);
|
|
28
|
+
const sourceSha = await computeSha256(testFile);
|
|
29
|
+
|
|
30
|
+
const port = 8921;
|
|
31
|
+
const server = await serve({ port, destinationDir: serverDir });
|
|
32
|
+
|
|
33
|
+
try {
|
|
34
|
+
const metrics = await pull({
|
|
35
|
+
sourceRemotePath: 'parallel-16mb.bin',
|
|
36
|
+
host: '127.0.0.1',
|
|
37
|
+
port,
|
|
38
|
+
channels: 4,
|
|
39
|
+
destinationDir: clientDir,
|
|
40
|
+
showProgress: false
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const pulledFile = path.join(clientDir, 'parallel-16mb.bin');
|
|
44
|
+
assert.ok(existsSync(pulledFile), 'Pulled file must exist');
|
|
45
|
+
assert.strictEqual(await fs.stat(pulledFile).then((s) => s.size), size, 'Size must match');
|
|
46
|
+
|
|
47
|
+
const pulledSha = await computeSha256(pulledFile);
|
|
48
|
+
assert.strictEqual(pulledSha, sourceSha, 'SHA-256 checksum must match source exactly');
|
|
49
|
+
|
|
50
|
+
// Verify atomic rename and state cleanup
|
|
51
|
+
assert.strictEqual(existsSync(`${pulledFile}.teyyare-part`), false, 'Part file must be cleaned up');
|
|
52
|
+
assert.strictEqual(existsSync(`${pulledFile}.teyyare-state`), false, 'State file must be cleaned up');
|
|
53
|
+
|
|
54
|
+
const snap = metrics.snapshot();
|
|
55
|
+
assert.strictEqual(snap.fileBytesWritten, size);
|
|
56
|
+
assert.strictEqual(snap.chunksVerified, 16);
|
|
57
|
+
assert.ok(snap.throughputMBs > 0, 'Throughput must be positive');
|
|
58
|
+
} finally {
|
|
59
|
+
await server.close();
|
|
60
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test('Integration - multi-channel parallel send transfer (4 channels) with SHA-256 verification', async () => {
|
|
65
|
+
const testDir = path.resolve('tmp/multichannel-send-test');
|
|
66
|
+
const sourceDir = path.join(testDir, 'source');
|
|
67
|
+
const destDir = path.join(testDir, 'dest');
|
|
68
|
+
|
|
69
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
70
|
+
await fs.mkdir(sourceDir, { recursive: true });
|
|
71
|
+
await fs.mkdir(destDir, { recursive: true });
|
|
72
|
+
|
|
73
|
+
// Generate 12 MB test file
|
|
74
|
+
const testFile = path.join(sourceDir, 'upload-12mb.bin');
|
|
75
|
+
const size = 12 * 1024 * 1024;
|
|
76
|
+
const data = crypto.randomBytes(size);
|
|
77
|
+
await fs.writeFile(testFile, data);
|
|
78
|
+
const sourceSha = await computeSha256(testFile);
|
|
79
|
+
|
|
80
|
+
const port = 8922;
|
|
81
|
+
const server = await serve({ port, destinationDir: destDir });
|
|
82
|
+
|
|
83
|
+
try {
|
|
84
|
+
const metrics = await send({
|
|
85
|
+
sourcePath: testFile,
|
|
86
|
+
host: '127.0.0.1',
|
|
87
|
+
port,
|
|
88
|
+
channels: 4,
|
|
89
|
+
showProgress: false
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
await server.receiver.waitForCompletion();
|
|
93
|
+
|
|
94
|
+
const receivedFile = path.join(destDir, 'upload-12mb.bin');
|
|
95
|
+
assert.ok(existsSync(receivedFile), 'Received file must exist');
|
|
96
|
+
assert.strictEqual(await fs.stat(receivedFile).then((s) => s.size), size, 'Size must match');
|
|
97
|
+
|
|
98
|
+
const receivedSha = await computeSha256(receivedFile);
|
|
99
|
+
assert.strictEqual(receivedSha, sourceSha, 'SHA-256 checksum must match source');
|
|
100
|
+
|
|
101
|
+
const snap = metrics.snapshot();
|
|
102
|
+
assert.strictEqual(snap.fileBytesRead, size);
|
|
103
|
+
assert.strictEqual(snap.chunksCreated, 12);
|
|
104
|
+
} finally {
|
|
105
|
+
await server.close();
|
|
106
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
107
|
+
}
|
|
108
|
+
});
|