teyyare 0.2.9 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/teyyare.js +24 -1
- package/package.json +7 -7
- package/src/cli/progress.js +47 -8
- package/src/index.js +152 -23
- package/src/metrics/collector.js +1 -0
- package/src/receiver/pipeline.js +31 -5
package/bin/teyyare.js
CHANGED
|
@@ -4,6 +4,28 @@
|
|
|
4
4
|
* Teyyare CLI
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
import os from 'node:os';
|
|
8
|
+
import { spawn } from 'node:child_process';
|
|
9
|
+
|
|
10
|
+
// Option C: Ensure libuv threadpool has sufficient C++ workers for multi-channel I/O.
|
|
11
|
+
// Node.js defaults to only 4 threads. Multi-channel file transfers (writev, read, sync)
|
|
12
|
+
// require at least 16 threads to avoid C++ threadpool contention.
|
|
13
|
+
const RECOMMENDED_POOL_SIZE = Math.min(128, Math.max(16, (os.availableParallelism ? os.availableParallelism() : 8) * 2));
|
|
14
|
+
if (!process.env.TEYYARE_BOOTSTRAPPED && (!process.env.UV_THREADPOOL_SIZE || Number(process.env.UV_THREADPOOL_SIZE) < 16)) {
|
|
15
|
+
process.env.UV_THREADPOOL_SIZE = String(RECOMMENDED_POOL_SIZE);
|
|
16
|
+
process.env.TEYYARE_BOOTSTRAPPED = '1';
|
|
17
|
+
const child = spawn(process.execPath, process.argv.slice(1), {
|
|
18
|
+
stdio: 'inherit',
|
|
19
|
+
env: process.env
|
|
20
|
+
});
|
|
21
|
+
child.on('exit', (code, signal) => {
|
|
22
|
+
if (signal) process.kill(process.pid, signal);
|
|
23
|
+
else process.exit(code ?? 0);
|
|
24
|
+
});
|
|
25
|
+
// Hold parent until child process finishes
|
|
26
|
+
await new Promise(() => {});
|
|
27
|
+
}
|
|
28
|
+
|
|
7
29
|
import path from 'node:path';
|
|
8
30
|
import { serve, send, pull } from '../src/index.js';
|
|
9
31
|
|
|
@@ -203,7 +225,8 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
|
203
225
|
`);
|
|
204
226
|
process.exit(0);
|
|
205
227
|
} else if (command === 'status') {
|
|
206
|
-
|
|
228
|
+
const poolSize = Number(process.env.UV_THREADPOOL_SIZE) || 4;
|
|
229
|
+
console.log(`\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.9 status: ready (libuv threadpool: ${poolSize})`);
|
|
207
230
|
} else {
|
|
208
231
|
console.error(`Unknown command: ${command}`);
|
|
209
232
|
printUsage();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "teyyare",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -8,12 +8,12 @@
|
|
|
8
8
|
"teyyare": "bin/teyyare.js"
|
|
9
9
|
},
|
|
10
10
|
"scripts": {
|
|
11
|
-
"test": "node --test test/**/*.test.js",
|
|
12
|
-
"bench": "node bench/run-all.js",
|
|
13
|
-
"e2e": "node test/e2e/e2e.js",
|
|
14
|
-
"e2e:profile": "softscope test/e2e/e2e.js",
|
|
15
|
-
"e2e:live": "softscope --live test/e2e/e2e.js",
|
|
16
|
-
"e2e:md": "softscope --md test/e2e/e2e.js"
|
|
11
|
+
"test": "UV_THREADPOOL_SIZE=16 node --test test/**/*.test.js",
|
|
12
|
+
"bench": "UV_THREADPOOL_SIZE=16 node bench/run-all.js",
|
|
13
|
+
"e2e": "UV_THREADPOOL_SIZE=16 node test/e2e/e2e.js",
|
|
14
|
+
"e2e:profile": "UV_THREADPOOL_SIZE=16 softscope test/e2e/e2e.js",
|
|
15
|
+
"e2e:live": "UV_THREADPOOL_SIZE=16 softscope --live test/e2e/e2e.js",
|
|
16
|
+
"e2e:md": "UV_THREADPOOL_SIZE=16 softscope --md test/e2e/e2e.js"
|
|
17
17
|
},
|
|
18
18
|
"keywords": [
|
|
19
19
|
"file-transfer",
|
package/src/cli/progress.js
CHANGED
|
@@ -17,6 +17,7 @@ export class ProgressReporter {
|
|
|
17
17
|
|
|
18
18
|
this.fileName = '';
|
|
19
19
|
this.bytesDone = 0;
|
|
20
|
+
this.networkBytesDone = 0;
|
|
20
21
|
this.totalBytes = 0;
|
|
21
22
|
this.chunksDone = 0;
|
|
22
23
|
this.totalChunks = 0;
|
|
@@ -35,12 +36,18 @@ export class ProgressReporter {
|
|
|
35
36
|
this._timer = null;
|
|
36
37
|
this._linesRendered = 0;
|
|
37
38
|
this._speedHistory = [];
|
|
39
|
+
this._netSpeedHistory = [];
|
|
40
|
+
this._emaNetSpeed = 0;
|
|
41
|
+
this._emaSpeed = 0;
|
|
38
42
|
}
|
|
39
43
|
|
|
40
44
|
start() {
|
|
41
45
|
if (!this.enabled) return;
|
|
42
46
|
this.startTime = Date.now();
|
|
43
47
|
this._speedHistory = [{ time: this.startTime, bytes: this.bytesDone }];
|
|
48
|
+
this._netSpeedHistory = [{ time: this.startTime, bytes: this.networkBytesDone || this.bytesDone }];
|
|
49
|
+
this._emaNetSpeed = 0;
|
|
50
|
+
this._emaSpeed = 0;
|
|
44
51
|
this._timer = setInterval(() => this.render(), this.intervalMs);
|
|
45
52
|
}
|
|
46
53
|
|
|
@@ -58,6 +65,7 @@ export class ProgressReporter {
|
|
|
58
65
|
update(state = {}) {
|
|
59
66
|
if (state.fileName !== undefined) this.fileName = state.fileName;
|
|
60
67
|
if (state.bytesDone !== undefined) this.bytesDone = state.bytesDone;
|
|
68
|
+
if (state.networkBytesDone !== undefined) this.networkBytesDone = state.networkBytesDone;
|
|
61
69
|
if (state.totalBytes !== undefined) this.totalBytes = state.totalBytes;
|
|
62
70
|
if (state.chunksDone !== undefined) this.chunksDone = state.chunksDone;
|
|
63
71
|
if (state.totalChunks !== undefined && state.totalChunks > 0) this.totalChunks = state.totalChunks;
|
|
@@ -91,7 +99,7 @@ export class ProgressReporter {
|
|
|
91
99
|
const now = Date.now();
|
|
92
100
|
const elapsedSec = Math.max(0.001, (now - this.startTime) / 1000);
|
|
93
101
|
|
|
94
|
-
//
|
|
102
|
+
// 1. Committed bytes speed history (disk commit rate & upload throughput)
|
|
95
103
|
this._speedHistory.push({ time: now, bytes: this.bytesDone });
|
|
96
104
|
while (this._speedHistory.length > 2 && (now - this._speedHistory[0].time) > 2000) {
|
|
97
105
|
this._speedHistory.shift();
|
|
@@ -101,15 +109,45 @@ export class ProgressReporter {
|
|
|
101
109
|
const windowBytes = Math.max(0, this.bytesDone - oldest.bytes);
|
|
102
110
|
|
|
103
111
|
const avgSpeedMBs = (this.bytesDone / (1024 * 1024)) / elapsedSec;
|
|
104
|
-
const currentSpeedMBs = windowSec >= 0.
|
|
105
|
-
const
|
|
112
|
+
const currentSpeedMBs = windowSec >= 0.3 ? (windowBytes / (1024 * 1024)) / windowSec : avgSpeedMBs;
|
|
113
|
+
const rawSpeed = currentSpeedMBs > 0.05 ? currentSpeedMBs : avgSpeedMBs;
|
|
114
|
+
|
|
115
|
+
// Apply Exponential Moving Average (EMA, alpha = 0.20) to filter micro-pauses and jitter
|
|
116
|
+
if (!this._emaSpeed || this._emaSpeed === 0) {
|
|
117
|
+
this._emaSpeed = rawSpeed;
|
|
118
|
+
} else {
|
|
119
|
+
this._emaSpeed = (0.20 * rawSpeed) + (0.80 * this._emaSpeed);
|
|
120
|
+
}
|
|
121
|
+
const effectiveSpeed = this._emaSpeed;
|
|
122
|
+
|
|
123
|
+
// 2. Real wire network bytes speed history (streaming sockets)
|
|
124
|
+
const netBytes = this.networkBytesDone > 0 ? this.networkBytesDone : this.bytesDone;
|
|
125
|
+
this._netSpeedHistory.push({ time: now, bytes: netBytes });
|
|
126
|
+
while (this._netSpeedHistory.length > 2 && (now - this._netSpeedHistory[0].time) > 2000) {
|
|
127
|
+
this._netSpeedHistory.shift();
|
|
128
|
+
}
|
|
129
|
+
const netOldest = this._netSpeedHistory[0];
|
|
130
|
+
const netWindowSec = Math.max(0.001, (now - netOldest.time) / 1000);
|
|
131
|
+
const netWindowBytes = Math.max(0, netBytes - netOldest.bytes);
|
|
132
|
+
const netAvgSpeedMBs = (netBytes / (1024 * 1024)) / elapsedSec;
|
|
133
|
+
const netInstantSpeedMBs = netWindowSec >= 0.3 ? (netWindowBytes / (1024 * 1024)) / netWindowSec : netAvgSpeedMBs;
|
|
134
|
+
const netRawSpeed = netInstantSpeedMBs > 0.05 ? netInstantSpeedMBs : netAvgSpeedMBs;
|
|
135
|
+
|
|
136
|
+
// Smooth network wire rate with Exponential Moving Average (EMA, alpha = 0.20)
|
|
137
|
+
if (!this._emaNetSpeed || this._emaNetSpeed === 0) {
|
|
138
|
+
this._emaNetSpeed = netRawSpeed;
|
|
139
|
+
} else {
|
|
140
|
+
this._emaNetSpeed = (0.20 * netRawSpeed) + (0.80 * this._emaNetSpeed);
|
|
141
|
+
}
|
|
142
|
+
const networkSpeedMBs = this._emaNetSpeed;
|
|
106
143
|
|
|
107
144
|
const percent = this.totalBytes > 0 ? ((this.bytesDone / this.totalBytes) * 100).toFixed(1) : '0.0';
|
|
108
145
|
|
|
146
|
+
const activeSpeed = (this.role === 'receiver' && networkSpeedMBs > 0) ? networkSpeedMBs : effectiveSpeed;
|
|
109
147
|
let etaStr = '--:--';
|
|
110
|
-
if (
|
|
148
|
+
if (activeSpeed > 0 && this.totalBytes > this.bytesDone) {
|
|
111
149
|
const remainingBytes = this.totalBytes - this.bytesDone;
|
|
112
|
-
const remainingSec = Math.round(remainingBytes / (
|
|
150
|
+
const remainingSec = Math.round(remainingBytes / (activeSpeed * 1024 * 1024));
|
|
113
151
|
const mm = String(Math.floor(remainingSec / 60)).padStart(2, '0');
|
|
114
152
|
const ss = String(remainingSec % 60).padStart(2, '0');
|
|
115
153
|
etaStr = `${mm}:${ss}`;
|
|
@@ -117,7 +155,7 @@ export class ProgressReporter {
|
|
|
117
155
|
|
|
118
156
|
let lines;
|
|
119
157
|
if (this.role === 'receiver') {
|
|
120
|
-
const diskStr =
|
|
158
|
+
const diskStr = `${effectiveSpeed.toFixed(1)} MB/s`;
|
|
121
159
|
lines = [
|
|
122
160
|
'\x1b[1m\x1b[36mTeyyare [Download]\x1b[0m',
|
|
123
161
|
'',
|
|
@@ -126,7 +164,7 @@ export class ProgressReporter {
|
|
|
126
164
|
`${this.formatBytes(this.bytesDone)} / ${this.formatBytes(this.totalBytes)}`,
|
|
127
165
|
`\x1b[32m${percent}%\x1b[0m`,
|
|
128
166
|
'',
|
|
129
|
-
`network speed \x1b[1m${
|
|
167
|
+
`network speed \x1b[1m${networkSpeedMBs.toFixed(1)} MB/s\x1b[0m`,
|
|
130
168
|
`disk speed \x1b[1m${diskStr}\x1b[0m`,
|
|
131
169
|
`channels ${this.channels}`,
|
|
132
170
|
`ETA ${etaStr}`,
|
|
@@ -167,7 +205,8 @@ export class ProgressReporter {
|
|
|
167
205
|
process.stdout.write(lines.join('\n') + '\n');
|
|
168
206
|
this._linesRendered = lines.length;
|
|
169
207
|
} else {
|
|
170
|
-
|
|
208
|
+
const reportSpeed = (this.role === 'receiver' && networkSpeedMBs > 0) ? networkSpeedMBs : effectiveSpeed;
|
|
209
|
+
process.stdout.write(`[Teyyare] ${percent}% | ${reportSpeed.toFixed(1)} MB/s | Chunks: ${this.chunksDone}/${this.totalChunks || this.chunksDone} | ETA: ${etaStr}\n`);
|
|
171
210
|
}
|
|
172
211
|
}
|
|
173
212
|
}
|
package/src/index.js
CHANGED
|
@@ -51,6 +51,10 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
51
51
|
const headerScratch = Buffer.allocUnsafe(HEADER_SIZE);
|
|
52
52
|
|
|
53
53
|
socket.on('data', (chunk) => {
|
|
54
|
+
this.bytesReceived = (this.bytesReceived || 0) + chunk.length;
|
|
55
|
+
if (this.onDataBytes) {
|
|
56
|
+
this.onDataBytes(chunk.length);
|
|
57
|
+
}
|
|
54
58
|
chunks.push(chunk);
|
|
55
59
|
bufferedLength += chunk.length;
|
|
56
60
|
|
|
@@ -180,11 +184,19 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
|
|
|
180
184
|
}
|
|
181
185
|
});
|
|
182
186
|
|
|
187
|
+
socket.on('end', () => {
|
|
188
|
+
socket.destroy();
|
|
189
|
+
});
|
|
190
|
+
|
|
183
191
|
socket.on('error', () => {
|
|
184
192
|
socket.destroy();
|
|
185
193
|
});
|
|
186
194
|
};
|
|
187
195
|
|
|
196
|
+
function isSocketReady(s) {
|
|
197
|
+
return Boolean(s && !s.destroyed && s.writable && !s.readableEnded && !s.writableEnded && s.readyState === 'open');
|
|
198
|
+
}
|
|
199
|
+
|
|
188
200
|
TCPTransport.prototype.connectChannels = async function(peer, channelCount = 4) {
|
|
189
201
|
if (channelCount <= 1) return;
|
|
190
202
|
const addr = this.peerAddresses.get(peer);
|
|
@@ -193,7 +205,7 @@ TCPTransport.prototype.connectChannels = async function(peer, channelCount = 4)
|
|
|
193
205
|
}
|
|
194
206
|
|
|
195
207
|
const pool = this._getPeerPool(peer);
|
|
196
|
-
const activeSockets = pool.filter(
|
|
208
|
+
const activeSockets = pool.filter(isSocketReady);
|
|
197
209
|
const needed = channelCount - activeSockets.length;
|
|
198
210
|
if (needed <= 0) return;
|
|
199
211
|
|
|
@@ -243,30 +255,75 @@ TCPTransport.prototype.connectChannels = async function(peer, channelCount = 4)
|
|
|
243
255
|
await new Promise((r) => setTimeout(r, 20));
|
|
244
256
|
};
|
|
245
257
|
|
|
258
|
+
const origGetOrCreate = TCPTransport.prototype._getOrCreateSocket;
|
|
259
|
+
TCPTransport.prototype._getOrCreateSocket = function(peer) {
|
|
260
|
+
if (!this.started) {
|
|
261
|
+
return Promise.reject(new Error(`[TCPTransport ${this.id}] Cannot connect: transport closed`));
|
|
262
|
+
}
|
|
263
|
+
return origGetOrCreate.call(this, peer).then((socket) => {
|
|
264
|
+
const pool = this._getPeerPool(peer);
|
|
265
|
+
if (!pool.includes(socket)) {
|
|
266
|
+
pool.push(socket);
|
|
267
|
+
}
|
|
268
|
+
return socket;
|
|
269
|
+
});
|
|
270
|
+
};
|
|
271
|
+
|
|
246
272
|
TCPTransport.prototype.sendv = function(peer, buffers) {
|
|
273
|
+
if (!this.started) return;
|
|
247
274
|
if (!this._channelIndex) this._channelIndex = new Map();
|
|
248
275
|
|
|
249
276
|
const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
|
|
277
|
+
if (pool && pool.length > 0) {
|
|
278
|
+
// Purge dead sockets from pool
|
|
279
|
+
for (let i = pool.length - 1; i >= 0; i--) {
|
|
280
|
+
if (!isSocketReady(pool[i])) {
|
|
281
|
+
pool.splice(i, 1);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
250
286
|
if (pool && pool.length > 0) {
|
|
251
287
|
let idx = this._channelIndex.get(peer) || 0;
|
|
252
288
|
const poolLen = pool.length;
|
|
289
|
+
let bestSocket = null;
|
|
290
|
+
let bestWritableLength = Infinity;
|
|
291
|
+
let bestIndex = idx;
|
|
292
|
+
|
|
253
293
|
for (let attempts = 0; attempts < poolLen; attempts++) {
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
s.
|
|
258
|
-
|
|
259
|
-
|
|
294
|
+
const candidateIdx = (idx + attempts) % poolLen;
|
|
295
|
+
const s = pool[candidateIdx];
|
|
296
|
+
if (isSocketReady(s)) {
|
|
297
|
+
const qLen = s.writableLength || 0;
|
|
298
|
+
// Priority 1: Pick idle/drained socket immediately
|
|
299
|
+
if (qLen === 0) {
|
|
300
|
+
bestSocket = s;
|
|
301
|
+
bestIndex = candidateIdx;
|
|
302
|
+
break;
|
|
260
303
|
}
|
|
261
|
-
|
|
262
|
-
|
|
304
|
+
// Priority 2: Track least-buffered socket to prevent head-of-line convoy stalls
|
|
305
|
+
if (qLen < bestWritableLength) {
|
|
306
|
+
bestWritableLength = qLen;
|
|
307
|
+
bestSocket = s;
|
|
308
|
+
bestIndex = candidateIdx;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
if (bestSocket) {
|
|
314
|
+
this._channelIndex.set(peer, (bestIndex + 1) % poolLen);
|
|
315
|
+
bestSocket.cork();
|
|
316
|
+
for (let i = 0; i < buffers.length; i++) {
|
|
317
|
+
bestSocket.write(buffers[i]);
|
|
263
318
|
}
|
|
319
|
+
bestSocket.uncork();
|
|
320
|
+
return;
|
|
264
321
|
}
|
|
265
322
|
}
|
|
266
323
|
|
|
267
324
|
this._getOrCreateSocket(peer)
|
|
268
325
|
.then((socket) => {
|
|
269
|
-
if (
|
|
326
|
+
if (isSocketReady(socket)) {
|
|
270
327
|
socket.cork();
|
|
271
328
|
for (let i = 0; i < buffers.length; i++) {
|
|
272
329
|
socket.write(buffers[i]);
|
|
@@ -278,25 +335,56 @@ TCPTransport.prototype.sendv = function(peer, buffers) {
|
|
|
278
335
|
};
|
|
279
336
|
|
|
280
337
|
TCPTransport.prototype.send = function(peer, buffer) {
|
|
338
|
+
if (!this.started) return;
|
|
281
339
|
if (!this._channelIndex) this._channelIndex = new Map();
|
|
282
340
|
|
|
283
341
|
const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
|
|
342
|
+
if (pool && pool.length > 0) {
|
|
343
|
+
// Purge dead sockets from pool
|
|
344
|
+
for (let i = pool.length - 1; i >= 0; i--) {
|
|
345
|
+
if (!isSocketReady(pool[i])) {
|
|
346
|
+
pool.splice(i, 1);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
284
351
|
if (pool && pool.length > 0) {
|
|
285
352
|
let idx = this._channelIndex.get(peer) || 0;
|
|
286
353
|
const poolLen = pool.length;
|
|
354
|
+
let bestSocket = null;
|
|
355
|
+
let bestWritableLength = Infinity;
|
|
356
|
+
let bestIndex = idx;
|
|
357
|
+
|
|
287
358
|
for (let attempts = 0; attempts < poolLen; attempts++) {
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
s.
|
|
292
|
-
|
|
359
|
+
const candidateIdx = (idx + attempts) % poolLen;
|
|
360
|
+
const s = pool[candidateIdx];
|
|
361
|
+
if (isSocketReady(s)) {
|
|
362
|
+
const qLen = s.writableLength || 0;
|
|
363
|
+
// Priority 1: Pick idle/drained socket immediately
|
|
364
|
+
if (qLen === 0) {
|
|
365
|
+
bestSocket = s;
|
|
366
|
+
bestIndex = candidateIdx;
|
|
367
|
+
break;
|
|
368
|
+
}
|
|
369
|
+
// Priority 2: Track least-buffered socket to prevent head-of-line convoy stalls
|
|
370
|
+
if (qLen < bestWritableLength) {
|
|
371
|
+
bestWritableLength = qLen;
|
|
372
|
+
bestSocket = s;
|
|
373
|
+
bestIndex = candidateIdx;
|
|
374
|
+
}
|
|
293
375
|
}
|
|
294
376
|
}
|
|
377
|
+
|
|
378
|
+
if (bestSocket) {
|
|
379
|
+
this._channelIndex.set(peer, (bestIndex + 1) % poolLen);
|
|
380
|
+
bestSocket.write(buffer);
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
295
383
|
}
|
|
296
384
|
|
|
297
385
|
this._getOrCreateSocket(peer)
|
|
298
386
|
.then((socket) => {
|
|
299
|
-
if (
|
|
387
|
+
if (isSocketReady(socket)) {
|
|
300
388
|
socket.write(buffer);
|
|
301
389
|
}
|
|
302
390
|
})
|
|
@@ -305,6 +393,7 @@ TCPTransport.prototype.send = function(peer, buffer) {
|
|
|
305
393
|
|
|
306
394
|
const origClose = TCPTransport.prototype.close;
|
|
307
395
|
TCPTransport.prototype.close = async function() {
|
|
396
|
+
this.started = false;
|
|
308
397
|
if (this._peerSocketPool) {
|
|
309
398
|
for (const pool of this._peerSocketPool.values()) {
|
|
310
399
|
for (const socket of pool) {
|
|
@@ -503,6 +592,7 @@ export async function serve({
|
|
|
503
592
|
await s.abort();
|
|
504
593
|
}
|
|
505
594
|
replication.reset();
|
|
595
|
+
receiver.reset();
|
|
506
596
|
};
|
|
507
597
|
|
|
508
598
|
replication.onBatch(async (msg) => {
|
|
@@ -551,9 +641,18 @@ export async function serve({
|
|
|
551
641
|
}));
|
|
552
642
|
}
|
|
553
643
|
|
|
644
|
+
// Detect peer channels for optimal pipeline scaling
|
|
645
|
+
const pool = transport._peerSocketPool ? transport._peerSocketPool.get(entries[0].sourceNode || 1) : null;
|
|
646
|
+
const detectedChannels = pool && pool.length > 0 ? pool.length : 1;
|
|
647
|
+
const effectiveBatchBytes = maxBytes || ((chunkSize || 1024 * 1024) * (maxEntries || 8));
|
|
648
|
+
|
|
649
|
+
// Dynamically scale replication in-flight window for remote pull client
|
|
650
|
+
replication.maxInflightBytes = Math.max(128 * 1024 * 1024, detectedChannels * effectiveBatchBytes * 3);
|
|
651
|
+
replication.maxInflightBatches = Math.max(32, detectedChannels * 4);
|
|
652
|
+
|
|
554
653
|
const senderOpts = {
|
|
555
|
-
maxImmutableBuffers: 32,
|
|
556
|
-
maxConcurrentFlushes: 32
|
|
654
|
+
maxImmutableBuffers: Math.max(32, detectedChannels * 4),
|
|
655
|
+
maxConcurrentFlushes: Math.max(32, detectedChannels * 4)
|
|
557
656
|
};
|
|
558
657
|
if (chunkSize) senderOpts.chunkSize = chunkSize;
|
|
559
658
|
if (maxEntries) senderOpts.maxEntries = maxEntries;
|
|
@@ -647,20 +746,46 @@ export async function send({
|
|
|
647
746
|
await transport.connectChannels(2, channels);
|
|
648
747
|
}
|
|
649
748
|
|
|
749
|
+
const isRemote = host !== '127.0.0.1' && host !== 'localhost';
|
|
750
|
+
const defaultChunkSize = isRemote ? 2 * 1024 * 1024 : 1024 * 1024;
|
|
751
|
+
const defaultMaxEntries = 8;
|
|
752
|
+
const defaultMaxBytes = defaultChunkSize * defaultMaxEntries;
|
|
753
|
+
|
|
754
|
+
const effectiveChunkSize = pipelineOptions.chunkSize || defaultChunkSize;
|
|
755
|
+
const effectiveMaxEntries = pipelineOptions.maxEntries || defaultMaxEntries;
|
|
756
|
+
const effectiveMaxBytes = pipelineOptions.maxBytes || defaultMaxBytes;
|
|
757
|
+
|
|
758
|
+
const scaledInflightBytes = pipelineOptions.maxInflightBytes || Math.max(128 * 1024 * 1024, channels * effectiveMaxBytes * 3);
|
|
759
|
+
const scaledInflightBatches = pipelineOptions.maxInflightBatches || Math.max(32, channels * 4);
|
|
760
|
+
|
|
650
761
|
const replication = new ReplicationPipeline({
|
|
651
762
|
transport,
|
|
652
763
|
localNode: 1,
|
|
653
764
|
remoteNode: 2,
|
|
654
|
-
maxInflightBatches:
|
|
655
|
-
maxInflightBytes:
|
|
765
|
+
maxInflightBatches: scaledInflightBatches,
|
|
766
|
+
maxInflightBytes: scaledInflightBytes
|
|
656
767
|
});
|
|
657
768
|
|
|
658
769
|
const manifest = await TransferManifest.build(sourcePath);
|
|
659
770
|
|
|
771
|
+
const effectiveSenderOptions = {
|
|
772
|
+
chunkSize: effectiveChunkSize,
|
|
773
|
+
maxEntries: effectiveMaxEntries,
|
|
774
|
+
maxBytes: effectiveMaxBytes,
|
|
775
|
+
...pipelineOptions
|
|
776
|
+
};
|
|
777
|
+
|
|
778
|
+
if (!effectiveSenderOptions.maxImmutableBuffers && channels > 1) {
|
|
779
|
+
effectiveSenderOptions.maxImmutableBuffers = Math.max(32, channels * 4);
|
|
780
|
+
}
|
|
781
|
+
if (!effectiveSenderOptions.maxConcurrentFlushes && channels > 1) {
|
|
782
|
+
effectiveSenderOptions.maxConcurrentFlushes = effectiveSenderOptions.maxImmutableBuffers;
|
|
783
|
+
}
|
|
784
|
+
|
|
660
785
|
const sender = new SenderPipeline({
|
|
661
786
|
replication,
|
|
662
787
|
manifest,
|
|
663
|
-
options:
|
|
788
|
+
options: effectiveSenderOptions,
|
|
664
789
|
metrics,
|
|
665
790
|
progress,
|
|
666
791
|
knownResumeState
|
|
@@ -669,6 +794,7 @@ export async function send({
|
|
|
669
794
|
try {
|
|
670
795
|
await sender.transfer();
|
|
671
796
|
} finally {
|
|
797
|
+
if (progress) progress.stop();
|
|
672
798
|
await sender.abort();
|
|
673
799
|
metrics.stop();
|
|
674
800
|
await transport.close();
|
|
@@ -727,12 +853,15 @@ export async function pull({
|
|
|
727
853
|
await transport.connectChannels(2, channels);
|
|
728
854
|
}
|
|
729
855
|
|
|
856
|
+
const scaledInflightBytes = pipelineOptions.maxInflightBytes || Math.max(128 * 1024 * 1024, channels * effectiveMaxBytes * 3);
|
|
857
|
+
const scaledInflightBatches = pipelineOptions.maxInflightBatches || Math.max(32, channels * 4);
|
|
858
|
+
|
|
730
859
|
const replication = new ReplicationPipeline({
|
|
731
860
|
transport,
|
|
732
861
|
localNode: 1,
|
|
733
862
|
remoteNode: 2,
|
|
734
|
-
maxInflightBatches:
|
|
735
|
-
maxInflightBytes:
|
|
863
|
+
maxInflightBatches: scaledInflightBatches,
|
|
864
|
+
maxInflightBytes: scaledInflightBytes
|
|
736
865
|
});
|
|
737
866
|
|
|
738
867
|
const receiver = new ReceiverPipeline({
|
package/src/metrics/collector.js
CHANGED
package/src/receiver/pipeline.js
CHANGED
|
@@ -52,6 +52,18 @@ export class ReceiverPipeline {
|
|
|
52
52
|
this._batchQueue = [];
|
|
53
53
|
this._processingQueue = false;
|
|
54
54
|
|
|
55
|
+
this.networkBytesReceived = 0;
|
|
56
|
+
if (this.replication && this.replication.transport) {
|
|
57
|
+
const prevOnDataBytes = this.replication.transport.onDataBytes;
|
|
58
|
+
this.replication.transport.onDataBytes = (bytes) => {
|
|
59
|
+
if (prevOnDataBytes) prevOnDataBytes(bytes);
|
|
60
|
+
this.networkBytesReceived += bytes;
|
|
61
|
+
if (this.progress) {
|
|
62
|
+
this.progress.update({ networkBytesDone: this.networkBytesReceived });
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
55
67
|
// Hook up Raptiye batch handler if autoHookBatch is true
|
|
56
68
|
if (this.replication && autoHookBatch) {
|
|
57
69
|
this.replication.onBatch(async (msg) => {
|
|
@@ -100,6 +112,24 @@ export class ReceiverPipeline {
|
|
|
100
112
|
}
|
|
101
113
|
}
|
|
102
114
|
|
|
115
|
+
reset() {
|
|
116
|
+
this.currentTransferId = null;
|
|
117
|
+
this.transferFinished = false;
|
|
118
|
+
this.transferError = null;
|
|
119
|
+
this._pendingTransferEnd = false;
|
|
120
|
+
this.expectedFileCount = null;
|
|
121
|
+
this._completedFileCount = 0;
|
|
122
|
+
this.completedFiles.clear();
|
|
123
|
+
this.activeFiles.clear();
|
|
124
|
+
this._pendingChunks.clear();
|
|
125
|
+
this._pendingFileEnds.clear();
|
|
126
|
+
this._batchQueue = [];
|
|
127
|
+
this._processingQueue = false;
|
|
128
|
+
this.networkBytesReceived = 0;
|
|
129
|
+
this._finishResolvers = [];
|
|
130
|
+
this._errorRejecters = [];
|
|
131
|
+
}
|
|
132
|
+
|
|
103
133
|
async waitForCompletion() {
|
|
104
134
|
if (this.transferError) throw this.transferError;
|
|
105
135
|
if (this.transferFinished) return;
|
|
@@ -484,9 +514,6 @@ export class ReceiverPipeline {
|
|
|
484
514
|
|
|
485
515
|
// 3. Update progress reporter
|
|
486
516
|
if (this.progress) {
|
|
487
|
-
const diskTimeSec = Math.max(0.001, (this.metrics ? this.metrics.diskWriteTimeMs : 1) / 1000);
|
|
488
|
-
const writtenMB = (this.metrics ? this.metrics.fileBytesWritten : 0) / (1024 * 1024);
|
|
489
|
-
const diskMBs = Number((writtenMB / diskTimeSec).toFixed(1));
|
|
490
517
|
for (const item of toWrite) {
|
|
491
518
|
file.maxChunkSize = Math.max(file.maxChunkSize || 0, item.meta.length);
|
|
492
519
|
}
|
|
@@ -496,8 +523,7 @@ export class ReceiverPipeline {
|
|
|
496
523
|
bytesDone: this.metrics ? this.metrics.fileBytesWritten : 0,
|
|
497
524
|
chunksDone: this.metrics ? this.metrics.chunksVerified : 0,
|
|
498
525
|
totalChunks: realTotalChunks > 0 ? realTotalChunks : undefined,
|
|
499
|
-
resumeHits: this.metrics ? this.metrics.resumeHits : 0
|
|
500
|
-
diskSpeedMBs: diskMBs
|
|
526
|
+
resumeHits: this.metrics ? this.metrics.resumeHits : 0
|
|
501
527
|
});
|
|
502
528
|
}
|
|
503
529
|
|