teyyare 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/teyyare.js CHANGED
@@ -13,15 +13,20 @@ function printUsage() {
13
13
 
14
14
  \x1b[1mUSAGE:\x1b[0m
15
15
  teyyare serve [--port 7421] [--dir <path>]
16
- teyyare send <source> <destination>
17
- teyyare pull <source> <destination>
16
+ teyyare send <source> <destination> [-c <channels>]
17
+ teyyare pull <source> [destination] [-c <channels>]
18
18
  teyyare status
19
19
 
20
+ \x1b[1mOPTIONS:\x1b[0m
21
+ -c, --channels <n> Parallel TCP connections (default: 4 for remote, 1 for local)
22
+ --port <port> Port number (default: 7421)
23
+ --dir <path> Serve root directory (default: current dir)
24
+
20
25
  \x1b[1mEXAMPLES:\x1b[0m
21
26
  teyyare serve --port 7421
22
27
  teyyare send ./10gb.bin localhost:/tmp/10gb.bin
23
- teyyare send ./dist 127.0.0.1:7421:/var/www/app
24
- teyyare pull server:7421:/backup/db.tar.zst ./
28
+ teyyare send ./dist 127.0.0.1:7421:/var/www/app -c 4
29
+ teyyare pull server:7421:/backup/db.tar.zst ./ -c 4
25
30
  `);
26
31
  }
27
32
 
@@ -51,7 +56,7 @@ async function main() {
51
56
  }
52
57
 
53
58
  if (command === '--version' || command === '-v' || command === 'version') {
54
- console.log('0.2.4');
59
+ console.log('0.2.6');
55
60
  process.exit(0);
56
61
  }
57
62
 
@@ -82,8 +87,18 @@ async function main() {
82
87
  // Keep alive
83
88
  await new Promise(() => {});
84
89
  } else if (command === 'send') {
85
- const source = args[1];
86
- const target = args[2];
90
+ let channels = undefined;
91
+ const positional = [];
92
+ for (let i = 1; i < args.length; i++) {
93
+ if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
94
+ channels = Number(args[++i]);
95
+ } else {
96
+ positional.push(args[i]);
97
+ }
98
+ }
99
+
100
+ const source = positional[0];
101
+ const target = positional[1];
87
102
 
88
103
  if (!source || !target) {
89
104
  console.error('Error: teyyare send requires <source> and <destination>');
@@ -96,6 +111,7 @@ async function main() {
96
111
  sourcePath: source,
97
112
  host,
98
113
  port,
114
+ channels,
99
115
  showProgress: true
100
116
  });
101
117
 
@@ -112,8 +128,18 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
112
128
  `);
113
129
  process.exit(0);
114
130
  } else if (command === 'pull') {
115
- const source = args[1];
116
- const target = args[2] || './';
131
+ let channels = undefined;
132
+ const positional = [];
133
+ for (let i = 1; i < args.length; i++) {
134
+ if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
135
+ channels = Number(args[++i]);
136
+ } else {
137
+ positional.push(args[i]);
138
+ }
139
+ }
140
+
141
+ const source = positional[0];
142
+ const target = positional[1] || './';
117
143
 
118
144
  if (!source) {
119
145
  console.error('Error: teyyare pull requires <source> and [destination]');
@@ -126,6 +152,7 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
126
152
  sourceRemotePath: remotePath,
127
153
  host,
128
154
  port,
155
+ channels,
129
156
  destinationDir: target,
130
157
  showProgress: true
131
158
  });
@@ -143,7 +170,7 @@ Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
143
170
  `);
144
171
  process.exit(0);
145
172
  } else if (command === 'status') {
146
- console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.4 status: ready');
173
+ console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.6 status: ready');
147
174
  } else {
148
175
  console.error(`Unknown command: ${command}`);
149
176
  printUsage();
package/package.json CHANGED
@@ -1,11 +1,11 @@
1
1
  {
2
2
  "name": "teyyare",
3
- "version": "0.2.4",
3
+ "version": "0.2.6",
4
4
  "description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
5
5
  "main": "src/index.js",
6
6
  "type": "module",
7
7
  "bin": {
8
- "teyyare": "./bin/teyyare.js"
8
+ "teyyare": "bin/teyyare.js"
9
9
  },
10
10
  "scripts": {
11
11
  "test": "node --test test/**/*.test.js",
@@ -28,6 +28,7 @@ export class ProgressReporter {
28
28
  this.tcpPressure = 0;
29
29
  this.resumeHits = 0;
30
30
  this.diskSpeedMBs = 0;
31
+ this.channels = 1;
31
32
 
32
33
  this.startTime = Date.now();
33
34
  this.lastRender = 0;
@@ -68,6 +69,7 @@ export class ProgressReporter {
68
69
  if (state.tcpPressure !== undefined) this.tcpPressure = state.tcpPressure;
69
70
  if (state.resumeHits !== undefined) this.resumeHits = state.resumeHits;
70
71
  if (state.diskSpeedMBs !== undefined) this.diskSpeedMBs = state.diskSpeedMBs;
72
+ if (state.channels !== undefined) this.channels = state.channels;
71
73
  }
72
74
 
73
75
  formatBytes(bytes) {
@@ -126,6 +128,7 @@ export class ProgressReporter {
126
128
  '',
127
129
  `network speed \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
128
130
  `disk speed \x1b[1m${diskStr}\x1b[0m`,
131
+ `channels ${this.channels}`,
129
132
  `ETA ${etaStr}`,
130
133
  '',
131
134
  `chunks ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
@@ -143,6 +146,7 @@ export class ProgressReporter {
143
146
  `\x1b[32m${percent}%\x1b[0m`,
144
147
  '',
145
148
  `throughput \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
149
+ `channels ${this.channels}`,
146
150
  `ETA ${etaStr}`,
147
151
  '',
148
152
  `chunks ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
@@ -165,14 +165,23 @@ export function unpackTransferEnd(buf) {
165
165
  }
166
166
 
167
167
  /**
168
- * Pack PULL_REQUEST metadata.
168
+ * Pack PULL_REQUEST metadata with optional verifiedChunks bitmap/list for resume.
169
169
  */
170
- export function packPullRequest({ remotePath }) {
170
+ export function packPullRequest({ remotePath, verifiedChunks = [] }) {
171
171
  const pathBytes = encoder.encode(remotePath);
172
- const buf = new Uint8Array(2 + pathBytes.byteLength);
172
+ const count = verifiedChunks.length;
173
+ const buf = new Uint8Array(2 + pathBytes.byteLength + 4 + count * 4);
173
174
  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
174
175
  view.setUint16(0, pathBytes.byteLength, false);
175
176
  buf.set(pathBytes, 2);
177
+
178
+ let off = 2 + pathBytes.byteLength;
179
+ view.setUint32(off, count, false);
180
+ off += 4;
181
+ for (let i = 0; i < count; i++) {
182
+ view.setUint32(off, verifiedChunks[i], false);
183
+ off += 4;
184
+ }
176
185
  return buf;
177
186
  }
178
187
 
@@ -180,7 +189,19 @@ export function unpackPullRequest(buf) {
180
189
  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
181
190
  const pathLen = view.getUint16(0, false);
182
191
  const remotePath = decoder.decode(buf.subarray(2, 2 + pathLen));
183
- return { remotePath };
192
+
193
+ const verifiedChunks = [];
194
+ let off = 2 + pathLen;
195
+ if (buf.byteLength >= off + 4) {
196
+ const count = view.getUint32(off, false);
197
+ off += 4;
198
+ for (let i = 0; i < count && off + 4 <= buf.byteLength; i++) {
199
+ verifiedChunks.push(view.getUint32(off, false));
200
+ off += 4;
201
+ }
202
+ }
203
+
204
+ return { remotePath, verifiedChunks };
184
205
  }
185
206
 
186
207
  /**
package/src/index.js CHANGED
@@ -3,9 +3,10 @@
3
3
  */
4
4
 
5
5
  import path from 'node:path';
6
+ import net from 'node:net';
6
7
  import { existsSync } from 'node:fs';
7
8
  import { TCPTransport, ReplicationPipeline, BatchReceipt } from 'raptiye';
8
- import { readHeader, decodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
9
+ import { readHeader, decodeMessage, encodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
9
10
  import { MessageType } from 'raptiye/src/types.js';
10
11
  import { crc32Buffers } from './core/crc32.js';
11
12
  import { SenderPipeline } from './sender/pipeline.js';
@@ -25,8 +26,19 @@ import {
25
26
  unpackPullError
26
27
  } from './core/protocol.js';
27
28
 
28
- // High-performance streaming TCP framer patch for TCPTransport
29
- // Eliminates O(N^2) Buffer.concat and ensures clean socket lifecycle management
29
+ // High-performance streaming TCP framer & Multi-Channel socket pool patch for TCPTransport
30
+ TCPTransport.prototype._getPeerPool = function(peer) {
31
+ if (!this._peerSocketPool) {
32
+ this._peerSocketPool = new Map();
33
+ }
34
+ let pool = this._peerSocketPool.get(peer);
35
+ if (!pool) {
36
+ pool = [];
37
+ this._peerSocketPool.set(peer, pool);
38
+ }
39
+ return pool;
40
+ };
41
+
30
42
  TCPTransport.prototype._attachSocketReader = function(socket) {
31
43
  socket.setNoDelay(true);
32
44
  if (socket._readableState) socket._readableState.highWaterMark = 4 * 1024 * 1024;
@@ -102,6 +114,10 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
102
114
  const msg = decodeMessage(frame, 0, false);
103
115
  if (msg.sourceNode) {
104
116
  this._outboundSockets.set(msg.sourceNode, socket);
117
+ const pool = this._getPeerPool(msg.sourceNode);
118
+ if (!pool.includes(socket)) {
119
+ pool.push(socket);
120
+ }
105
121
  }
106
122
  if (this._messageHandler) {
107
123
  this._messageHandler(msg.sourceNode, msg);
@@ -119,6 +135,14 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
119
135
  this._outboundSockets.delete(peer);
120
136
  }
121
137
  }
138
+ if (this._peerSocketPool) {
139
+ for (const [peer, pool] of this._peerSocketPool.entries()) {
140
+ const idx = pool.indexOf(socket);
141
+ if (idx !== -1) {
142
+ pool.splice(idx, 1);
143
+ }
144
+ }
145
+ }
122
146
  });
123
147
 
124
148
  socket.on('error', () => {
@@ -126,6 +150,136 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
126
150
  });
127
151
  };
128
152
 
153
+ TCPTransport.prototype.connectChannels = async function(peer, channelCount = 4) {
154
+ if (channelCount <= 1) return;
155
+ const addr = this.peerAddresses.get(peer);
156
+ if (!addr) {
157
+ throw new Error(`[TCPTransport ${this.id}] No address registered for peer ${peer}`);
158
+ }
159
+
160
+ const pool = this._getPeerPool(peer);
161
+ const activeSockets = pool.filter((s) => !s.destroyed && s.writable);
162
+ const needed = channelCount - activeSockets.length;
163
+ if (needed <= 0) return;
164
+
165
+ const connectPromises = [];
166
+ for (let c = 0; c < needed; c++) {
167
+ const p = new Promise((resolve, reject) => {
168
+ const socket = net.createConnection({ host: addr.host, port: addr.port }, () => {
169
+ this._attachSocketReader(socket);
170
+ if (!this._outboundSockets.has(peer)) {
171
+ this._outboundSockets.set(peer, socket);
172
+ }
173
+ const currentPool = this._getPeerPool(peer);
174
+ if (!currentPool.includes(socket)) {
175
+ currentPool.push(socket);
176
+ }
177
+
178
+ // Send registration HEARTBEAT frame so remote peer associates this socket with this node
179
+ try {
180
+ const hbChunks = encodeMessage({
181
+ type: MessageType.HEARTBEAT,
182
+ sourceNode: this.id,
183
+ destNode: peer,
184
+ leaderId: this.id,
185
+ leaderCommit: 0n
186
+ });
187
+ socket.cork();
188
+ for (let i = 0; i < hbChunks.length; i++) {
189
+ socket.write(hbChunks[i]);
190
+ }
191
+ socket.uncork();
192
+ } catch (err) {
193
+ // Non-fatal
194
+ }
195
+
196
+ resolve(socket);
197
+ });
198
+
199
+ socket.on('error', (err) => {
200
+ reject(err);
201
+ });
202
+ });
203
+ connectPromises.push(p);
204
+ }
205
+
206
+ await Promise.all(connectPromises);
207
+ // Short pause to allow remote peer to process initial registration frames
208
+ await new Promise((r) => setTimeout(r, 20));
209
+ };
210
+
211
+ TCPTransport.prototype.sendv = function(peer, buffers) {
212
+ if (!this._channelIndex) this._channelIndex = new Map();
213
+
214
+ const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
215
+ if (pool && pool.length > 0) {
216
+ const active = pool.filter((s) => !s.destroyed && s.writable);
217
+ if (active.length > 0) {
218
+ let idx = this._channelIndex.get(peer) || 0;
219
+ const socket = active[idx % active.length];
220
+ this._channelIndex.set(peer, idx + 1);
221
+
222
+ socket.cork();
223
+ for (let i = 0; i < buffers.length; i++) {
224
+ socket.write(buffers[i]);
225
+ }
226
+ socket.uncork();
227
+ return;
228
+ }
229
+ }
230
+
231
+ this._getOrCreateSocket(peer)
232
+ .then((socket) => {
233
+ if (!socket.destroyed && socket.writable) {
234
+ socket.cork();
235
+ for (let i = 0; i < buffers.length; i++) {
236
+ socket.write(buffers[i]);
237
+ }
238
+ socket.uncork();
239
+ }
240
+ })
241
+ .catch(() => {});
242
+ };
243
+
244
+ TCPTransport.prototype.send = function(peer, buffer) {
245
+ if (!this._channelIndex) this._channelIndex = new Map();
246
+
247
+ const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
248
+ if (pool && pool.length > 0) {
249
+ const active = pool.filter((s) => !s.destroyed && s.writable);
250
+ if (active.length > 0) {
251
+ let idx = this._channelIndex.get(peer) || 0;
252
+ const socket = active[idx % active.length];
253
+ this._channelIndex.set(peer, idx + 1);
254
+ socket.write(buffer);
255
+ return;
256
+ }
257
+ }
258
+
259
+ this._getOrCreateSocket(peer)
260
+ .then((socket) => {
261
+ if (!socket.destroyed && socket.writable) {
262
+ socket.write(buffer);
263
+ }
264
+ })
265
+ .catch(() => {});
266
+ };
267
+
268
+ const origClose = TCPTransport.prototype.close;
269
+ TCPTransport.prototype.close = async function() {
270
+ if (this._peerSocketPool) {
271
+ for (const pool of this._peerSocketPool.values()) {
272
+ for (const socket of pool) {
273
+ if (!socket.destroyed) {
274
+ socket.destroy();
275
+ }
276
+ }
277
+ }
278
+ this._peerSocketPool.clear();
279
+ }
280
+ return origClose.call(this);
281
+ };
282
+
129
283
  // Fast SIMD-accelerated BATCH_DATA encoder
130
284
  function fastEncodeBatchData(msg) {
131
285
  const bufs = msg.buffers || [];
@@ -302,12 +456,14 @@ export async function serve({
302
456
  autoHookBatch: false
303
457
  });
304
458
 
459
+ let activeSender = null;
460
+
305
461
  replication.onBatch(async (msg) => {
306
462
  if (!msg.buffers || msg.buffers.length < 2) return;
307
463
  const [metaBuf, payloadBuf] = msg.buffers;
308
464
  const entries = unpackGeneration(metaBuf, payloadBuf);
309
465
  if (entries.length > 0 && entries[0].type === Opcodes.PULL_REQUEST) {
310
- const { remotePath } = unpackPullRequest(entries[0].key);
466
+ const { remotePath, verifiedChunks = [] } = unpackPullRequest(entries[0].key);
311
467
  let resolved = path.isAbsolute(remotePath) ? remotePath : path.resolve(destinationDir, remotePath);
312
468
  if (!existsSync(resolved)) {
313
469
  const alt = path.resolve(destinationDir, remotePath);
@@ -323,22 +479,47 @@ export async function serve({
323
479
  return;
324
480
  }
325
481
 
482
+ // Abort any existing running sender before starting new transfer
483
+ if (activeSender) {
484
+ activeSender.abort();
485
+ activeSender = null;
486
+ }
487
+
326
488
  const manifest = await TransferManifest.build(resolved);
327
489
  replication.reset();
490
+
491
+ // Seed sender with verified chunks sent by pull client for resume
492
+ const knownResumeState = new Map();
493
+ if (verifiedChunks && verifiedChunks.length > 0) {
494
+ knownResumeState.set(1, new FileResumeState({
495
+ fileId: 1,
496
+ verifiedChunks: new Set(verifiedChunks)
497
+ }));
498
+ }
499
+
328
500
  const sender = new SenderPipeline({
329
501
  replication,
330
502
  manifest,
503
+ knownResumeState,
331
504
  metrics
332
505
  });
506
+ activeSender = sender;
507
+
333
508
  setImmediate(async () => {
334
509
  try {
335
510
  await sender.transfer();
336
511
  } catch (err) {
337
- await submitSingleMutation(
338
- replication,
339
- Opcodes.PULL_ERROR,
340
- packPullError({ errorCode: 500, message: err.message })
341
- );
512
+ if (!sender.aborted) {
513
+ await submitSingleMutation(
514
+ replication,
515
+ Opcodes.PULL_ERROR,
516
+ packPullError({ errorCode: 500, message: err.message })
517
+ );
518
+ }
519
+ } finally {
520
+ if (activeSender === sender) {
521
+ activeSender = null;
522
+ }
342
523
  }
343
524
  });
344
525
  return;
@@ -374,12 +555,14 @@ export async function send({
374
555
  sourcePath,
375
556
  host = '127.0.0.1',
376
557
  port = 7421,
558
+ channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
377
559
  pipelineOptions = {},
378
560
  showProgress = true,
379
561
  knownResumeState = new Map()
380
562
  }) {
381
563
  const metrics = new MetricsCollector();
382
564
  const progress = showProgress ? new ProgressReporter({ role: 'sender' }) : null;
565
+ if (progress) progress.update({ channels });
383
566
 
384
567
  // Ephemeral sender port
385
568
  const senderPort = 10000 + Math.floor(Math.random() * 20000);
@@ -391,6 +574,9 @@ export async function send({
391
574
  });
392
575
 
393
576
  await transport.start();
577
+ if (channels > 1) {
578
+ await transport.connectChannels(2, channels);
579
+ }
394
580
 
395
581
  const replication = new ReplicationPipeline({
396
582
  transport,
@@ -427,6 +613,7 @@ export async function send({
427
613
  * @param {string} options.sourceRemotePath
428
614
  * @param {string} [options.host='127.0.0.1']
429
615
  * @param {number} [options.port=7421]
616
+ * @param {number} [options.channels]
430
617
  * @param {string} [options.destinationDir='./']
431
618
  * @param {object} [options.pipelineOptions]
432
619
  * @param {boolean} [options.showProgress=true]
@@ -436,13 +623,17 @@ export async function pull({
436
623
  sourceRemotePath,
437
624
  host = '127.0.0.1',
438
625
  port = 7421,
626
+ channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
439
627
  destinationDir = './',
440
628
  pipelineOptions = {},
441
629
  showProgress = true
442
630
  }) {
443
631
  const metrics = new MetricsCollector();
444
632
  const progress = showProgress ? new ProgressReporter({ role: 'receiver' }) : null;
445
- if (progress) progress.start();
633
+ if (progress) {
634
+ progress.update({ channels });
635
+ progress.start();
636
+ }
446
637
 
447
638
  const clientPort = 10000 + Math.floor(Math.random() * 20000);
448
639
  const transport = new TCPTransport({
@@ -453,6 +644,9 @@ export async function pull({
453
644
  });
454
645
 
455
646
  await transport.start();
647
+ if (channels > 1) {
648
+ await transport.connectChannels(2, channels);
649
+ }
456
650
 
457
651
  const replication = new ReplicationPipeline({
458
652
  transport,
@@ -470,11 +664,23 @@ export async function pull({
470
664
  });
471
665
 
472
666
  try {
473
- // Submit PULL_REQUEST batch
667
+ // Check if resume state exists for this file
668
+ const fileName = path.basename(sourceRemotePath);
669
+ const targetFile = path.resolve(destinationDir, fileName);
670
+ const statePath = `${targetFile}.teyyare-state`;
671
+ let verifiedChunks = [];
672
+ try {
673
+ const existingState = await FileResumeState.load(statePath);
674
+ if (existingState && existingState.verifiedChunks) {
675
+ verifiedChunks = Array.from(existingState.verifiedChunks);
676
+ }
677
+ } catch {}
678
+
679
+ // Submit PULL_REQUEST batch with verified chunks for resume
474
680
  await submitSingleMutation(
475
681
  replication,
476
682
  Opcodes.PULL_REQUEST,
477
- packPullRequest({ remotePath: sourceRemotePath })
683
+ packPullRequest({ remotePath: sourceRemotePath, verifiedChunks })
478
684
  );
479
685
 
480
686
  // Wait for receiver pipeline to receive and write all files
@@ -37,6 +37,8 @@ export class ReceiverPipeline {
37
37
 
38
38
  this.activeFiles = new Map(); // fileId -> { destPath, partPath, statePath, fileHandle, resumeState, ... }
39
39
  this.completedFiles = new Set();
40
+ this._pendingChunks = new Map(); // fileId -> Array<{ key, val }>
41
+ this._pendingTransferEnd = false;
40
42
  this.transferFinished = false;
41
43
  this.transferError = null;
42
44
  this._finishResolvers = [];
@@ -135,6 +137,7 @@ export class ReceiverPipeline {
135
137
  case Opcodes.TRANSFER_BEGIN: {
136
138
  this.transferFinished = false;
137
139
  this.transferError = null;
140
+ this._pendingTransferEnd = false;
138
141
  const tinfo = unpackTransferBegin(key);
139
142
  break;
140
143
  }
@@ -155,12 +158,11 @@ export class ReceiverPipeline {
155
158
  }
156
159
 
157
160
  case Opcodes.TRANSFER_END: {
158
- this.transferFinished = true;
159
- for (const resolve of this._finishResolvers) {
160
- resolve();
161
+ if (this.activeFiles.size === 0) {
162
+ this._finishTransfer();
163
+ } else {
164
+ this._pendingTransferEnd = true;
161
165
  }
162
- this._finishResolvers = [];
163
- this._errorRejecters = [];
164
166
  break;
165
167
  }
166
168
 
@@ -179,6 +181,15 @@ export class ReceiverPipeline {
179
181
  }
180
182
  }
181
183
 
184
+ _finishTransfer() {
185
+ this.transferFinished = true;
186
+ for (const resolve of this._finishResolvers) {
187
+ resolve();
188
+ }
189
+ this._finishResolvers = [];
190
+ this._errorRejecters = [];
191
+ }
192
+
182
193
  async _handleFileBegin(key) {
183
194
  const info = unpackFileBegin(key);
184
195
  const destPath = path.resolve(this.destinationDir, info.path);
@@ -200,7 +211,7 @@ export class ReceiverPipeline {
200
211
 
201
212
  const fileHandle = await fs.open(partPath, 'a+');
202
213
 
203
- this.activeFiles.set(info.fileId, {
214
+ const file = {
204
215
  fileId: info.fileId,
205
216
  relativePath: info.path,
206
217
  destPath,
@@ -210,27 +221,58 @@ export class ReceiverPipeline {
210
221
  resumeState,
211
222
  fileSize: Number(info.fileSize),
212
223
  sha256: info.sha256,
213
- chunksWrittenSinceSave: 0
214
- });
224
+ chunksWrittenSinceSave: 0,
225
+ pendingEnd: false,
226
+ endMeta: null
227
+ };
228
+
229
+ this.activeFiles.set(info.fileId, file);
215
230
 
216
231
  if (this.progress) {
217
232
  const fileSize = Number(info.fileSize);
218
233
  const estimatedChunks = Math.max(1, Math.ceil(fileSize / (1024 * 1024)));
234
+ const verifiedBytes = resumeState.verifiedCount * (resumeState.chunkSize || 1024 * 1024);
219
235
  this.progress.update({
220
236
  fileName: info.path,
221
237
  totalBytes: fileSize,
222
- totalChunks: estimatedChunks
238
+ totalChunks: estimatedChunks,
239
+ bytesDone: verifiedBytes,
240
+ chunksDone: resumeState.verifiedCount,
241
+ resumeHits: resumeState.verifiedCount
223
242
  });
224
243
  }
244
+
245
+ // Flush any pending chunks that arrived out-of-order before FILE_BEGIN
246
+ const pending = this._pendingChunks.get(info.fileId);
247
+ if (pending && pending.length > 0) {
248
+ this._pendingChunks.delete(info.fileId);
249
+ for (const item of pending) {
250
+ const meta = unpackChunkMeta(item.key);
251
+ await this._writeChunk(file, meta, item.val);
252
+ }
253
+ }
225
254
  }
226
255
 
227
256
  async _handleChunk(key, val) {
228
257
  const meta = unpackChunkMeta(key);
229
258
  const file = this.activeFiles.get(meta.fileId);
230
259
  if (!file) {
231
- throw new Error(`Unknown fileId ${meta.fileId} in CHUNK mutation`);
260
+ if (this.completedFiles.has(meta.fileId)) {
261
+ return;
262
+ }
263
+ let pending = this._pendingChunks.get(meta.fileId);
264
+ if (!pending) {
265
+ pending = [];
266
+ this._pendingChunks.set(meta.fileId, pending);
267
+ }
268
+ pending.push({ key, val: new Uint8Array(val) });
269
+ return;
232
270
  }
233
271
 
272
+ await this._writeChunk(file, meta, val);
273
+ }
274
+
275
+ async _writeChunk(file, meta, val) {
234
276
  // Idempotency / resume hit check
235
277
  if (file.resumeState.isVerified(meta.chunkIndex)) {
236
278
  if (this.metrics) {
@@ -291,6 +333,14 @@ export class ReceiverPipeline {
291
333
  file.chunksWrittenSinceSave = 0;
292
334
  await file.resumeState.save(file.statePath);
293
335
  }
336
+
337
+ // If FILE_END arrived earlier while chunks were still arriving
338
+ if (file.pendingEnd && (file.fileSize === 0 || file.resumeState.isComplete())) {
339
+ await this._finalizeFile(file);
340
+ if (this._pendingTransferEnd && this.activeFiles.size === 0) {
341
+ this._finishTransfer();
342
+ }
343
+ }
294
344
  }
295
345
 
296
346
  async _handleFileEnd(key) {
@@ -298,6 +348,22 @@ export class ReceiverPipeline {
298
348
  const file = this.activeFiles.get(meta.fileId);
299
349
  if (!file) return;
300
350
 
351
+ if (meta.sha256) {
352
+ file.sha256 = meta.sha256;
353
+ }
354
+
355
+ if (file.fileSize === 0 || file.resumeState.isComplete()) {
356
+ await this._finalizeFile(file);
357
+ if (this._pendingTransferEnd && this.activeFiles.size === 0) {
358
+ this._finishTransfer();
359
+ }
360
+ } else {
361
+ file.pendingEnd = true;
362
+ file.endMeta = meta;
363
+ }
364
+ }
365
+
366
+ async _finalizeFile(file) {
301
367
  // Save final state before rename
302
368
  await file.resumeState.save(file.statePath);
303
369
 
@@ -306,7 +372,7 @@ export class ReceiverPipeline {
306
372
  await file.fileHandle.close();
307
373
 
308
374
  // Verify whole-file checksum if available
309
- const expectedSha = meta.sha256 || file.sha256;
375
+ const expectedSha = file.sha256;
310
376
  if (expectedSha) {
311
377
  const actualSha = await this._computeFileSha256(file.partPath);
312
378
  if (actualSha !== expectedSha) {
@@ -318,7 +384,8 @@ export class ReceiverPipeline {
318
384
  await fs.rename(file.partPath, file.destPath);
319
385
  await FileResumeState.cleanup(file.statePath);
320
386
 
321
- this.activeFiles.delete(meta.fileId);
387
+ this.activeFiles.delete(file.fileId);
388
+ this.completedFiles.add(file.fileId);
322
389
  this.completedFiles.add(file.destPath);
323
390
  }
324
391
 
@@ -141,13 +141,26 @@ export class SenderPipeline {
141
141
  }
142
142
  }
143
143
  });
144
+ this.aborted = false;
145
+ }
146
+
147
+ abort() {
148
+ this.aborted = true;
149
+ if (this.engine) {
150
+ try {
151
+ this.engine.halt();
152
+ } catch {}
153
+ }
144
154
  }
145
155
 
146
156
  async _append(opcode, key, value = undefined, keyBytes = 0, valueBytes = 0) {
157
+ if (this.aborted) return;
147
158
  while (this.engine.isBackpressured()) {
159
+ if (this.aborted) return;
148
160
  await this.engine.ready();
149
161
  await new Promise((r) => setImmediate(r));
150
162
  }
163
+ if (this.aborted) return;
151
164
  const kLen = keyBytes || (key ? key.byteLength : 0);
152
165
  const vLen = valueBytes || (value ? value.byteLength : 0);
153
166
  return this.engine.custom(opcode, key, value, kLen, vLen);
@@ -205,6 +218,7 @@ export class SenderPipeline {
205
218
  const handle = await fs.open(filePath, 'r');
206
219
  try {
207
220
  for (let i = 0; i < totalFileChunks; i++) {
221
+ if (this.aborted) break;
208
222
  const offset = BigInt(i * this.chunkSize);
209
223
  const remaining = file.size - (i * this.chunkSize);
210
224
  const chunkLen = Math.min(this.chunkSize, remaining);
@@ -272,11 +286,15 @@ export class SenderPipeline {
272
286
  }
273
287
  }
274
288
 
289
+ if (this.aborted) return;
290
+
275
291
  // OP_FILE_END
276
292
  const fend = packFileEnd({ fileId: file.fileId, sha256: file.sha256 || '' });
277
293
  await this._append(Opcodes.FILE_END, fend);
278
294
  }
279
295
 
296
+ if (this.aborted) return;
297
+
280
298
  // 3. OP_TRANSFER_END
281
299
  const tend = packTransferEnd({ transferId: this.transferId, status: 0 });
282
300
  await this._append(Opcodes.TRANSFER_END, tend);
@@ -0,0 +1,108 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert';
3
+ import fs from 'node:fs/promises';
4
+ import { existsSync } from 'node:fs';
5
+ import path from 'node:path';
6
+ import crypto from 'node:crypto';
7
+ import { serve, send, pull } from '../../src/index.js';
8
+
9
+ async function computeSha256(filePath) {
10
+ const content = await fs.readFile(filePath);
11
+ return crypto.createHash('sha256').update(content).digest('hex');
12
+ }
13
+
14
+ test('Integration - multi-channel parallel pull transfer (4 channels) with SHA-256 verification', async () => {
15
+ const testDir = path.resolve('tmp/multichannel-pull-test');
16
+ const serverDir = path.join(testDir, 'server');
17
+ const clientDir = path.join(testDir, 'client');
18
+
19
+ await fs.rm(testDir, { recursive: true, force: true });
20
+ await fs.mkdir(serverDir, { recursive: true });
21
+ await fs.mkdir(clientDir, { recursive: true });
22
+
23
+ // Generate 16 MB test file
24
+ const testFile = path.join(serverDir, 'parallel-16mb.bin');
25
+ const size = 16 * 1024 * 1024;
26
+ const data = crypto.randomBytes(size);
27
+ await fs.writeFile(testFile, data);
28
+ const sourceSha = await computeSha256(testFile);
29
+
30
+ const port = 8921;
31
+ const server = await serve({ port, destinationDir: serverDir });
32
+
33
+ try {
34
+ const metrics = await pull({
35
+ sourceRemotePath: 'parallel-16mb.bin',
36
+ host: '127.0.0.1',
37
+ port,
38
+ channels: 4,
39
+ destinationDir: clientDir,
40
+ showProgress: false
41
+ });
42
+
43
+ const pulledFile = path.join(clientDir, 'parallel-16mb.bin');
44
+ assert.ok(existsSync(pulledFile), 'Pulled file must exist');
45
+ assert.strictEqual(await fs.stat(pulledFile).then((s) => s.size), size, 'Size must match');
46
+
47
+ const pulledSha = await computeSha256(pulledFile);
48
+ assert.strictEqual(pulledSha, sourceSha, 'SHA-256 checksum must match source exactly');
49
+
50
+ // Verify atomic rename and state cleanup
51
+ assert.strictEqual(existsSync(`${pulledFile}.teyyare-part`), false, 'Part file must be cleaned up');
52
+ assert.strictEqual(existsSync(`${pulledFile}.teyyare-state`), false, 'State file must be cleaned up');
53
+
54
+ const snap = metrics.snapshot();
55
+ assert.strictEqual(snap.fileBytesWritten, size);
56
+ assert.strictEqual(snap.chunksVerified, 16);
57
+ assert.ok(snap.throughputMBs > 0, 'Throughput must be positive');
58
+ } finally {
59
+ await server.close();
60
+ await fs.rm(testDir, { recursive: true, force: true });
61
+ }
62
+ });
63
+
64
+ test('Integration - multi-channel parallel send transfer (4 channels) with SHA-256 verification', async () => {
65
+ const testDir = path.resolve('tmp/multichannel-send-test');
66
+ const sourceDir = path.join(testDir, 'source');
67
+ const destDir = path.join(testDir, 'dest');
68
+
69
+ await fs.rm(testDir, { recursive: true, force: true });
70
+ await fs.mkdir(sourceDir, { recursive: true });
71
+ await fs.mkdir(destDir, { recursive: true });
72
+
73
+ // Generate 12 MB test file
74
+ const testFile = path.join(sourceDir, 'upload-12mb.bin');
75
+ const size = 12 * 1024 * 1024;
76
+ const data = crypto.randomBytes(size);
77
+ await fs.writeFile(testFile, data);
78
+ const sourceSha = await computeSha256(testFile);
79
+
80
+ const port = 8922;
81
+ const server = await serve({ port, destinationDir: destDir });
82
+
83
+ try {
84
+ const metrics = await send({
85
+ sourcePath: testFile,
86
+ host: '127.0.0.1',
87
+ port,
88
+ channels: 4,
89
+ showProgress: false
90
+ });
91
+
92
+ await server.receiver.waitForCompletion();
93
+
94
+ const receivedFile = path.join(destDir, 'upload-12mb.bin');
95
+ assert.ok(existsSync(receivedFile), 'Received file must exist');
96
+ assert.strictEqual(await fs.stat(receivedFile).then((s) => s.size), size, 'Size must match');
97
+
98
+ const receivedSha = await computeSha256(receivedFile);
99
+ assert.strictEqual(receivedSha, sourceSha, 'SHA-256 checksum must match source');
100
+
101
+ const snap = metrics.snapshot();
102
+ assert.strictEqual(snap.fileBytesRead, size);
103
+ assert.strictEqual(snap.chunksCreated, 12);
104
+ } finally {
105
+ await server.close();
106
+ await fs.rm(testDir, { recursive: true, force: true });
107
+ }
108
+ });