teyyare 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/teyyare.js +30 -4
- package/package.json +1 -1
- package/src/core/generation.js +107 -0
- package/src/core/protocol.js +44 -1
- package/src/index.js +128 -4
- package/src/receiver/pipeline.js +53 -14
- package/src/sender/pipeline.js +12 -7
- package/test/integration/pull.test.js +131 -0
- package/test/unit/protocol.test.js +19 -1
package/bin/teyyare.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import path from 'node:path';
|
|
8
|
-
import { serve, send } from '../src/index.js';
|
|
8
|
+
import { serve, send, pull } from '../src/index.js';
|
|
9
9
|
|
|
10
10
|
function printUsage() {
|
|
11
11
|
console.log(`
|
|
@@ -106,9 +106,35 @@ Protocol overhead:${snap.protocolOverheadPercent.toFixed(2)}%
|
|
|
106
106
|
Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
107
107
|
`);
|
|
108
108
|
} else if (command === 'pull') {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
109
|
+
const source = args[1];
|
|
110
|
+
const target = args[2] || './';
|
|
111
|
+
|
|
112
|
+
if (!source) {
|
|
113
|
+
console.error('Error: teyyare pull requires <source> and [destination]');
|
|
114
|
+
printUsage();
|
|
115
|
+
process.exit(1);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const { host, port, destPath: remotePath } = parseDestination(source);
|
|
119
|
+
const metrics = await pull({
|
|
120
|
+
sourceRemotePath: remotePath,
|
|
121
|
+
host,
|
|
122
|
+
port,
|
|
123
|
+
destinationDir: target,
|
|
124
|
+
showProgress: true
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
const snap = metrics.snapshot();
|
|
128
|
+
console.log(`
|
|
129
|
+
\x1b[1m\x1b[32mPull completed successfully!\x1b[0m
|
|
130
|
+
Throughput: \x1b[1m${snap.throughputMBs.toFixed(2)} MB/s\x1b[0m
|
|
131
|
+
Duration: ${(snap.elapsedMs / 1000).toFixed(2)}s
|
|
132
|
+
Bytes: ${(snap.fileBytesWritten / (1024 * 1024)).toFixed(2)} MB
|
|
133
|
+
Generations: ${snap.generations}
|
|
134
|
+
Backpressure: ${snap.backpressureCount}
|
|
135
|
+
Protocol overhead:${snap.protocolOverheadPercent.toFixed(2)}%
|
|
136
|
+
Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
|
|
137
|
+
`);
|
|
112
138
|
} else if (command === 'status') {
|
|
113
139
|
console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.1.0 status: ready');
|
|
114
140
|
} else {
|
package/package.json
CHANGED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Zero-copy generation framing and buffer extraction utilities.
|
|
3
|
+
* Ensures complete compatibility across all versions of Muttafa and Raptiye.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Extract scatter/gather buffers [metaBuf, payloadBuf] from any Muttafa generation.
|
|
8
|
+
* @param {object} batch
|
|
9
|
+
* @returns {Uint8Array[]}
|
|
10
|
+
*/
|
|
11
|
+
export function extractGenerationBuffers(batch) {
|
|
12
|
+
if (typeof batch.buffers === 'function') {
|
|
13
|
+
return batch.buffers();
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const store = batch.store || batch;
|
|
17
|
+
if (typeof store.buffers === 'function') {
|
|
18
|
+
return store.buffers();
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Direct extraction from ArenaStore
|
|
22
|
+
const rawBuf = store._allocator?.rawBuffer;
|
|
23
|
+
const payload = rawBuf ? rawBuf.subarray(0, store._allocator.usedBytes) : new Uint8Array(0);
|
|
24
|
+
const metaBuf = new Uint8Array(store.length * 25);
|
|
25
|
+
const view = new DataView(metaBuf.buffer, metaBuf.byteOffset, metaBuf.byteLength);
|
|
26
|
+
|
|
27
|
+
let off = 0;
|
|
28
|
+
const chunks = store._chunks || [];
|
|
29
|
+
for (let c = 0; c < chunks.length; c++) {
|
|
30
|
+
const chunk = chunks[c];
|
|
31
|
+
for (let i = 0; i < chunk.count; i++) {
|
|
32
|
+
view.setFloat64(off, chunk.seqs[i], true);
|
|
33
|
+
view.setUint8(off + 8, chunk.types[i]);
|
|
34
|
+
view.setUint32(off + 9, chunk.keyOffsets[i], true);
|
|
35
|
+
view.setUint32(off + 13, chunk.keyLengths[i], true);
|
|
36
|
+
view.setUint32(off + 17, chunk.valOffsets[i], true);
|
|
37
|
+
view.setUint32(off + 21, chunk.valLengths[i], true);
|
|
38
|
+
off += 25;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return [metaBuf, payload];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Fast zero-copy generator / unpacker for received generation buffers.
|
|
47
|
+
* @param {Uint8Array} metaBuf
|
|
48
|
+
* @param {Uint8Array} payloadBuf
|
|
49
|
+
* @returns {Array<{ seq: number, type: number, key: Uint8Array, val: Uint8Array }>}
|
|
50
|
+
*/
|
|
51
|
+
export function unpackGeneration(metaBuf, payloadBuf) {
|
|
52
|
+
const entryCount = (metaBuf.byteLength / 25) | 0;
|
|
53
|
+
const view = new DataView(metaBuf.buffer, metaBuf.byteOffset, metaBuf.byteLength);
|
|
54
|
+
const entries = new Array(entryCount);
|
|
55
|
+
|
|
56
|
+
let off = 0;
|
|
57
|
+
for (let i = 0; i < entryCount; i++) {
|
|
58
|
+
const seq = view.getFloat64(off, true);
|
|
59
|
+
const type = view.getUint8(off + 8);
|
|
60
|
+
const keyOffset = view.getUint32(off + 9, true);
|
|
61
|
+
const keyLength = view.getUint32(off + 13, true);
|
|
62
|
+
const valOffset = view.getUint32(off + 17, true);
|
|
63
|
+
const valLength = view.getUint32(off + 21, true);
|
|
64
|
+
off += 25;
|
|
65
|
+
|
|
66
|
+
entries[i] = {
|
|
67
|
+
seq,
|
|
68
|
+
type,
|
|
69
|
+
key: payloadBuf.subarray(keyOffset, keyOffset + keyLength),
|
|
70
|
+
val: payloadBuf.subarray(valOffset, valOffset + valLength)
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return entries;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Pack a single-mutation batch directly into scatter/gather wire buffers.
|
|
79
|
+
* @param {number} generationId
|
|
80
|
+
* @param {number} opcode
|
|
81
|
+
* @param {Uint8Array} key
|
|
82
|
+
* @param {Uint8Array} [val]
|
|
83
|
+
* @returns {object}
|
|
84
|
+
*/
|
|
85
|
+
export function packSingleMutationBatch(generationId, opcode, key, val = new Uint8Array(0)) {
|
|
86
|
+
const metaBuf = new Uint8Array(25);
|
|
87
|
+
const view = new DataView(metaBuf.buffer, metaBuf.byteOffset, 25);
|
|
88
|
+
view.setFloat64(0, 1, true); // seq
|
|
89
|
+
view.setUint8(8, opcode);
|
|
90
|
+
view.setUint32(9, 0, true); // keyOffset
|
|
91
|
+
view.setUint32(13, key.byteLength, true); // keyLength
|
|
92
|
+
view.setUint32(17, key.byteLength, true); // valOffset
|
|
93
|
+
view.setUint32(21, val.byteLength, true); // valLength
|
|
94
|
+
|
|
95
|
+
const payloadBuf = new Uint8Array(key.byteLength + val.byteLength);
|
|
96
|
+
payloadBuf.set(key, 0);
|
|
97
|
+
payloadBuf.set(val, key.byteLength);
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
generationId,
|
|
101
|
+
firstSequence: 1,
|
|
102
|
+
lastSequence: 1,
|
|
103
|
+
entryCount: 1,
|
|
104
|
+
byteLength: metaBuf.byteLength + payloadBuf.byteLength,
|
|
105
|
+
buffers: [metaBuf, payloadBuf]
|
|
106
|
+
};
|
|
107
|
+
}
|
package/src/core/protocol.js
CHANGED
|
@@ -11,7 +11,9 @@ export const Opcodes = Object.freeze({
|
|
|
11
11
|
FILE_BEGIN: 11,
|
|
12
12
|
CHUNK: 12,
|
|
13
13
|
FILE_END: 13,
|
|
14
|
-
TRANSFER_END: 14
|
|
14
|
+
TRANSFER_END: 14,
|
|
15
|
+
PULL_REQUEST: 15,
|
|
16
|
+
PULL_ERROR: 16
|
|
15
17
|
});
|
|
16
18
|
|
|
17
19
|
export const CHUNK_META_SIZE = 25;
|
|
@@ -161,3 +163,44 @@ export function unpackTransferEnd(buf) {
|
|
|
161
163
|
const transferId = decoder.decode(buf.subarray(3, 3 + idLen));
|
|
162
164
|
return { transferId, status };
|
|
163
165
|
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Pack PULL_REQUEST metadata.
|
|
169
|
+
*/
|
|
170
|
+
export function packPullRequest({ remotePath }) {
|
|
171
|
+
const pathBytes = encoder.encode(remotePath);
|
|
172
|
+
const buf = new Uint8Array(2 + pathBytes.byteLength);
|
|
173
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
174
|
+
view.setUint16(0, pathBytes.byteLength, false);
|
|
175
|
+
buf.set(pathBytes, 2);
|
|
176
|
+
return buf;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export function unpackPullRequest(buf) {
|
|
180
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
181
|
+
const pathLen = view.getUint16(0, false);
|
|
182
|
+
const remotePath = decoder.decode(buf.subarray(2, 2 + pathLen));
|
|
183
|
+
return { remotePath };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Pack PULL_ERROR metadata.
|
|
188
|
+
*/
|
|
189
|
+
export function packPullError({ errorCode = 1, message = '' }) {
|
|
190
|
+
const msgBytes = encoder.encode(message);
|
|
191
|
+
const buf = new Uint8Array(6 + msgBytes.byteLength);
|
|
192
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
193
|
+
view.setUint32(0, errorCode, false);
|
|
194
|
+
view.setUint16(4, msgBytes.byteLength, false);
|
|
195
|
+
buf.set(msgBytes, 6);
|
|
196
|
+
return buf;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export function unpackPullError(buf) {
|
|
200
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
201
|
+
const errorCode = view.getUint32(0, false);
|
|
202
|
+
const msgLen = view.getUint16(4, false);
|
|
203
|
+
const message = decoder.decode(buf.subarray(6, 6 + msgLen));
|
|
204
|
+
return { errorCode, message };
|
|
205
|
+
}
|
|
206
|
+
|
package/src/index.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import path from 'node:path';
|
|
6
|
+
import { existsSync } from 'node:fs';
|
|
6
7
|
import { TCPTransport, ReplicationPipeline } from 'raptiye';
|
|
7
8
|
import { SenderPipeline } from './sender/pipeline.js';
|
|
8
9
|
import { ReceiverPipeline } from './receiver/pipeline.js';
|
|
@@ -10,7 +11,16 @@ import { TransferManifest } from './manifest/manifest.js';
|
|
|
10
11
|
import { FileResumeState } from './resume/state.js';
|
|
11
12
|
import { MetricsCollector } from './metrics/collector.js';
|
|
12
13
|
import { ProgressReporter } from './cli/progress.js';
|
|
13
|
-
import {
|
|
14
|
+
import { packSingleMutationBatch, unpackGeneration } from './core/generation.js';
|
|
15
|
+
import {
|
|
16
|
+
Opcodes,
|
|
17
|
+
packChunkMeta,
|
|
18
|
+
unpackChunkMeta,
|
|
19
|
+
packPullRequest,
|
|
20
|
+
unpackPullRequest,
|
|
21
|
+
packPullError,
|
|
22
|
+
unpackPullError
|
|
23
|
+
} from './core/protocol.js';
|
|
14
24
|
|
|
15
25
|
export {
|
|
16
26
|
SenderPipeline,
|
|
@@ -21,11 +31,22 @@ export {
|
|
|
21
31
|
ProgressReporter,
|
|
22
32
|
Opcodes,
|
|
23
33
|
packChunkMeta,
|
|
24
|
-
unpackChunkMeta
|
|
34
|
+
unpackChunkMeta,
|
|
35
|
+
packPullRequest,
|
|
36
|
+
unpackPullRequest,
|
|
37
|
+
packPullError,
|
|
38
|
+
unpackPullError
|
|
25
39
|
};
|
|
26
40
|
|
|
41
|
+
async function submitSingleMutation(replication, opcode, key, value = new Uint8Array(0)) {
|
|
42
|
+
const batch = packSingleMutationBatch(Date.now() & 0x7fffffff, opcode, key, value);
|
|
43
|
+
const receipt = await replication.submitBatch(batch);
|
|
44
|
+
await receipt.delivered();
|
|
45
|
+
}
|
|
46
|
+
|
|
27
47
|
/**
|
|
28
|
-
* Start a Teyyare receiver server.
|
|
48
|
+
* Start a Teyyare receiver/file server.
|
|
49
|
+
* Handles both incoming push transfers (send) and remote pull requests (pull).
|
|
29
50
|
* @param {object} options
|
|
30
51
|
* @param {number} [options.port=7421]
|
|
31
52
|
* @param {string} [options.host='0.0.0.0']
|
|
@@ -61,7 +82,42 @@ export async function serve({
|
|
|
61
82
|
replication,
|
|
62
83
|
destinationDir,
|
|
63
84
|
metrics,
|
|
64
|
-
writeDelayMs
|
|
85
|
+
writeDelayMs,
|
|
86
|
+
autoHookBatch: false
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
replication.onBatch(async (msg) => {
|
|
90
|
+
if (!msg.buffers || msg.buffers.length < 2) return;
|
|
91
|
+
const [metaBuf, payloadBuf] = msg.buffers;
|
|
92
|
+
const entries = unpackGeneration(metaBuf, payloadBuf);
|
|
93
|
+
if (entries.length > 0 && entries[0].type === Opcodes.PULL_REQUEST) {
|
|
94
|
+
const { remotePath } = unpackPullRequest(entries[0].key);
|
|
95
|
+
let resolved = path.isAbsolute(remotePath) ? remotePath : path.resolve(destinationDir, remotePath);
|
|
96
|
+
if (!existsSync(resolved)) {
|
|
97
|
+
const alt = path.resolve(destinationDir, remotePath);
|
|
98
|
+
if (existsSync(alt)) resolved = alt;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if (!existsSync(resolved)) {
|
|
102
|
+
await submitSingleMutation(
|
|
103
|
+
replication,
|
|
104
|
+
Opcodes.PULL_ERROR,
|
|
105
|
+
packPullError({ errorCode: 404, message: `Path not found on remote: ${remotePath}` })
|
|
106
|
+
);
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const manifest = await TransferManifest.build(resolved);
|
|
111
|
+
const sender = new SenderPipeline({
|
|
112
|
+
replication,
|
|
113
|
+
manifest,
|
|
114
|
+
metrics
|
|
115
|
+
});
|
|
116
|
+
await sender.transfer();
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
await receiver._processGenerationBatch(msg);
|
|
65
121
|
});
|
|
66
122
|
|
|
67
123
|
return {
|
|
@@ -137,3 +193,71 @@ export async function send({
|
|
|
137
193
|
|
|
138
194
|
return metrics;
|
|
139
195
|
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Pull a file or directory from a remote Teyyare server.
|
|
199
|
+
* @param {object} options
|
|
200
|
+
* @param {string} options.sourceRemotePath
|
|
201
|
+
* @param {string} [options.host='127.0.0.1']
|
|
202
|
+
* @param {number} [options.port=7421]
|
|
203
|
+
* @param {string} [options.destinationDir='./']
|
|
204
|
+
* @param {object} [options.pipelineOptions]
|
|
205
|
+
* @param {boolean} [options.showProgress=true]
|
|
206
|
+
* @returns {Promise<MetricsCollector>}
|
|
207
|
+
*/
|
|
208
|
+
export async function pull({
|
|
209
|
+
sourceRemotePath,
|
|
210
|
+
host = '127.0.0.1',
|
|
211
|
+
port = 7421,
|
|
212
|
+
destinationDir = './',
|
|
213
|
+
pipelineOptions = {},
|
|
214
|
+
showProgress = true
|
|
215
|
+
}) {
|
|
216
|
+
const metrics = new MetricsCollector();
|
|
217
|
+
const progress = showProgress ? new ProgressReporter() : null;
|
|
218
|
+
if (progress) progress.start();
|
|
219
|
+
|
|
220
|
+
const clientPort = 10000 + Math.floor(Math.random() * 20000);
|
|
221
|
+
const transport = new TCPTransport({
|
|
222
|
+
id: 1,
|
|
223
|
+
port: clientPort,
|
|
224
|
+
host: '127.0.0.1',
|
|
225
|
+
peerAddresses: new Map([[2, { host, port }]])
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
await transport.start();
|
|
229
|
+
|
|
230
|
+
const replication = new ReplicationPipeline({
|
|
231
|
+
transport,
|
|
232
|
+
localNode: 1,
|
|
233
|
+
remoteNode: 2,
|
|
234
|
+
maxInflightBatches: pipelineOptions.maxInflightBatches || 16,
|
|
235
|
+
maxInflightBytes: pipelineOptions.maxInflightBytes || 64 * 1024 * 1024
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
const receiver = new ReceiverPipeline({
|
|
239
|
+
replication,
|
|
240
|
+
destinationDir,
|
|
241
|
+
metrics,
|
|
242
|
+
progress
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
try {
|
|
246
|
+
// Submit PULL_REQUEST batch
|
|
247
|
+
await submitSingleMutation(
|
|
248
|
+
replication,
|
|
249
|
+
Opcodes.PULL_REQUEST,
|
|
250
|
+
packPullRequest({ remotePath: sourceRemotePath })
|
|
251
|
+
);
|
|
252
|
+
|
|
253
|
+
// Wait for receiver pipeline to receive and write all files
|
|
254
|
+
await receiver.waitForCompletion();
|
|
255
|
+
} finally {
|
|
256
|
+
if (progress) progress.stop();
|
|
257
|
+
metrics.stop();
|
|
258
|
+
await transport.close();
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
return metrics;
|
|
262
|
+
}
|
|
263
|
+
|
package/src/receiver/pipeline.js
CHANGED
|
@@ -8,9 +8,9 @@ import fs from 'node:fs/promises';
|
|
|
8
8
|
import { existsSync, createReadStream } from 'node:fs';
|
|
9
9
|
import path from 'node:path';
|
|
10
10
|
import crypto from 'node:crypto';
|
|
11
|
-
import { FrozenArenaStore } from 'muttafa';
|
|
12
11
|
import { crc32 } from 'raptiye';
|
|
13
|
-
import { Opcodes, unpackChunkMeta, unpackFileBegin, unpackTransferBegin, unpackFileEnd } from '../core/protocol.js';
|
|
12
|
+
import { Opcodes, unpackChunkMeta, unpackFileBegin, unpackTransferBegin, unpackFileEnd, unpackPullError } from '../core/protocol.js';
|
|
13
|
+
import { unpackGeneration } from '../core/generation.js';
|
|
14
14
|
import { FileResumeState } from '../resume/state.js';
|
|
15
15
|
|
|
16
16
|
export class ReceiverPipeline {
|
|
@@ -25,27 +25,38 @@ export class ReceiverPipeline {
|
|
|
25
25
|
replication,
|
|
26
26
|
destinationDir,
|
|
27
27
|
metrics = null,
|
|
28
|
-
|
|
28
|
+
progress = null,
|
|
29
|
+
writeDelayMs = 0,
|
|
30
|
+
autoHookBatch = true
|
|
29
31
|
}) {
|
|
30
32
|
this.replication = replication;
|
|
31
33
|
this.destinationDir = path.resolve(destinationDir);
|
|
32
34
|
this.metrics = metrics;
|
|
35
|
+
this.progress = progress;
|
|
33
36
|
this.writeDelayMs = writeDelayMs;
|
|
34
37
|
|
|
35
38
|
this.activeFiles = new Map(); // fileId -> { destPath, partPath, statePath, fileHandle, resumeState, ... }
|
|
36
39
|
this.completedFiles = new Set();
|
|
37
40
|
this.transferFinished = false;
|
|
41
|
+
this.transferError = null;
|
|
38
42
|
this._finishResolvers = [];
|
|
43
|
+
this._errorRejecters = [];
|
|
39
44
|
|
|
40
|
-
// Hook up Raptiye batch handler
|
|
41
|
-
this.replication
|
|
42
|
-
|
|
43
|
-
|
|
45
|
+
// Hook up Raptiye batch handler if autoHookBatch is true
|
|
46
|
+
if (this.replication && autoHookBatch) {
|
|
47
|
+
this.replication.onBatch(async (msg) => {
|
|
48
|
+
await this._processGenerationBatch(msg);
|
|
49
|
+
});
|
|
50
|
+
}
|
|
44
51
|
}
|
|
45
52
|
|
|
46
53
|
async waitForCompletion() {
|
|
54
|
+
if (this.transferError) throw this.transferError;
|
|
47
55
|
if (this.transferFinished) return;
|
|
48
|
-
return new Promise((resolve) =>
|
|
56
|
+
return new Promise((resolve, reject) => {
|
|
57
|
+
this._finishResolvers.push(resolve);
|
|
58
|
+
this._errorRejecters.push(reject);
|
|
59
|
+
});
|
|
49
60
|
}
|
|
50
61
|
|
|
51
62
|
async _processGenerationBatch(msg) {
|
|
@@ -59,14 +70,12 @@ export class ReceiverPipeline {
|
|
|
59
70
|
this.metrics.generationBytes += (metaBuf.byteLength + payloadBuf.byteLength);
|
|
60
71
|
}
|
|
61
72
|
|
|
62
|
-
// Zero-copy
|
|
63
|
-
const
|
|
64
|
-
const count =
|
|
73
|
+
// Zero-copy unpacking of generation entries
|
|
74
|
+
const entries = unpackGeneration(metaBuf, payloadBuf);
|
|
75
|
+
const count = entries.length;
|
|
65
76
|
|
|
66
77
|
for (let i = 0; i < count; i++) {
|
|
67
|
-
const type =
|
|
68
|
-
const key = store.getKey(i);
|
|
69
|
-
const val = store.getValue(i);
|
|
78
|
+
const { type, key, val } = entries[i];
|
|
70
79
|
|
|
71
80
|
if (this.metrics) {
|
|
72
81
|
this.metrics.mutations++;
|
|
@@ -74,6 +83,8 @@ export class ReceiverPipeline {
|
|
|
74
83
|
|
|
75
84
|
switch (type) {
|
|
76
85
|
case Opcodes.TRANSFER_BEGIN: {
|
|
86
|
+
this.transferFinished = false;
|
|
87
|
+
this.transferError = null;
|
|
77
88
|
const tinfo = unpackTransferBegin(key);
|
|
78
89
|
break;
|
|
79
90
|
}
|
|
@@ -99,8 +110,21 @@ export class ReceiverPipeline {
|
|
|
99
110
|
resolve();
|
|
100
111
|
}
|
|
101
112
|
this._finishResolvers = [];
|
|
113
|
+
this._errorRejecters = [];
|
|
102
114
|
break;
|
|
103
115
|
}
|
|
116
|
+
|
|
117
|
+
case Opcodes.PULL_ERROR: {
|
|
118
|
+
const { errorCode, message } = unpackPullError(key);
|
|
119
|
+
const err = new Error(`Remote pull error [${errorCode}]: ${message}`);
|
|
120
|
+
this.transferError = err;
|
|
121
|
+
for (const reject of this._errorRejecters) {
|
|
122
|
+
reject(err);
|
|
123
|
+
}
|
|
124
|
+
this._finishResolvers = [];
|
|
125
|
+
this._errorRejecters = [];
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
104
128
|
}
|
|
105
129
|
}
|
|
106
130
|
}
|
|
@@ -138,6 +162,13 @@ export class ReceiverPipeline {
|
|
|
138
162
|
sha256: info.sha256,
|
|
139
163
|
chunksWrittenSinceSave: 0
|
|
140
164
|
});
|
|
165
|
+
|
|
166
|
+
if (this.progress) {
|
|
167
|
+
this.progress.update({
|
|
168
|
+
fileName: info.path,
|
|
169
|
+
totalBytes: Number(info.fileSize)
|
|
170
|
+
});
|
|
171
|
+
}
|
|
141
172
|
}
|
|
142
173
|
|
|
143
174
|
async _handleChunk(key, val) {
|
|
@@ -184,6 +215,14 @@ export class ReceiverPipeline {
|
|
|
184
215
|
this.metrics.chunksVerified++;
|
|
185
216
|
}
|
|
186
217
|
|
|
218
|
+
if (this.progress) {
|
|
219
|
+
this.progress.update({
|
|
220
|
+
fileName: file.relativePath,
|
|
221
|
+
bytesDone: this.metrics ? this.metrics.fileBytesWritten : 0,
|
|
222
|
+
chunksDone: this.metrics ? this.metrics.chunksVerified : 0
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
|
|
187
226
|
file.resumeState.markVerified(meta.chunkIndex);
|
|
188
227
|
file.chunksWrittenSinceSave++;
|
|
189
228
|
|
package/src/sender/pipeline.js
CHANGED
|
@@ -17,6 +17,8 @@ import {
|
|
|
17
17
|
packTransferBegin,
|
|
18
18
|
packTransferEnd
|
|
19
19
|
} from '../core/protocol.js';
|
|
20
|
+
import { extractGenerationBuffers } from '../core/generation.js';
|
|
21
|
+
import { FileResumeState } from '../resume/state.js';
|
|
20
22
|
|
|
21
23
|
class RaptiyeSink extends Sink {
|
|
22
24
|
/**
|
|
@@ -36,20 +38,23 @@ class RaptiyeSink extends Sink {
|
|
|
36
38
|
* @param {import('muttafa').ImmutableBuffer} batch
|
|
37
39
|
*/
|
|
38
40
|
async flush(batch) {
|
|
39
|
-
const buffers = batch
|
|
41
|
+
const buffers = extractGenerationBuffers(batch);
|
|
40
42
|
const t0 = Date.now();
|
|
43
|
+
const generationId = batch.generationId || batch.generation || 1;
|
|
44
|
+
const entryCount = batch.entryCount || batch.store?.length || 1;
|
|
45
|
+
const byteLength = batch.byteLength || (buffers[0].byteLength + buffers[1].byteLength);
|
|
41
46
|
|
|
42
47
|
if (this.metrics) {
|
|
43
48
|
this.metrics.generations++;
|
|
44
|
-
this.metrics.generationBytes +=
|
|
49
|
+
this.metrics.generationBytes += byteLength;
|
|
45
50
|
}
|
|
46
51
|
|
|
47
52
|
const receipt = await this.replication.submitBatch({
|
|
48
|
-
generationId
|
|
49
|
-
firstSequence: batch.firstSequence,
|
|
50
|
-
lastSequence: batch.lastSequence,
|
|
51
|
-
entryCount
|
|
52
|
-
byteLength
|
|
53
|
+
generationId,
|
|
54
|
+
firstSequence: batch.firstSequence || 1,
|
|
55
|
+
lastSequence: batch.lastSequence || 1,
|
|
56
|
+
entryCount,
|
|
57
|
+
byteLength,
|
|
53
58
|
buffers
|
|
54
59
|
});
|
|
55
60
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert';
|
|
3
|
+
import fs from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import crypto from 'node:crypto';
|
|
6
|
+
import { serve, pull } from '../../src/index.js';
|
|
7
|
+
|
|
8
|
+
async function computeSha256(filePath) {
|
|
9
|
+
const content = await fs.readFile(filePath);
|
|
10
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
test('Integration - end-to-end file pull with SHA-256 verification', async () => {
|
|
14
|
+
const testDir = path.resolve('tmp/pull-test-single');
|
|
15
|
+
const serverDir = path.join(testDir, 'server');
|
|
16
|
+
const clientDir = path.join(testDir, 'client');
|
|
17
|
+
|
|
18
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
19
|
+
await fs.mkdir(serverDir, { recursive: true });
|
|
20
|
+
await fs.mkdir(clientDir, { recursive: true });
|
|
21
|
+
|
|
22
|
+
// Create 5 MB test file
|
|
23
|
+
const testFile = path.join(serverDir, 'archive.bin');
|
|
24
|
+
const size = 5 * 1024 * 1024;
|
|
25
|
+
const buf = Buffer.alloc(size);
|
|
26
|
+
for (let i = 0; i < size; i += 4096) {
|
|
27
|
+
buf.writeUInt32LE((i * 17) >>> 0, i);
|
|
28
|
+
}
|
|
29
|
+
await fs.writeFile(testFile, buf);
|
|
30
|
+
const sourceSha = await computeSha256(testFile);
|
|
31
|
+
|
|
32
|
+
const port = 8821;
|
|
33
|
+
const server = await serve({ port, destinationDir: serverDir });
|
|
34
|
+
|
|
35
|
+
try {
|
|
36
|
+
const metrics = await pull({
|
|
37
|
+
sourceRemotePath: 'archive.bin',
|
|
38
|
+
host: '127.0.0.1',
|
|
39
|
+
port,
|
|
40
|
+
destinationDir: clientDir,
|
|
41
|
+
showProgress: false
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
const pulledFile = path.join(clientDir, 'archive.bin');
|
|
45
|
+
assert.strictEqual(await fs.stat(pulledFile).then(s => s.size), size);
|
|
46
|
+
|
|
47
|
+
const pulledSha = await computeSha256(pulledFile);
|
|
48
|
+
assert.strictEqual(pulledSha, sourceSha);
|
|
49
|
+
|
|
50
|
+
const snap = metrics.snapshot();
|
|
51
|
+
assert.strictEqual(snap.fileBytesWritten, size);
|
|
52
|
+
} finally {
|
|
53
|
+
await server.close();
|
|
54
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('Integration - directory pull with nested structure', async () => {
|
|
59
|
+
const testDir = path.resolve('tmp/pull-test-dir');
|
|
60
|
+
const serverDir = path.join(testDir, 'server');
|
|
61
|
+
const clientDir = path.join(testDir, 'client');
|
|
62
|
+
|
|
63
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
64
|
+
await fs.mkdir(serverDir, { recursive: true });
|
|
65
|
+
await fs.mkdir(clientDir, { recursive: true });
|
|
66
|
+
|
|
67
|
+
const tree = {
|
|
68
|
+
'docs/readme.txt': 'Hello world documentation',
|
|
69
|
+
'src/index.js': 'console.log("hello");',
|
|
70
|
+
'src/nested/deep.json': JSON.stringify({ ok: true, nested: [1, 2, 3] })
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
for (const [relPath, content] of Object.entries(tree)) {
|
|
74
|
+
const fullPath = path.join(serverDir, 'my-pkg', relPath);
|
|
75
|
+
await fs.mkdir(path.dirname(fullPath), { recursive: true });
|
|
76
|
+
await fs.writeFile(fullPath, content);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const port = 8822;
|
|
80
|
+
const server = await serve({ port, destinationDir: serverDir });
|
|
81
|
+
|
|
82
|
+
try {
|
|
83
|
+
await pull({
|
|
84
|
+
sourceRemotePath: 'my-pkg',
|
|
85
|
+
host: '127.0.0.1',
|
|
86
|
+
port,
|
|
87
|
+
destinationDir: clientDir,
|
|
88
|
+
showProgress: false
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
for (const [relPath, content] of Object.entries(tree)) {
|
|
92
|
+
const destFile = path.join(clientDir, relPath);
|
|
93
|
+
const read = await fs.readFile(destFile, 'utf8');
|
|
94
|
+
assert.strictEqual(read, content);
|
|
95
|
+
}
|
|
96
|
+
} finally {
|
|
97
|
+
await server.close();
|
|
98
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('Integration - pull non-existent path throws remote error', async () => {
|
|
103
|
+
const testDir = path.resolve('tmp/pull-test-err');
|
|
104
|
+
const serverDir = path.join(testDir, 'server');
|
|
105
|
+
const clientDir = path.join(testDir, 'client');
|
|
106
|
+
|
|
107
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
108
|
+
await fs.mkdir(serverDir, { recursive: true });
|
|
109
|
+
await fs.mkdir(clientDir, { recursive: true });
|
|
110
|
+
|
|
111
|
+
const port = 8823;
|
|
112
|
+
const server = await serve({ port, destinationDir: serverDir });
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
await assert.rejects(
|
|
116
|
+
async () => {
|
|
117
|
+
await pull({
|
|
118
|
+
sourceRemotePath: 'does-not-exist.tar',
|
|
119
|
+
host: '127.0.0.1',
|
|
120
|
+
port,
|
|
121
|
+
destinationDir: clientDir,
|
|
122
|
+
showProgress: false
|
|
123
|
+
});
|
|
124
|
+
},
|
|
125
|
+
/Remote pull error \[404\]: Path not found on remote/
|
|
126
|
+
);
|
|
127
|
+
} finally {
|
|
128
|
+
await server.close();
|
|
129
|
+
await fs.rm(testDir, { recursive: true, force: true });
|
|
130
|
+
}
|
|
131
|
+
});
|
|
@@ -10,7 +10,11 @@ import {
|
|
|
10
10
|
packTransferBegin,
|
|
11
11
|
unpackTransferBegin,
|
|
12
12
|
packFileEnd,
|
|
13
|
-
unpackFileEnd
|
|
13
|
+
unpackFileEnd,
|
|
14
|
+
packPullRequest,
|
|
15
|
+
unpackPullRequest,
|
|
16
|
+
packPullError,
|
|
17
|
+
unpackPullError
|
|
14
18
|
} from '../../src/core/protocol.js';
|
|
15
19
|
|
|
16
20
|
test('Protocol - pack and unpack chunk metadata', () => {
|
|
@@ -71,3 +75,17 @@ test('Protocol - pack and unpack transfer begin', () => {
|
|
|
71
75
|
assert.strictEqual(unpacked.fileCount, 250);
|
|
72
76
|
assert.strictEqual(unpacked.totalBytes, 53687091200n);
|
|
73
77
|
});
|
|
78
|
+
|
|
79
|
+
test('Protocol - pack and unpack pull request', () => {
|
|
80
|
+
const req = packPullRequest({ remotePath: '/root/cdn/app.zip' });
|
|
81
|
+
const unpacked = unpackPullRequest(req);
|
|
82
|
+
assert.strictEqual(unpacked.remotePath, '/root/cdn/app.zip');
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test('Protocol - pack and unpack pull error', () => {
|
|
86
|
+
const err = packPullError({ errorCode: 404, message: 'File not found: /nonexistent' });
|
|
87
|
+
const unpacked = unpackPullError(err);
|
|
88
|
+
assert.strictEqual(unpacked.errorCode, 404);
|
|
89
|
+
assert.strictEqual(unpacked.message, 'File not found: /nonexistent');
|
|
90
|
+
});
|
|
91
|
+
|