swarm-deploy 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +680 -49
- package/RELEASING.md +2 -2
- package/SECURITY.md +2 -3
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +79 -22
- package/dist/cli.js.map +1 -1
- package/dist/client.d.ts +19 -29
- package/dist/client.js +46 -55
- package/dist/client.js.map +1 -1
- package/dist/errors.d.ts +4 -0
- package/dist/errors.js +4 -0
- package/dist/errors.js.map +1 -1
- package/dist/files.d.ts +13 -28
- package/dist/files.js +13 -78
- package/dist/files.js.map +1 -1
- package/dist/hooks-module.d.ts +9 -0
- package/dist/hooks-module.js +118 -0
- package/dist/hooks-module.js.map +1 -0
- package/dist/hooks.d.ts +69 -0
- package/dist/hooks.js +88 -0
- package/dist/hooks.js.map +1 -0
- package/dist/index.d.ts +7 -3
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/release.d.ts +26 -0
- package/dist/release.js +419 -0
- package/dist/release.js.map +1 -0
- package/dist/server.d.ts +86 -0
- package/dist/server.js +405 -53
- package/dist/server.js.map +1 -1
- package/dist/storage/commit-journal.d.ts +39 -2
- package/dist/storage/commit-journal.js +140 -2
- package/dist/storage/commit-journal.js.map +1 -1
- package/dist/storage/commit-store.d.ts +69 -8
- package/dist/storage/commit-store.js +374 -22
- package/dist/storage/commit-store.js.map +1 -1
- package/dist/storage/layout.js +7 -1
- package/dist/storage/layout.js.map +1 -1
- package/dist/storage/link-store.d.ts +70 -0
- package/dist/storage/link-store.js +363 -0
- package/dist/storage/link-store.js.map +1 -0
- package/dist/storage/recovery.d.ts +2 -1
- package/dist/storage/recovery.js +8 -1
- package/dist/storage/recovery.js.map +1 -1
- package/dist/storage/retention.d.ts +41 -21
- package/dist/storage/retention.js +162 -8
- package/dist/storage/retention.js.map +1 -1
- package/dist/storage/tar-session-store.d.ts +36 -5
- package/dist/storage/tar-session-store.js +283 -55
- package/dist/storage/tar-session-store.js.map +1 -1
- package/dist/storage/tree-fs.d.ts +44 -0
- package/dist/storage/tree-fs.js +580 -0
- package/dist/storage/tree-fs.js.map +1 -0
- package/dist/storage/tree-staging.d.ts +9 -0
- package/dist/storage/tree-staging.js +64 -0
- package/dist/storage/tree-staging.js.map +1 -0
- package/dist/storage/types.d.ts +15 -0
- package/dist/symlinks.d.ts +36 -0
- package/dist/symlinks.js +128 -0
- package/dist/symlinks.js.map +1 -0
- package/dist/tar-protocol/controls.d.ts +25 -0
- package/dist/tar-protocol/controls.js +108 -2
- package/dist/tar-protocol/controls.js.map +1 -1
- package/dist/tar-protocol/direct-wire.d.ts +3 -3
- package/dist/tar-protocol/direct-wire.js +2 -2
- package/dist/tar-protocol/direct-wire.js.map +1 -1
- package/dist/tar-protocol/manifest.d.ts +13 -1
- package/dist/tar-protocol/manifest.js +26 -1
- package/dist/tar-protocol/manifest.js.map +1 -1
- package/dist/tar-protocol/tree-extract.d.ts +37 -0
- package/dist/tar-protocol/tree-extract.js +240 -0
- package/dist/tar-protocol/tree-extract.js.map +1 -0
- package/dist/tar-protocol/tree-manifest.d.ts +45 -0
- package/dist/tar-protocol/tree-manifest.js +277 -0
- package/dist/tar-protocol/tree-manifest.js.map +1 -0
- package/dist/tar-protocol/tree-path.d.ts +15 -0
- package/dist/tar-protocol/tree-path.js +68 -0
- package/dist/tar-protocol/tree-path.js.map +1 -0
- package/dist/tar-protocol/tree.d.ts +68 -0
- package/dist/tar-protocol/tree.js +269 -0
- package/dist/tar-protocol/tree.js.map +1 -0
- package/dist/tar-protocol/ustar.d.ts +14 -0
- package/dist/tar-protocol/ustar.js +81 -1
- package/dist/tar-protocol/ustar.js.map +1 -1
- package/dist/types.d.ts +3 -0
- package/package.json +3 -1
package/dist/server.js
CHANGED
|
@@ -3,25 +3,34 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.FINGERPRINT_LENGTH = exports.MAX_ACTIVE_UPLOADS = exports.MAX_CONNECTIONS = exports.DEFAULT_MIN_FREE_BYTES = exports.DEFAULT_MAX_ACTIVE_UPLOADS = exports.DEFAULT_MAX_CONNECTIONS = exports.Server = void 0;
|
|
6
|
+
exports.FINGERPRINT_LENGTH = exports.MAX_ACTIVE_UPLOADS = exports.MAX_CONNECTIONS = exports.DEFAULT_MIN_FREE_BYTES = exports.DEFAULT_MAX_ACTIVE_UPLOADS = exports.DEFAULT_MAX_CONNECTIONS = exports.Server = exports.MAX_OWED_POST_COMMIT_RETENTION = void 0;
|
|
7
7
|
exports.fingerprint = fingerprint;
|
|
8
8
|
const b4a_1 = __importDefault(require("b4a"));
|
|
9
9
|
const _events_1 = __importDefault(require("#events"));
|
|
10
10
|
const _fs_1 = __importDefault(require("#fs"));
|
|
11
|
+
const _path_1 = __importDefault(require("#path"));
|
|
11
12
|
const sodium_native_1 = __importDefault(require("sodium-native"));
|
|
12
13
|
const abort_js_1 = require("./abort.js");
|
|
13
14
|
const direct_dht_js_1 = require("./direct-dht.js");
|
|
14
15
|
const errors_js_1 = require("./errors.js");
|
|
15
16
|
const files_js_1 = require("./files.js");
|
|
17
|
+
const hooks_js_1 = require("./hooks.js");
|
|
16
18
|
const identity_js_1 = require("./identity.js");
|
|
19
|
+
const release_js_1 = require("./release.js");
|
|
17
20
|
const commit_store_js_1 = require("./storage/commit-store.js");
|
|
21
|
+
const link_store_js_1 = require("./storage/link-store.js");
|
|
18
22
|
const layout_js_1 = require("./storage/layout.js");
|
|
23
|
+
const root_coordinator_js_1 = require("./storage/root-coordinator.js");
|
|
19
24
|
const recovery_js_1 = require("./storage/recovery.js");
|
|
20
25
|
const retention_js_1 = require("./storage/retention.js");
|
|
21
26
|
const session_store_js_1 = require("./storage/session-store.js");
|
|
27
|
+
const tree_fs_js_1 = require("./storage/tree-fs.js");
|
|
28
|
+
const controls_js_1 = require("./tar-protocol/controls.js");
|
|
22
29
|
const direct_wire_js_1 = require("./tar-protocol/direct-wire.js");
|
|
23
30
|
const hash_js_1 = require("./tar-protocol/hash.js");
|
|
24
31
|
const manifest_js_1 = require("./tar-protocol/manifest.js");
|
|
32
|
+
const tree_manifest_js_1 = require("./tar-protocol/tree-manifest.js");
|
|
33
|
+
const symlinks_js_1 = require("./symlinks.js");
|
|
25
34
|
const EventEmitter = _events_1.default.EventEmitter;
|
|
26
35
|
const DEFAULT_MAX_CONNECTIONS = 64;
|
|
27
36
|
exports.DEFAULT_MAX_CONNECTIONS = DEFAULT_MAX_CONNECTIONS;
|
|
@@ -36,6 +45,15 @@ exports.MAX_ACTIVE_UPLOADS = MAX_ACTIVE_UPLOADS;
|
|
|
36
45
|
const FINGERPRINT_LENGTH = 12;
|
|
37
46
|
exports.FINGERPRINT_LENGTH = FINGERPRINT_LENGTH;
|
|
38
47
|
const TAR_DURABILITY_BATCH_BYTES = 1024 * 1024;
|
|
48
|
+
/**
|
|
49
|
+
* How many transfers may owe a deferred post-commit retention pass at once.
|
|
50
|
+
*
|
|
51
|
+
* The set grows only when an `afterCommit` hook fails after a durable commit,
|
|
52
|
+
* and entries are removed as soon as a retry succeeds, so reaching the bound
|
|
53
|
+
* means a persistently failing hook. Capping it at the connection ceiling
|
|
54
|
+
* keeps a broken deployment step from growing server memory without limit.
|
|
55
|
+
*/
|
|
56
|
+
exports.MAX_OWED_POST_COMMIT_RETENTION = MAX_CONNECTIONS;
|
|
39
57
|
function fail(code, message, cause = null) {
|
|
40
58
|
return new errors_js_1.SwarmDeployError(code, message, cause);
|
|
41
59
|
}
|
|
@@ -91,6 +109,11 @@ class Server extends EventEmitter {
|
|
|
91
109
|
minFreeBytes;
|
|
92
110
|
maxAge;
|
|
93
111
|
maxStorageBytes;
|
|
112
|
+
artifactPatterns;
|
|
113
|
+
maxCount;
|
|
114
|
+
maxVersions;
|
|
115
|
+
versionGranularity;
|
|
116
|
+
releaseMatcher;
|
|
94
117
|
allowedKeySnapshot;
|
|
95
118
|
logger;
|
|
96
119
|
dht;
|
|
@@ -98,6 +121,10 @@ class Server extends EventEmitter {
|
|
|
98
121
|
storage;
|
|
99
122
|
scheduler;
|
|
100
123
|
replaceNames;
|
|
124
|
+
hooks;
|
|
125
|
+
symlinks;
|
|
126
|
+
linkNames;
|
|
127
|
+
links = null;
|
|
101
128
|
listening = false;
|
|
102
129
|
closed = false;
|
|
103
130
|
keyPair;
|
|
@@ -105,6 +132,24 @@ class Server extends EventEmitter {
|
|
|
105
132
|
abort = (0, abort_js_1.createAbortController)();
|
|
106
133
|
active = new Map();
|
|
107
134
|
activeUploads = new Set();
|
|
135
|
+
/**
|
|
136
|
+
* Transfer IDs this process committed but whose deferred post-commit
|
|
137
|
+
* retention is still owed because `afterCommit` has not yet succeeded. Only
|
|
138
|
+
* these may start a retention pass from an already-committed retry, so an
|
|
139
|
+
* ordinary duplicate offer cannot be used to force repeated full passes.
|
|
140
|
+
* The set is in-memory only and bounded by `MAX_OWED_POST_COMMIT_RETENTION`;
|
|
141
|
+
* a restart or an eviction loses an entry, and the documented fallback is the
|
|
142
|
+
* next startup, scheduled, or commit-triggered pass.
|
|
143
|
+
*/
|
|
144
|
+
pendingAfterCommit = new Set();
|
|
145
|
+
/**
|
|
146
|
+
* Authenticated transfer IDs a connection currently owns. One connection
|
|
147
|
+
* holds at most one entry and releases it in a `finally`, so the set is
|
|
148
|
+
* bounded by `maxConnections`. A second connection offering the same
|
|
149
|
+
* transfer is rejected with `FILE_BUSY` rather than running the lifecycle
|
|
150
|
+
* and its hooks concurrently against one staging and commit identity.
|
|
151
|
+
*/
|
|
152
|
+
activeTransfers = new Set();
|
|
108
153
|
receives = new Set();
|
|
109
154
|
layout = null;
|
|
110
155
|
sessions = null;
|
|
@@ -149,6 +194,38 @@ class Server extends EventEmitter {
|
|
|
149
194
|
}
|
|
150
195
|
this.maxAge = options.maxAge;
|
|
151
196
|
this.maxStorageBytes = options.maxStorageBytes;
|
|
197
|
+
const patterns = options.artifactPatterns ?? [];
|
|
198
|
+
if (typeof patterns[Symbol.iterator] !== 'function') {
|
|
199
|
+
throw fail(errors_js_1.ERRORS.PROTOCOL_INVALID, 'Invalid artifact patterns');
|
|
200
|
+
}
|
|
201
|
+
try {
|
|
202
|
+
this.artifactPatterns = Object.freeze([...patterns]);
|
|
203
|
+
this.releaseMatcher = new release_js_1.ReleaseMatcher(this.artifactPatterns);
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
throw fail(errors_js_1.ERRORS.PROTOCOL_INVALID, 'Invalid artifact patterns', error);
|
|
207
|
+
}
|
|
208
|
+
if (options.maxCount !== undefined) {
|
|
209
|
+
positive(options.maxCount, 'maxCount');
|
|
210
|
+
if (this.releaseMatcher.size === 0) {
|
|
211
|
+
throw fail(errors_js_1.ERRORS.PROTOCOL_INVALID, 'maxCount requires artifact patterns');
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
if (options.maxVersions !== undefined) {
|
|
215
|
+
positive(options.maxVersions, 'maxVersions');
|
|
216
|
+
if (!this.releaseMatcher.hasVersionPattern) {
|
|
217
|
+
throw fail(errors_js_1.ERRORS.PROTOCOL_INVALID, 'maxVersions requires a {version} artifact pattern');
|
|
218
|
+
}
|
|
219
|
+
if (options.versionGranularity !== 'major' && options.versionGranularity !== 'minor') {
|
|
220
|
+
throw fail(errors_js_1.ERRORS.PROTOCOL_INVALID, 'Invalid versionGranularity');
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
else if (options.versionGranularity !== undefined) {
|
|
224
|
+
throw fail(errors_js_1.ERRORS.PROTOCOL_INVALID, 'versionGranularity requires maxVersions');
|
|
225
|
+
}
|
|
226
|
+
this.maxCount = options.maxCount;
|
|
227
|
+
this.maxVersions = options.maxVersions;
|
|
228
|
+
this.versionGranularity = options.versionGranularity;
|
|
152
229
|
const allow = [];
|
|
153
230
|
if (!options.allowedKeys ||
|
|
154
231
|
typeof options.allowedKeys[Symbol.iterator] !== 'function') {
|
|
@@ -167,9 +244,23 @@ class Server extends EventEmitter {
|
|
|
167
244
|
this.storage = options.storage || _fs_1.default.promises;
|
|
168
245
|
this.scheduler = options.scheduler || { setTimeout, clearTimeout, setInterval, clearInterval };
|
|
169
246
|
this.replaceNames = (0, files_js_1.validateReplaceNames)(options.replaceNames);
|
|
247
|
+
this.hooks = (0, hooks_js_1.snapshotHooks)(options.hooks);
|
|
248
|
+
this.symlinks = (0, symlinks_js_1.compileSymlinkRules)(options.symlinks);
|
|
249
|
+
this.linkNames = (0, symlinks_js_1.symlinkRuleNames)(this.symlinks);
|
|
250
|
+
if (this.symlinks.length > 0)
|
|
251
|
+
(0, tree_fs_js_1.assertSymlinkCapable)(this.storage);
|
|
170
252
|
this.logger = safeLogger(options.logger);
|
|
171
253
|
this.signal = this.abort.signal;
|
|
172
254
|
}
|
|
255
|
+
artifactKind(metadata) {
|
|
256
|
+
return (0, controls_js_1.isTreeMetadata)(metadata) ? 'directory' : 'file';
|
|
257
|
+
}
|
|
258
|
+
payloadBytes(metadata) {
|
|
259
|
+
return (0, controls_js_1.isTreeMetadata)(metadata) ? metadata.payloadBytes : metadata.fileSize;
|
|
260
|
+
}
|
|
261
|
+
payloadDigest(metadata) {
|
|
262
|
+
return (0, controls_js_1.isTreeMetadata)(metadata) ? metadata.treeSha256 : metadata.fileSha256;
|
|
263
|
+
}
|
|
173
264
|
emitSafe(type, payload) {
|
|
174
265
|
try {
|
|
175
266
|
this.emit(type, payload);
|
|
@@ -189,9 +280,109 @@ class Server extends EventEmitter {
|
|
|
189
280
|
return {
|
|
190
281
|
transfer: fingerprint(b4a_1.default.from(metadata.transferId, 'hex')),
|
|
191
282
|
name: metadata.name,
|
|
192
|
-
|
|
283
|
+
kind: this.artifactKind(metadata),
|
|
284
|
+
size: this.payloadBytes(metadata)
|
|
193
285
|
};
|
|
194
286
|
}
|
|
287
|
+
async rejectOffer(socket, event, owner, reason) {
|
|
288
|
+
await (0, direct_wire_js_1.writeAdmission)(socket, { v: 1, status: 'REJECTED', code: reason }, { signal: this.signal, timeout: this.idleTimeout });
|
|
289
|
+
this.emitSafe('offer', {
|
|
290
|
+
...event,
|
|
291
|
+
fingerprint: fingerprint(owner),
|
|
292
|
+
status: 'rejected',
|
|
293
|
+
reason
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
hookArtifact(metadata, release) {
|
|
297
|
+
return Object.freeze({
|
|
298
|
+
name: metadata.name,
|
|
299
|
+
kind: this.artifactKind(metadata),
|
|
300
|
+
size: this.payloadBytes(metadata),
|
|
301
|
+
sha256: this.payloadDigest(metadata),
|
|
302
|
+
transferId: metadata.transferId,
|
|
303
|
+
...((0, controls_js_1.isTreeMetadata)(metadata) ? { entryCount: metadata.entryCount } : {}),
|
|
304
|
+
...(metadata.sourceParent === undefined ? {} : { sourceParent: metadata.sourceParent }),
|
|
305
|
+
...(release === null
|
|
306
|
+
? {}
|
|
307
|
+
: {
|
|
308
|
+
release: Object.freeze({
|
|
309
|
+
series: release.series,
|
|
310
|
+
...(release.version === undefined ? {} : { version: release.version })
|
|
311
|
+
})
|
|
312
|
+
})
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Computes and converges the desired links from durable commit records.
|
|
317
|
+
* Called with the root lease already held.
|
|
318
|
+
*/
|
|
319
|
+
async reconcileLinksUnlocked(records) {
|
|
320
|
+
if (this.symlinks.length === 0 || !this.links)
|
|
321
|
+
return new Set();
|
|
322
|
+
const desired = (0, symlinks_js_1.selectDesiredLinks)(this.symlinks, records);
|
|
323
|
+
await this.links.reconcile(desired, this.linkNames, {
|
|
324
|
+
managedArtifactNames: new Set(records.map((record) => record.name))
|
|
325
|
+
});
|
|
326
|
+
return new Set(desired.map((link) => link.transferId));
|
|
327
|
+
}
|
|
328
|
+
/** Pins link targets when reconciliation fails but retention must continue. */
|
|
329
|
+
async linkPinsFallbackUnlocked(records) {
|
|
330
|
+
const pins = new Set((0, symlinks_js_1.selectDesiredLinks)(this.symlinks, records).map((link) => link.transferId));
|
|
331
|
+
if (!this.links)
|
|
332
|
+
return pins;
|
|
333
|
+
for (const ledger of await this.links.list()) {
|
|
334
|
+
pins.add(ledger.transferId);
|
|
335
|
+
const target = records.find((record) => record.name === ledger.target);
|
|
336
|
+
if (target)
|
|
337
|
+
pins.add(target.transferId);
|
|
338
|
+
}
|
|
339
|
+
return pins;
|
|
340
|
+
}
|
|
341
|
+
/** Reconciles after a durable commit; this is the only caller that leases the root. */
|
|
342
|
+
async reconcileAfterCommit() {
|
|
343
|
+
if (this.symlinks.length === 0 || !this.links || !this.commits)
|
|
344
|
+
return;
|
|
345
|
+
try {
|
|
346
|
+
await (0, root_coordinator_js_1.withRootLease)(this.layout.root, async () => this.reconcileLinksUnlocked(await this.commits.list()));
|
|
347
|
+
}
|
|
348
|
+
catch (error) {
|
|
349
|
+
if (error instanceof errors_js_1.SwarmDeployError && error.code === errors_js_1.ERRORS.LINK_CONFLICT)
|
|
350
|
+
throw error;
|
|
351
|
+
throw new errors_js_1.SwarmDeployError(errors_js_1.ERRORS.LINK_FAILED, 'Unable to reconcile managed symlinks', error);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
/** Runs a gating callback; only callback exceptions become HOOK_FAILED. */
|
|
355
|
+
async runHook(phase, hook, context) {
|
|
356
|
+
if (hook === undefined)
|
|
357
|
+
return;
|
|
358
|
+
(0, abort_js_1.throwIfAborted)(this.signal);
|
|
359
|
+
let outcome;
|
|
360
|
+
try {
|
|
361
|
+
outcome = await (0, hooks_js_1.invokeHook)(hook, context, this.signal);
|
|
362
|
+
}
|
|
363
|
+
catch (cause) {
|
|
364
|
+
throw (0, hooks_js_1.hookError)(phase, cause);
|
|
365
|
+
}
|
|
366
|
+
if (outcome === 'aborted')
|
|
367
|
+
throw (0, abort_js_1.abortError)();
|
|
368
|
+
}
|
|
369
|
+
/** Runs onFailure once; its own failure is logged and never replaces the original. */
|
|
370
|
+
async runFailureHook(context, owner, transfer) {
|
|
371
|
+
const hook = this.hooks.onFailure;
|
|
372
|
+
if (hook === undefined)
|
|
373
|
+
return;
|
|
374
|
+
try {
|
|
375
|
+
await (0, hooks_js_1.invokeHook)(hook, context, this.signal);
|
|
376
|
+
}
|
|
377
|
+
catch {
|
|
378
|
+
this.logger.warn('Failure hook failed', {
|
|
379
|
+
fingerprint: owner ? fingerprint(owner) : 'invalid',
|
|
380
|
+
transfer,
|
|
381
|
+
phase: context.phase,
|
|
382
|
+
reason: errors_js_1.ERRORS.HOOK_FAILED
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
}
|
|
195
386
|
async receive(socket) {
|
|
196
387
|
const owner = socket.remotePublicKey;
|
|
197
388
|
if (this.closed || !this.allowed(owner) || !this.sessions || !this.commits) {
|
|
@@ -231,30 +422,155 @@ class Server extends EventEmitter {
|
|
|
231
422
|
let verificationSucceeded = false;
|
|
232
423
|
let commitStarted = false;
|
|
233
424
|
let commitSucceeded = false;
|
|
425
|
+
let artifact = null;
|
|
426
|
+
let phase = 'offer';
|
|
427
|
+
let hookPath = null;
|
|
428
|
+
let resumed = false;
|
|
429
|
+
let alreadyCommitted = false;
|
|
430
|
+
let hooksFinished = false;
|
|
431
|
+
let finalStarted = false;
|
|
432
|
+
let failureReported = false;
|
|
433
|
+
let guardedTransfer = null;
|
|
434
|
+
/**
|
|
435
|
+
* Hands the transfer back before this connection's last await. Clearing
|
|
436
|
+
* `guardedTransfer` makes the `finally` a no-op, so a late release can
|
|
437
|
+
* never take ownership away from the connection that acquired it next.
|
|
438
|
+
*/
|
|
439
|
+
const releaseTransfer = () => {
|
|
440
|
+
if (guardedTransfer === null)
|
|
441
|
+
return;
|
|
442
|
+
this.activeTransfers.delete(guardedTransfer);
|
|
443
|
+
guardedTransfer = null;
|
|
444
|
+
};
|
|
445
|
+
// Runs onFailure at most once per connection, after the client was answered.
|
|
446
|
+
const reportFailure = async (error) => {
|
|
447
|
+
if (failureReported || !event || !artifact)
|
|
448
|
+
return;
|
|
449
|
+
failureReported = true;
|
|
450
|
+
// The guard serializes the commit lifecycle, and this connection has
|
|
451
|
+
// finished its own. `onFailure` only observes, so holding the transfer
|
|
452
|
+
// across a slow callback would reject the client's legitimate retry.
|
|
453
|
+
releaseTransfer();
|
|
454
|
+
await this.runFailureHook(Object.freeze({ artifact, path: hookPath, phase, resumed, alreadyCommitted, error }), owner, event.transfer);
|
|
455
|
+
};
|
|
456
|
+
const rejectEarly = async (reason, message) => {
|
|
457
|
+
await this.rejectOffer(socket, event, owner, reason);
|
|
458
|
+
sentAdmission = true;
|
|
459
|
+
await reportFailure(fail(reason, message));
|
|
460
|
+
};
|
|
234
461
|
try {
|
|
235
|
-
const metadata = await reader.control(
|
|
236
|
-
|
|
462
|
+
const metadata = await reader.control(controls_js_1.decodeAnyMetadataRecord, this.signal, this.idleTimeout);
|
|
463
|
+
// The decoded record is shape-validated but not yet authenticated; the
|
|
464
|
+
// artifact context carries only its non-secret descriptive fields.
|
|
237
465
|
event = this.transfer(metadata);
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
466
|
+
artifact = this.hookArtifact(metadata, null);
|
|
467
|
+
if ((0, controls_js_1.isTreeMetadata)(metadata))
|
|
468
|
+
(0, tree_manifest_js_1.assertTreeMetadataTransferId)(owner, metadata);
|
|
469
|
+
else
|
|
470
|
+
(0, manifest_js_1.assertMetadataTransferId)(owner, metadata);
|
|
471
|
+
// The transfer ID is authenticated from here on, so it is safe to key
|
|
472
|
+
// the single-owner guard on it.
|
|
473
|
+
if (this.activeTransfers.has(metadata.transferId)) {
|
|
474
|
+
await rejectEarly(errors_js_1.ERRORS.FILE_BUSY, 'Transfer is already in progress');
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
this.activeTransfers.add(metadata.transferId);
|
|
478
|
+
guardedTransfer = metadata.transferId;
|
|
479
|
+
let release = null;
|
|
480
|
+
if (this.releaseMatcher.size > 0) {
|
|
481
|
+
release = this.releaseMatcher.match(metadata.name, metadata.sourceParent);
|
|
482
|
+
if (release === null) {
|
|
483
|
+
await rejectEarly(errors_js_1.ERRORS.INVALID_FILENAME, 'Artifact name does not match a configured pattern');
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
artifact = this.hookArtifact(metadata, release);
|
|
487
|
+
}
|
|
488
|
+
if (this.linkNames.has(metadata.name)) {
|
|
489
|
+
await rejectEarly(errors_js_1.ERRORS.INVALID_FILENAME, 'Artifact name is a configured symlink name');
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
const hookArtifact = artifact;
|
|
493
|
+
const stagingHookPath = (0, controls_js_1.isTreeMetadata)(metadata)
|
|
494
|
+
? _path_1.default.join(this.layout.staging, `${metadata.transferId}.tree`)
|
|
495
|
+
: _path_1.default.join(this.layout.staging, `${metadata.transferId}.part`);
|
|
496
|
+
const finish = async (verified) => {
|
|
497
|
+
phase = 'beforeCommit';
|
|
498
|
+
hookPath = stagingHookPath;
|
|
499
|
+
await this.runHook('beforeCommit', this.hooks.beforeCommit, Object.freeze({
|
|
500
|
+
artifact: hookArtifact,
|
|
501
|
+
path: hookPath,
|
|
502
|
+
resumed,
|
|
503
|
+
alreadyCommitted: false
|
|
504
|
+
}));
|
|
505
|
+
phase = 'commit';
|
|
506
|
+
commitStarted = true;
|
|
507
|
+
await this.commits.commit(verified, {
|
|
508
|
+
retentionManager: this.retention,
|
|
509
|
+
signal: this.signal,
|
|
510
|
+
replaceNames: this.replaceNames,
|
|
511
|
+
release,
|
|
512
|
+
deferPostCommitRetention: this.hooks.afterCommit !== undefined
|
|
513
|
+
});
|
|
514
|
+
this.emitSafe('commit', {
|
|
244
515
|
...event,
|
|
245
516
|
fingerprint: fingerprint(owner),
|
|
246
|
-
status: '
|
|
247
|
-
reason
|
|
517
|
+
status: 'succeeded'
|
|
248
518
|
});
|
|
519
|
+
commitSucceeded = true;
|
|
520
|
+
hookPath = _path_1.default.join(this.layout.root, metadata.name);
|
|
521
|
+
await this.retireQuietly(verified.transferId, owner);
|
|
522
|
+
await this.reconcileAfterCommit();
|
|
523
|
+
phase = 'afterCommit';
|
|
524
|
+
if (this.hooks.afterCommit !== undefined) {
|
|
525
|
+
this.owePostCommitRetention(metadata.transferId);
|
|
526
|
+
}
|
|
527
|
+
await this.runHook('afterCommit', this.hooks.afterCommit, Object.freeze({
|
|
528
|
+
artifact: hookArtifact,
|
|
529
|
+
path: hookPath,
|
|
530
|
+
resumed,
|
|
531
|
+
alreadyCommitted: false
|
|
532
|
+
}));
|
|
533
|
+
hooksFinished = true;
|
|
534
|
+
await this.settleDeferredRetention(metadata.transferId, owner);
|
|
535
|
+
finalStarted = true;
|
|
536
|
+
await (0, direct_wire_js_1.writeFinal)(socket, { v: 1, status: 'COMMITTED' }, { signal: this.signal, timeout: this.idleTimeout });
|
|
537
|
+
};
|
|
538
|
+
const payloadBytes = this.payloadBytes(metadata);
|
|
539
|
+
if (payloadBytes > this.maxFileBytes || this.activeUploads.size >= this.maxActiveUploads) {
|
|
540
|
+
if (payloadBytes > this.maxFileBytes) {
|
|
541
|
+
await rejectEarly(errors_js_1.ERRORS.FILE_TOO_LARGE, 'File exceeds the maximum size');
|
|
542
|
+
}
|
|
543
|
+
else {
|
|
544
|
+
await rejectEarly(errors_js_1.ERRORS.ACTIVE_UPLOAD_LIMIT, 'Active upload capacity exceeded');
|
|
545
|
+
}
|
|
249
546
|
return;
|
|
250
547
|
}
|
|
251
548
|
const inspected = await this.commits.inspect(metadata.name, {
|
|
252
549
|
name: metadata.name,
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
550
|
+
kind: this.artifactKind(metadata),
|
|
551
|
+
size: payloadBytes,
|
|
552
|
+
digest: b4a_1.default.from(this.payloadDigest(metadata), 'hex'),
|
|
553
|
+
...((0, controls_js_1.isTreeMetadata)(metadata) ? { entryCount: metadata.entryCount } : {}),
|
|
554
|
+
transferId: b4a_1.default.from(metadata.transferId, 'hex'),
|
|
555
|
+
release
|
|
256
556
|
}, { replaceNames: this.replaceNames });
|
|
257
557
|
if (inspected.status === 'ALREADY_COMMITTED') {
|
|
558
|
+
alreadyCommitted = true;
|
|
559
|
+
hookPath = _path_1.default.join(this.layout.root, metadata.name);
|
|
560
|
+
await this.reconcileAfterCommit();
|
|
561
|
+
phase = 'afterCommit';
|
|
562
|
+
await this.runHook('afterCommit', this.hooks.afterCommit, Object.freeze({
|
|
563
|
+
artifact: hookArtifact,
|
|
564
|
+
path: hookPath,
|
|
565
|
+
resumed: false,
|
|
566
|
+
alreadyCommitted: true
|
|
567
|
+
}));
|
|
568
|
+
hooksFinished = true;
|
|
569
|
+
// Only a transfer this process committed can still owe a deferred
|
|
570
|
+
// pass; an ordinary duplicate offer never starts one.
|
|
571
|
+
if (this.pendingAfterCommit.has(metadata.transferId)) {
|
|
572
|
+
await this.settleDeferredRetention(metadata.transferId, owner);
|
|
573
|
+
}
|
|
258
574
|
await (0, direct_wire_js_1.writeAdmission)(socket, { v: 1, status: 'ALREADY_COMMITTED' }, { signal: this.signal, timeout: this.idleTimeout });
|
|
259
575
|
sentAdmission = true;
|
|
260
576
|
this.emitSafe('offer', {
|
|
@@ -265,25 +581,22 @@ class Server extends EventEmitter {
|
|
|
265
581
|
return;
|
|
266
582
|
}
|
|
267
583
|
if (inspected.status === 'FILE_EXISTS') {
|
|
268
|
-
|
|
584
|
+
await rejectEarly(errors_js_1.ERRORS.FILE_EXISTS, 'Destination already exists');
|
|
585
|
+
return;
|
|
269
586
|
}
|
|
270
587
|
if (this.activeUploads.size >= this.maxActiveUploads) {
|
|
271
|
-
await (
|
|
272
|
-
sentAdmission = true;
|
|
273
|
-
this.emitSafe('offer', {
|
|
274
|
-
...event,
|
|
275
|
-
fingerprint: fingerprint(owner),
|
|
276
|
-
status: 'rejected',
|
|
277
|
-
reason: errors_js_1.ERRORS.ACTIVE_UPLOAD_LIMIT
|
|
278
|
-
});
|
|
588
|
+
await rejectEarly(errors_js_1.ERRORS.ACTIVE_UPLOAD_LIMIT, 'Active upload capacity exceeded');
|
|
279
589
|
return;
|
|
280
590
|
}
|
|
281
591
|
this.activeUploads.add(socket);
|
|
282
592
|
const admission = await this.sessions.admit(owner, metadata);
|
|
283
593
|
current.transfer = metadata.transferId;
|
|
594
|
+
resumed = admission.status !== 'ACCEPT';
|
|
284
595
|
if (admission.status === 'VERIFIED') {
|
|
285
596
|
await (0, direct_wire_js_1.writeAdmission)(socket, { v: 1, status: 'VERIFIED' }, { signal: this.signal, timeout: this.idleTimeout });
|
|
286
597
|
sentAdmission = true;
|
|
598
|
+
phase = 'verification';
|
|
599
|
+
hookPath = stagingHookPath;
|
|
287
600
|
this.emitSafe('verification', {
|
|
288
601
|
...event,
|
|
289
602
|
fingerprint: fingerprint(owner),
|
|
@@ -297,20 +610,7 @@ class Server extends EventEmitter {
|
|
|
297
610
|
status: 'succeeded'
|
|
298
611
|
});
|
|
299
612
|
verificationSucceeded = true;
|
|
300
|
-
|
|
301
|
-
await this.commits.commit(verified, {
|
|
302
|
-
retentionManager: this.retention,
|
|
303
|
-
signal: this.signal,
|
|
304
|
-
replaceNames: this.replaceNames
|
|
305
|
-
});
|
|
306
|
-
this.emitSafe('commit', {
|
|
307
|
-
...event,
|
|
308
|
-
fingerprint: fingerprint(owner),
|
|
309
|
-
status: 'succeeded'
|
|
310
|
-
});
|
|
311
|
-
commitSucceeded = true;
|
|
312
|
-
await this.retireQuietly(b4a_1.default.from(metadata.transferId, 'hex'), owner);
|
|
313
|
-
await (0, direct_wire_js_1.writeFinal)(socket, { v: 1, status: 'COMMITTED' }, { signal: this.signal, timeout: this.idleTimeout });
|
|
613
|
+
await finish(verified);
|
|
314
614
|
return;
|
|
315
615
|
}
|
|
316
616
|
await (0, direct_wire_js_1.writeAdmission)(socket, admission.status === 'ACCEPT'
|
|
@@ -322,6 +622,8 @@ class Server extends EventEmitter {
|
|
|
322
622
|
prefixSha256: b4a_1.default.toString(admission.prefixSha256, 'hex')
|
|
323
623
|
}, { signal: this.signal, timeout: this.idleTimeout });
|
|
324
624
|
sentAdmission = true;
|
|
625
|
+
phase = 'transfer';
|
|
626
|
+
hookPath = _path_1.default.join(this.layout.staging, `${metadata.transferId}.tar.part`);
|
|
325
627
|
this.emitSafe('offer', {
|
|
326
628
|
...event,
|
|
327
629
|
fingerprint: fingerprint(owner),
|
|
@@ -360,6 +662,7 @@ class Server extends EventEmitter {
|
|
|
360
662
|
}, this.signal, this.idleTimeout);
|
|
361
663
|
await flush();
|
|
362
664
|
await reader.requireEnd(this.signal, this.idleTimeout);
|
|
665
|
+
phase = 'verification';
|
|
363
666
|
this.emitSafe('verification', {
|
|
364
667
|
...event,
|
|
365
668
|
fingerprint: fingerprint(owner),
|
|
@@ -373,20 +676,7 @@ class Server extends EventEmitter {
|
|
|
373
676
|
status: 'succeeded'
|
|
374
677
|
});
|
|
375
678
|
verificationSucceeded = true;
|
|
376
|
-
|
|
377
|
-
await this.commits.commit(verified, {
|
|
378
|
-
retentionManager: this.retention,
|
|
379
|
-
signal: this.signal,
|
|
380
|
-
replaceNames: this.replaceNames
|
|
381
|
-
});
|
|
382
|
-
this.emitSafe('commit', {
|
|
383
|
-
...event,
|
|
384
|
-
fingerprint: fingerprint(owner),
|
|
385
|
-
status: 'succeeded'
|
|
386
|
-
});
|
|
387
|
-
commitSucceeded = true;
|
|
388
|
-
await this.retireQuietly(verified.transferId, owner);
|
|
389
|
-
await (0, direct_wire_js_1.writeFinal)(socket, { v: 1, status: 'COMMITTED' }, { signal: this.signal, timeout: this.idleTimeout });
|
|
679
|
+
await finish(verified);
|
|
390
680
|
}
|
|
391
681
|
catch (error) {
|
|
392
682
|
const reason = codeOf(error);
|
|
@@ -415,7 +705,7 @@ class Server extends EventEmitter {
|
|
|
415
705
|
if (!sentAdmission) {
|
|
416
706
|
await (0, direct_wire_js_1.writeAdmission)(socket, { v: 1, status: 'REJECTED', code: reason }, { timeout: this.idleTimeout });
|
|
417
707
|
}
|
|
418
|
-
else if (!
|
|
708
|
+
else if (!finalStarted) {
|
|
419
709
|
await (0, direct_wire_js_1.writeFinal)(socket, { v: 1, status: 'FAILED', code: reason }, { timeout: this.idleTimeout });
|
|
420
710
|
}
|
|
421
711
|
}
|
|
@@ -424,12 +714,59 @@ class Server extends EventEmitter {
|
|
|
424
714
|
socket.destroy();
|
|
425
715
|
}
|
|
426
716
|
catch { }
|
|
717
|
+
this.activeUploads.delete(socket);
|
|
718
|
+
if (!hooksFinished)
|
|
719
|
+
await reportFailure((0, hooks_js_1.callbackError)(error));
|
|
427
720
|
}
|
|
428
721
|
finally {
|
|
429
722
|
this.activeUploads.delete(socket);
|
|
723
|
+
releaseTransfer();
|
|
430
724
|
reader.closeReader();
|
|
431
725
|
}
|
|
432
726
|
}
|
|
727
|
+
/**
|
|
728
|
+
* Records that a transfer still owes its deferred post-commit retention
|
|
729
|
+
* pass, keeping at most `MAX_OWED_POST_COMMIT_RETENTION` entries.
|
|
730
|
+
*
|
|
731
|
+
* `Set` preserves insertion order, so the oldest owed transfer is evicted
|
|
732
|
+
* first and eviction is deterministic. An evicted transfer loses nothing
|
|
733
|
+
* durable: its artifact is already committed and its rotation falls back to
|
|
734
|
+
* the next startup, scheduled, or commit-triggered pass.
|
|
735
|
+
*/
|
|
736
|
+
owePostCommitRetention(transferId) {
|
|
737
|
+
if (this.pendingAfterCommit.has(transferId))
|
|
738
|
+
return;
|
|
739
|
+
if (this.pendingAfterCommit.size >= exports.MAX_OWED_POST_COMMIT_RETENTION) {
|
|
740
|
+
const oldest = this.pendingAfterCommit.values().next();
|
|
741
|
+
if (!oldest.done)
|
|
742
|
+
this.pendingAfterCommit.delete(oldest.value);
|
|
743
|
+
}
|
|
744
|
+
this.pendingAfterCommit.add(transferId);
|
|
745
|
+
}
|
|
746
|
+
/**
|
|
747
|
+
* Runs the post-commit retention pass that a configured `afterCommit` hook
|
|
748
|
+
* deferred, only after that hook succeeded, and then stops owing it. The
|
|
749
|
+
* retention manager already reports its own failures as non-fatal, so
|
|
750
|
+
* nothing here can fail the upload; the pass is attempted exactly once per
|
|
751
|
+
* successful hook, which is why the transfer is cleared either way.
|
|
752
|
+
*/
|
|
753
|
+
async settleDeferredRetention(transferId, owner) {
|
|
754
|
+
if (this.hooks.afterCommit === undefined)
|
|
755
|
+
return;
|
|
756
|
+
try {
|
|
757
|
+
if (this.retention)
|
|
758
|
+
await this.retention.afterCommit();
|
|
759
|
+
}
|
|
760
|
+
catch (error) {
|
|
761
|
+
this.logger.warn('Post-commit retention failed', {
|
|
762
|
+
fingerprint: owner ? fingerprint(owner) : 'invalid',
|
|
763
|
+
reason: codeOf(error)
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
finally {
|
|
767
|
+
this.pendingAfterCommit.delete(transferId);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
433
770
|
/**
|
|
434
771
|
* Retires a committed session without letting cleanup undo a durable commit.
|
|
435
772
|
*
|
|
@@ -466,6 +803,12 @@ class Server extends EventEmitter {
|
|
|
466
803
|
storage: this.storage,
|
|
467
804
|
logger: this.logger
|
|
468
805
|
});
|
|
806
|
+
if (this.symlinks.length > 0) {
|
|
807
|
+
this.links = new link_store_js_1.LinkStore({
|
|
808
|
+
layout: this.layout,
|
|
809
|
+
storage: this.storage
|
|
810
|
+
});
|
|
811
|
+
}
|
|
469
812
|
this.emitSafe('recovery', { status: 'started' });
|
|
470
813
|
await (0, recovery_js_1.prepareStorageRecovery)({
|
|
471
814
|
layout: this.layout,
|
|
@@ -496,6 +839,9 @@ class Server extends EventEmitter {
|
|
|
496
839
|
sessionStore: this.sessions,
|
|
497
840
|
commitStore: this.commits,
|
|
498
841
|
maxAge: this.maxAge,
|
|
842
|
+
maxCount: this.maxCount,
|
|
843
|
+
maxVersions: this.maxVersions,
|
|
844
|
+
versionGranularity: this.versionGranularity,
|
|
499
845
|
maxStorageBytes: this.maxStorageBytes,
|
|
500
846
|
resumeTtl: this.resumeTtl,
|
|
501
847
|
cleanupInterval: this.cleanupInterval,
|
|
@@ -504,6 +850,9 @@ class Server extends EventEmitter {
|
|
|
504
850
|
isSessionActive: (session) => [...this.active.values()].some((active) => active.transfer === session.id),
|
|
505
851
|
hasActiveUploads: () => this.activeUploads.size > 0,
|
|
506
852
|
isPinned: (record) => this.replaceNames.has(record.name),
|
|
853
|
+
managedLinkNames: () => this.linkNames,
|
|
854
|
+
reconcileLinks: (records) => this.reconcileLinksUnlocked(records),
|
|
855
|
+
linkPinsFallback: (records) => this.linkPinsFallbackUnlocked(records),
|
|
507
856
|
logger: this.logger,
|
|
508
857
|
onEvent: ({ type, ...event }) => this.emitSafe(type, event)
|
|
509
858
|
});
|
|
@@ -550,8 +899,11 @@ class Server extends EventEmitter {
|
|
|
550
899
|
this.transport = null;
|
|
551
900
|
await Promise.allSettled([...this.receives]);
|
|
552
901
|
this.active.clear();
|
|
902
|
+
this.activeTransfers.clear();
|
|
903
|
+
this.pendingAfterCommit.clear();
|
|
553
904
|
await this.retention?.stop().catch(() => { });
|
|
554
905
|
this.retention = null;
|
|
906
|
+
this.links = null;
|
|
555
907
|
await this.sessions?.close().catch(() => { });
|
|
556
908
|
this.sessions = null;
|
|
557
909
|
if (this.releaseLock)
|