swarm-deploy 0.3.1 → 0.4.2

package/dist/server.js

@@ -46,15 +46,21 @@ exports.MAX_ACTIVE_UPLOADS = MAX_ACTIVE_UPLOADS;
 const FINGERPRINT_LENGTH = 12;
 exports.FINGERPRINT_LENGTH = FINGERPRINT_LENGTH;
 const TAR_DURABILITY_BATCH_BYTES = 1024 * 1024;
+/** How long a connection may linger so its peer can read a `HOOK_FAILED` answer before it is destroyed. */
+const ANSWER_CLOSE_GRACE = 5_000;
 /**
  * How many transfers may owe a deferred post-commit retention pass at once.
  *
- * The set grows only when an `afterCommit` hook fails after a durable commit,
+ * The set grows only when an `afterCommit` or `afterLink` hook fails after a durable commit,
  * and entries are removed as soon as a retry succeeds, so reaching the bound
  * means a persistently failing hook. Capping it at the connection ceiling
  * keeps a broken deployment step from growing server memory without limit.
  */
 exports.MAX_OWED_POST_COMMIT_RETENTION = MAX_CONNECTIONS;
+function releaseLinkHooks(pending) {
+    for (const item of pending)
+        item.turn.release();
+}
 function fail(code, message, cause = null) {
     return new errors_js_1.SwarmDeployError(code, message, cause);
 }
@@ -64,6 +70,12 @@ function codeOf(error) {
         ? error.code
         : errors_js_1.ERRORS.PROTOCOL_INVALID;
 }
+function frozenRelease(release) {
+    return Object.freeze({
+        series: release.series,
+        ...(release.version === undefined ? {} : { version: release.version })
+    });
+}
 function positive(value, name, max = Number.MAX_SAFE_INTEGER) {
     if (!Number.isSafeInteger(value) || value <= 0 || value > max) {
         throw fail(errors_js_1.ERRORS.PROTOCOL_INVALID, `Invalid ${name}`);
@@ -136,7 +148,7 @@ class Server extends EventEmitter {
     activeUploads = new Set();
     /**
      * Transfer IDs this process committed but whose deferred post-commit
-     * retention is still owed because `afterCommit` has not yet succeeded. Only
+     * retention is still owed because `afterCommit` or `afterLink` has not yet succeeded. Only
      * these may start a retention pass from an already-committed retry, so an
      * ordinary duplicate offer cannot be used to force repeated full passes.
      * The set is in-memory only and bounded by `MAX_OWED_POST_COMMIT_RETENTION`;
@@ -152,6 +164,13 @@ class Server extends EventEmitter {
      * and its hooks concurrently against one staging and commit identity.
      */
     activeTransfers = new Set();
+    /**
+     * One first-in, first-out line of `afterLink` calls per link name. A place is
+     * reserved while the root lease orders the link decision, and the hook runs
+     * after the lease is released, so a link's hooks start in decision order
+     * and never overlap, while hook code never runs under the lease.
+     */
+    linkHookTurns = new hooks_js_1.HookTurns();
     receives = new Set();
     layout = null;
     sessions = null;
@@ -305,25 +324,40 @@ class Server extends EventEmitter {
             transferId: metadata.transferId,
             ...((0, controls_js_1.isTreeMetadata)(metadata) ? { entryCount: metadata.entryCount } : {}),
             ...(metadata.sourceParent === undefined ? {} : { sourceParent: metadata.sourceParent }),
-            ...(release === null
-                ? {}
-                : {
-                    release: Object.freeze({
-                        series: release.series,
-                        ...(release.version === undefined ? {} : { version: release.version })
-                    })
-                })
+            ...(release === null ? {} : { release: frozenRelease(release) })
+        });
+    }
+    /** Describes a durably committed artifact from its commit record, as a link hook sees it. */
+    recordArtifact(record) {
+        return Object.freeze({
+            name: record.name,
+            kind: (0, commit_journal_js_1.commitRecordKind)(record),
+            size: record.size,
+            sha256: record.sha256,
+            transferId: record.transferId,
+            ...(record.entryCount === undefined ? {} : { entryCount: record.entryCount }),
+            ...(record.release === undefined ? {} : { release: frozenRelease(record.release) })
+        });
+    }
+    hookLink(link, mode) {
+        return Object.freeze({
+            name: link.name,
+            target: link.target,
+            mode,
+            path: _path_1.default.join(this.layout.root, link.name),
+            targetPath: _path_1.default.join(this.layout.root, link.target)
         });
     }
     /**
      * Computes and converges the desired links from durable commit records.
      * Called with the root lease already held.
      */
-    async reconcileLinksUnlocked(records) {
+    async convergeLinksUnlocked(records) {
         const desired = (0, symlinks_js_1.selectDesiredLinks)(this.symlinks, records);
-        if (!this.links)
-            return new Set(desired.map((link) => link.transferId));
-        await this.links.reconcile(desired, this.linkNames, {
+        if (!this.links) {
+            return { pins: new Set(desired.map((link) => link.transferId)), desired, result: null };
+        }
+        const result = await this.links.reconcile(desired, this.linkNames, {
             managedArtifactNames: new Set(records.map((record) => record.name))
         });
         const ledger = await this.links.list();
@@ -338,7 +372,10 @@ class Server extends EventEmitter {
             else
                 pins.add(link.transferId);
         }
-        return pins;
+        return { pins, desired, result };
+    }
+    async reconcileLinksUnlocked(records) {
+        return (await this.convergeLinksUnlocked(records)).pins;
     }
     /** Pins link targets when reconciliation fails but retention must continue. */
     async linkPinsFallbackUnlocked(records) {
@@ -353,12 +390,37 @@ class Server extends EventEmitter {
         }
         return pins;
     }
-    /** Reconciles after a durable commit; this is the only caller that leases the root. */
-    async reconcileAfterCommit() {
+    /**
+     * Reconciles after a durable commit; this is the only caller that leases the
+     * root for it. When an `afterLink` hook is configured it also returns one
+     * pending call for every automatic link that now designates `artifact`,
+     * whether this reconcile changed it or an earlier attempt already had, so a
+     * retry re-runs a hook that failed. The caller owns the returned places in
+     * the hook lines and must release them.
+     */
+    async reconcileAfterCommit(artifact) {
         if (!this.links || !this.commits)
-            return;
+            return [];
         try {
-            await (0, root_coordinator_js_1.withRootLease)(this.layout.root, async () => this.reconcileLinksUnlocked(await this.commits.list()));
+            return await (0, root_coordinator_js_1.withRootLease)(this.layout.root, async () => {
+                const { desired, result } = await this.convergeLinksUnlocked(await this.commits.list());
+                if (this.hooks.afterLink === undefined)
+                    return [];
+                const unchanged = new Set(result?.unchanged);
+                const pending = desired
+                    .filter((link) => link.transferId === artifact.transferId)
+                    .map((link) => ({
+                    link: this.hookLink(link, 'automatic'),
+                    artifact,
+                    status: unchanged.has(link.name) ? 'UNCHANGED' : 'LINKED',
+                    trigger: 'upload'
+                }));
+                // Reserved last and without a failure point after it, so no place can leak.
+                return pending.map((item) => ({
+                    ...item,
+                    turn: this.linkHookTurns.reserve(item.link.name)
+                }));
+            });
         }
         catch (error) {
             if (error instanceof errors_js_1.SwarmDeployError && error.code === errors_js_1.ERRORS.LINK_CONFLICT)
@@ -366,6 +428,41 @@ class Server extends EventEmitter {
             throw new errors_js_1.SwarmDeployError(errors_js_1.ERRORS.LINK_FAILED, 'Unable to reconcile managed symlinks', error);
         }
     }
+    /**
+     * Runs pending `afterLink` calls one after another, each once every earlier
+     * hook for its link has finished. No storage lease is held while hook code
+     * runs. `begin` is told which call is starting so a caller can attribute a
+     * failure. The first failure stops the rest; the caller releases the places.
+     */
+    async runLinkHooks(pending, owner, begin) {
+        for (const item of pending) {
+            begin(item);
+            if ((await item.turn.ready(this.signal)) === 'aborted')
+                throw (0, abort_js_1.abortError)();
+            try {
+                await this.runHook('afterLink', this.hooks.afterLink, Object.freeze({
+                    link: item.link,
+                    artifact: item.artifact,
+                    status: item.status,
+                    trigger: item.trigger
+                }));
+            }
+            catch (error) {
+                if (error instanceof errors_js_1.SwarmDeployError && error.code === errors_js_1.ERRORS.HOOK_FAILED) {
+                    this.logger.warn('Link hook failed', {
+                        fingerprint: owner ? fingerprint(owner) : 'invalid',
+                        name: item.link.name,
+                        target: item.link.target,
+                        trigger: item.trigger,
+                        status: item.status,
+                        reason: errors_js_1.ERRORS.HOOK_FAILED
+                    });
+                }
+                throw error;
+            }
+            item.turn.release();
+        }
+    }
     setManagedLinkNames(records) {
         this.managedLinkNamesSnapshot = new Set([
             ...this.linkNames,
@@ -382,7 +479,13 @@ class Server extends EventEmitter {
     async receiveLink(socket, reader, owner, request) {
         let emitted = false;
         let resultWriteStarted = false;
-        const emitTerminal = (status, reason) => {
+        // `durable` is set once the link is durable, so a later hook failure can report what the
+        // link did; `failed` is the hook that was running. Held in an object because a closure sets it.
+        const state = {
+            durable: null,
+            failed: null
+        };
+        const emitTerminal = (status, reason, result) => {
             if (emitted)
                 return;
             emitted = true;
@@ -391,7 +494,8 @@ class Server extends EventEmitter {
                 target: request.target,
                 name: request.name,
                 status,
-                ...(reason === undefined ? {} : { reason })
+                ...(reason === undefined ? {} : { reason }),
+                ...(result === undefined ? {} : { result })
             });
         };
         try {
@@ -406,7 +510,7 @@ class Server extends EventEmitter {
             if (this.linkNames.has(request.name)) {
                 throw fail(errors_js_1.ERRORS.LINK_NOT_ALLOWED, 'Manual link name is reserved by an automatic rule');
             }
-            const result = await (0, root_coordinator_js_1.withRootLease)(this.layout.root, async () => {
+            const decided = await (0, root_coordinator_js_1.withRootLease)(this.layout.root, async () => {
                 const records = await this.commits.list();
                 const target = records.find((record) => record.name === request.target && !(0, files_js_1.isReservedHistoryName)(record.name));
                 if (!target)
@@ -428,30 +532,58 @@ class Server extends EventEmitter {
                     automaticLinkNames: this.linkNames
                 });
                 await this.refreshManagedLinkNames();
-                return linked;
+                // The link is durable here. Its hook runs after this lease is released, so a
+                // slow or failing hook never blocks commits, retention, or other links.
+                const pending = this.hooks.afterLink === undefined
+                    ? null
+                    : {
+                        link: this.hookLink(link, 'manual'),
+                        artifact: this.recordArtifact(target),
+                        status: linked.status,
+                        trigger: 'request',
+                        turn: this.linkHookTurns.reserve(link.name)
+                    };
+                return { status: linked.status, pending };
             });
+            state.durable = decided.status;
+            if (decided.pending !== null) {
+                const pending = decided.pending;
+                try {
+                    await this.runLinkHooks([pending], owner, (item) => {
+                        state.failed = item;
+                    });
+                }
+                finally {
+                    pending.turn.release();
+                }
+            }
             resultWriteStarted = true;
-            await (0, direct_wire_js_1.writeLinkResult)(socket, { v: 1, status: result.status }, { signal: this.signal, timeout: this.idleTimeout });
-            emitTerminal(result.status === 'LINKED' ? 'linked' : 'unchanged');
+            await (0, direct_wire_js_1.writeLinkResult)(socket, { v: 1, status: decided.status }, { signal: this.signal, timeout: this.idleTimeout });
+            emitTerminal(decided.status === 'LINKED' ? 'linked' : 'unchanged');
         }
         catch (error) {
             let reason = error instanceof errors_js_1.SwarmDeployError &&
                 (error.code === errors_js_1.ERRORS.LINK_NOT_ALLOWED ||
                     error.code === errors_js_1.ERRORS.LINK_TARGET_NOT_FOUND ||
                     error.code === errors_js_1.ERRORS.LINK_CONFLICT ||
-                    error.code === errors_js_1.ERRORS.LINK_FAILED)
+                    error.code === errors_js_1.ERRORS.LINK_FAILED ||
+                    error.code === errors_js_1.ERRORS.HOOK_FAILED)
                 ? error.code
                 : errors_js_1.ERRORS.LINK_FAILED;
             const rejected = reason === errors_js_1.ERRORS.LINK_NOT_ALLOWED ||
                 reason === errors_js_1.ERRORS.LINK_TARGET_NOT_FOUND ||
                 reason === errors_js_1.ERRORS.LINK_CONFLICT;
+            let answered = false;
             if (!resultWriteStarted) {
                 resultWriteStarted = true;
                 try {
                     await (0, direct_wire_js_1.writeLinkResult)(socket, { v: 1, status: 'FAILED', code: reason }, { timeout: this.idleTimeout });
+                    answered = true;
                 }
                 catch {
-                    reason = errors_js_1.ERRORS.LINK_FAILED;
+                    // A failed hook stays observable here even when the client cannot be told.
+                    if (reason !== errors_js_1.ERRORS.HOOK_FAILED)
+                        reason = errors_js_1.ERRORS.LINK_FAILED;
                 }
             }
             else {
@@ -459,11 +591,61 @@ class Server extends EventEmitter {
             }
             this.logger.warn('Direct link failed', { fingerprint: fingerprint(owner), reason });
             this.emitSafe('failure', { fingerprint: fingerprint(owner), reason });
-            emitTerminal(rejected && reason !== errors_js_1.ERRORS.LINK_FAILED ? 'rejected' : 'failed', reason);
+            emitTerminal(rejected && reason !== errors_js_1.ERRORS.LINK_FAILED ? 'rejected' : 'failed', reason, reason === errors_js_1.ERRORS.HOOK_FAILED && state.durable !== null ? state.durable : undefined);
+            if (answered && reason === errors_js_1.ERRORS.HOOK_FAILED) {
+                this.closeAfterAnswer(socket);
+            }
+            else {
+                try {
+                    socket.destroy();
+                }
+                catch { }
+            }
+            // The client has been answered and the link is already durable; onFailure only observes.
+            if (reason === errors_js_1.ERRORS.HOOK_FAILED && state.failed !== null) {
+                await this.runFailureHook(Object.freeze({
+                    artifact: state.failed.artifact,
+                    path: state.failed.link.path,
+                    phase: 'afterLink',
+                    resumed: false,
+                    alreadyCommitted: true,
+                    error: (0, hooks_js_1.callbackError)(error),
+                    link: state.failed.link
+                }), owner, fingerprint(b4a_1.default.from(state.failed.artifact.transferId, 'hex')));
+            }
+        }
+    }
+    /**
+     * Closes a connection after its `HOOK_FAILED` answer was written. Destroying
+     * the stream right behind the write can reset it before the peer has read the
+     * record, so the caller would see a reset instead of the stable code. The
+     * write side is ended first and the connection is destroyed once the peer
+     * closes it, or after a grace period if it never does. The work that failed
+     * is already durable, so lingering costs only the connection.
+     */
+    closeAfterAnswer(socket) {
+        let timer;
+        const destroy = () => {
+            clearTimeout(timer);
             try {
                 socket.destroy();
             }
             catch { }
+        };
+        try {
+            ;
+            socket.end?.();
+        }
+        catch {
+            destroy();
+            return;
+        }
+        socket.once('close', () => clearTimeout(timer));
+        timer = setTimeout(destroy, ANSWER_CLOSE_GRACE);
+        // A lingering answer must never keep the process alive on its own.
+        if (typeof timer === 'object' && timer !== null && 'unref' in timer) {
+            if (typeof timer.unref === 'function')
+                timer.unref();
         }
     }
     /** Runs a gating callback; only callback exceptions become HOOK_FAILED. */
@@ -540,6 +722,7 @@ class Server extends EventEmitter {
         let artifact = null;
         let phase = 'offer';
         let hookPath = null;
+        let failedLink = null;
         let resumed = false;
         let alreadyCommitted = false;
         let hooksFinished = false;
@@ -566,7 +749,21 @@ class Server extends EventEmitter {
             // finished its own. `onFailure` only observes, so holding the transfer
             // across a slow callback would reject the client's legitimate retry.
             releaseTransfer();
-            await this.runFailureHook(Object.freeze({ artifact, path: hookPath, phase, resumed, alreadyCommitted, error }), owner, event.transfer);
+            await this.runFailureHook(Object.freeze({
+                artifact,
+                path: hookPath,
+                phase,
+                resumed,
+                alreadyCommitted,
+                error,
+                ...(phase === 'afterLink' && failedLink !== null ? { link: failedLink } : {})
+            }), owner, event.transfer);
+        };
+        // Attributes an `afterLink` failure to the link whose hook was starting.
+        const beginLinkHook = (item) => {
+            phase = 'afterLink';
+            failedLink = item.link;
+            hookPath = item.link.path;
         };
         const rejectEarly = async (reason, message) => {
             await this.rejectOffer(socket, event, owner, reason);
@@ -629,7 +826,7 @@ class Server extends EventEmitter {
                     signal: this.signal,
                     replaceNames: this.replaceNames,
                     release,
-                    deferPostCommitRetention: this.hooks.afterCommit !== undefined
+                    deferPostCommitRetention: this.defersPostCommitRetention()
                 });
                 this.emitSafe('commit', {
                     ...event,
@@ -639,17 +836,23 @@ class Server extends EventEmitter {
                 commitSucceeded = true;
                 hookPath = _path_1.default.join(this.layout.root, metadata.name);
                 await this.retireQuietly(verified.transferId, owner);
-                await this.reconcileAfterCommit();
-                phase = 'afterCommit';
-                if (this.hooks.afterCommit !== undefined) {
-                    this.owePostCommitRetention(metadata.transferId);
+                const linkHooks = await this.reconcileAfterCommit(hookArtifact);
+                try {
+                    phase = 'afterCommit';
+                    if (this.defersPostCommitRetention()) {
+                        this.owePostCommitRetention(metadata.transferId);
+                    }
+                    await this.runHook('afterCommit', this.hooks.afterCommit, Object.freeze({
+                        artifact: hookArtifact,
+                        path: hookPath,
+                        resumed,
+                        alreadyCommitted: false
+                    }));
+                    await this.runLinkHooks(linkHooks, owner, beginLinkHook);
+                }
+                finally {
+                    releaseLinkHooks(linkHooks);
                 }
-                await this.runHook('afterCommit', this.hooks.afterCommit, Object.freeze({
-                    artifact: hookArtifact,
-                    path: hookPath,
-                    resumed,
-                    alreadyCommitted: false
-                }));
                 hooksFinished = true;
                 await this.settleDeferredRetention(metadata.transferId, owner);
                 finalStarted = true;
@@ -677,14 +880,20 @@ class Server extends EventEmitter {
             if (inspected.status === 'ALREADY_COMMITTED') {
                 alreadyCommitted = true;
                 hookPath = _path_1.default.join(this.layout.root, metadata.name);
-                await this.reconcileAfterCommit();
-                phase = 'afterCommit';
-                await this.runHook('afterCommit', this.hooks.afterCommit, Object.freeze({
-                    artifact: hookArtifact,
-                    path: hookPath,
-                    resumed: false,
-                    alreadyCommitted: true
-                }));
+                const linkHooks = await this.reconcileAfterCommit(hookArtifact);
+                try {
+                    phase = 'afterCommit';
+                    await this.runHook('afterCommit', this.hooks.afterCommit, Object.freeze({
+                        artifact: hookArtifact,
+                        path: hookPath,
+                        resumed: false,
+                        alreadyCommitted: true
+                    }));
+                    await this.runLinkHooks(linkHooks, owner, beginLinkHook);
+                }
+                finally {
+                    releaseLinkHooks(linkHooks);
+                }
                 hooksFinished = true;
                 // Only a transfer this process committed can still owe a deferred
                 // pass; an ordinary duplicate offer never starts one.
@@ -821,19 +1030,27 @@ class Server extends EventEmitter {
                     reason
                 });
             }
+            let answered = false;
             try {
                 if (!sentAdmission) {
                     await (0, direct_wire_js_1.writeAdmission)(socket, { v: 1, status: 'REJECTED', code: reason }, { timeout: this.idleTimeout });
+                    answered = true;
                 }
                 else if (!finalStarted) {
                     await (0, direct_wire_js_1.writeFinal)(socket, { v: 1, status: 'FAILED', code: reason }, { timeout: this.idleTimeout });
+                    answered = true;
                 }
             }
             catch { }
-            try {
-                socket.destroy();
+            if (answered && reason === errors_js_1.ERRORS.HOOK_FAILED) {
+                this.closeAfterAnswer(socket);
+            }
+            else {
+                try {
+                    socket.destroy();
+                }
+                catch { }
             }
-            catch { }
             this.activeUploads.delete(socket);
             if (!hooksFinished)
                 await reportFailure((0, hooks_js_1.callbackError)(error));
@@ -864,14 +1081,26 @@ class Server extends EventEmitter {
         this.pendingAfterCommit.add(transferId);
     }
     /**
-     * Runs the post-commit retention pass that a configured `afterCommit` hook
-     * deferred, only after that hook succeeded, and then stops owing it. The
-     * retention manager already reports its own failures as non-fatal, so
-     * nothing here can fail the upload; the pass is attempted exactly once per
-     * successful hook, which is why the transfer is cleared either way.
+     * Whether the post-commit retention pass waits for the hooks that follow a
+     * commit. Either one defers it: `afterCommit` so its callback still finds
+     * the artifact, and `afterLink` so this commit's own link reconciliation runs
+     * before this commit's retention pass does. That narrows, but does not remove,
+     * the chance that the link already moved: another commit's retention or
+     * reconciliation can do it first, and the hook then sees `UNCHANGED`.
+     */
+    defersPostCommitRetention() {
+        return this.hooks.afterCommit !== undefined || this.hooks.afterLink !== undefined;
+    }
+    /**
+     * Runs the post-commit retention pass that configured `afterCommit` or
+     * `afterLink` hooks deferred, only after they succeeded, and then stops
+     * owing it. The retention manager already reports its own failures as
+     * non-fatal, so nothing here can fail the upload; the pass is attempted
+     * exactly once per successful hook run, which is why the transfer is cleared
+     * either way.
      */
     async settleDeferredRetention(transferId, owner) {
-        if (this.hooks.afterCommit === undefined)
+        if (!this.defersPostCommitRetention())
             return;
         try {
             if (this.retention)
@@ -910,6 +1139,7 @@ class Server extends EventEmitter {
         try {
             this.layout = (0, layout_js_1.initLayout)(this.storageDir);
             this.releaseLock = await (0, layout_js_1.acquireStorageLock)(this.layout, { storage: this.storage });
+            await (0, layout_js_1.assertNoPendingStorageReset)(this.layout, this.storage);
             this.sessions = new session_store_js_1.SessionStore({
                 layout: this.layout,
                 maxStagingBytes: this.maxStagingBytes,