trac-msb 0.2.26 → 0.2.27

package/scripts/hypercore-stallfix/hypercore-11.8.3.mjs

@@ -0,0 +1,348 @@
+// Exact-text patch set for hypercore 11.8.3 lib/replicator.js (OPENMAYHEM-STALLFIX v2).
+// Same logic as hypercore-11.18.3.mjs, written against the 11.8.3 source (older code style, NOT_AVAILABLE has no
+// reason field on the wire in 11.8.3, INVALID_REQUEST does not exist yet).
+// Each edit is [original, replacement]; the original must occur exactly once in the unpatched file.
+// Keep this file free of backticks and dollar-brace sequences inside the edit texts (String.raw templates).
+
+// Upstream c24f37a6 (#911): a close event that already fired cannot detach a later attachment.
+const closedStreamGuard = [
+    String.raw`  attachTo (protomux) {
+    if (this.core.closed) return
+`,
+    String.raw`  attachTo (protomux) {
+    if (this.core.closed) return
+    if (protomux.stream.destroying) return
+`
+]
+
+export default {
+    version: '11.8.3',
+    file: 'lib/replicator.js',
+    originalSha256: '1b4ddcd0d95d804b84a17a21b9076f990b133f78254b5255e5d32f0951cacf15',
+    patchedSha256: '0456d47322a34852e2c9244a0128971d8358fe7575d584e0f025eca3db5f654c',
+    // Only this exact released v1 file is eligible for an in-place upgrade.
+    previousPatchedSha256: '078afd835dc3e9cb0f7ee294a260528f633c05b54f8bf8276dd3d94aef15248b',
+    upgradeEdits: [
+        ['// OPENMAYHEM-STALLFIX v1 (hypercore 11.8.3).', '// OPENMAYHEM-STALLFIX v2 (hypercore 11.8.3).'],
+        ["const STALLFIX = 'openmayhem-stallfix-v1'", "const STALLFIX = 'openmayhem-stallfix-v2'"],
+        closedStreamGuard
+    ],
+    edits: [
+    // 1. Constants + marker.
+        [
+            String.raw`const MAX_RANGES = 64
+`,
+            String.raw`const MAX_RANGES = 64
+
+// OPENMAYHEM-STALLFIX v2 (hypercore 11.8.3). Install-time patch shipped by trac-msb and trac-peer
+// (scripts/hypercore-stallfix/). Fixes the waiting-side replication stall: an in-flight request that a peer
+// never answers is cancelled after STALLFIX_REQUEST_MAX_AGE and asked elsewhere; that peer gets a cool-down;
+// a peer we paused gets a capped NOT_AVAILABLE instead of silence. Wire format unchanged.
+const STALLFIX = 'openmayhem-stallfix-v2'
+const STALLFIX_REQUEST_MAX_AGE = 30000 // ms an in-flight request may wait for its answer
+const STALLFIX_SWEEP_INTERVAL = 5000 // one unref'd timer per replicator, only while requests are in flight
+const STALLFIX_MAX_SCAN = 4096 // request slots looked at per sweep
+const STALLFIX_MAX_EXPIRE = 64 // requests expired per sweep
+const STALLFIX_COOLDOWN_MIN = 30000 // first cool-down of a peer that let a request expire (doubles, capped)
+const STALLFIX_COOLDOWN_MAX = 240000
+const STALLFIX_NOT_AVAILABLE_WINDOW = 60000
+const STALLFIX_NOT_AVAILABLE_MAX = 16 // NOT_AVAILABLE replies per window to a peer we paused; silent beyond
+`
+        ],
+        // 2. Per-peer state.
+        [
+            String.raw`    this.lastExtensionRecv = ''
+
+    replicator._ifAvailable++
+    replicator._active++
+  }
+`,
+            String.raw`    this.lastExtensionRecv = ''
+
+    this._sfTimeouts = 0 // OPENMAYHEM-STALLFIX: cool-down level of this peer
+    this._sfCoolUntil = 0
+    this._sfNaWindow = 0
+    this._sfNaCount = 0
+
+    replicator._ifAvailable++
+    replicator._active++
+  }
+`
+        ],
+        // 3. Responder: reply "no data" (capped) to a peer we paused instead of dropping the request silently.
+        //    Also covers proof === null for an inactive peer (that case threw a TypeError here).
+        [
+            String.raw`    if (!this.isActive() && proof.block !== null) {
+      return
+    }
+`,
+            String.raw`    if (!this.isActive() && (proof === null || proof.block !== null)) {
+      // OPENMAYHEM-STALLFIX: upstream returns silently here and the remote waits forever.
+      this._sfNotAvailable(req.msg.id)
+      return
+    }
+`
+        ],
+        // 4. Peer helpers: capped "no data" and request expiry.
+        [
+            String.raw`  _cancelRequest (req) {
+    if (req.priority === PRIORITY.CANCELLED) return
+`,
+            String.raw`  // OPENMAYHEM-STALLFIX: at most STALLFIX_NOT_AVAILABLE_MAX replies per window, silent beyond the cap.
+  _sfNotAvailable (id) {
+    if (this.removed === true) return
+    const now = Date.now()
+    if (now - this._sfNaWindow >= STALLFIX_NOT_AVAILABLE_WINDOW) {
+      this._sfNaWindow = now
+      this._sfNaCount = 0
+    }
+    if (this._sfNaCount >= STALLFIX_NOT_AVAILABLE_MAX) {
+      this.replicator._sfStats.notAvailableSuppressed++
+      return
+    }
+    this._sfNaCount++
+    this.replicator._sfStats.notAvailableSent++
+    this.wireNoData.send({ request: id })
+  }
+
+  // OPENMAYHEM-STALLFIX: give up on one request this peer did not answer in time. Bookkeeping as for a
+  // "no data" reply (_clearRequest re-queues a wanted block) plus a wire cancel. The id is only reused
+  // after the next roundtrip (as in _cancelRequest), so a late reply finds no request and is dropped
+  // in ondata/onnodata: it is not invalid data and does not pause the peer.
+  _sfExpireRequest (req, now) {
+    if (req.priority === PRIORITY.CANCELLED) return false
+    req.priority = PRIORITY.CANCELLED
+
+    this.inflight--
+    this.replicator._requestDone(req.id, false)
+
+    if (this.roundtripQueue === null) this.roundtripQueue = new RoundtripQueue()
+    this.roundtripQueue.add(req.id)
+    if (this.removed === false) {
+      this.wireCancel.send({ request: req.id })
+      incrementTx(this.stats.wireCancel, this.replicator.stats.wireCancel)
+    }
+
+    if (this._sfCoolUntil <= now) {
+      if (this._sfTimeouts < 8) this._sfTimeouts++
+      const cooldown = this.replicator._sfCooldownMin * 2 ** (this._sfTimeouts - 1)
+      this._sfCoolUntil = now + Math.min(STALLFIX_COOLDOWN_MAX, cooldown)
+    }
+
+    if (this.replicator._manifestPeer === this && this.core.header.manifest === null) {
+      this.replicator._manifestPeer = null
+    }
+
+    this.replicator._clearRequest(this, req)
+    return true
+  }
+
+  _cancelRequest (req) {
+    if (req.priority === PRIORITY.CANCELLED) return
+`
+        ],
+        // 5. Arm the sweep when a request goes in flight.
+        [
+            String.raw`    this.inflight++
+    this.replicator._inflight.add(req)
+`,
+            String.raw`    this.inflight++
+    this.replicator._inflight.add(req)
+    if (this.replicator._sfTimer === null) this.replicator._sfArm() // OPENMAYHEM-STALLFIX
+`
+        ],
+        // 6. Marker on the Replicator class (runtime check: core.replicator.constructor.STALLFIX).
+        [
+            String.raw`  static Peer = Peer // hack to be able to access Peer from outside this module
+`,
+            String.raw`  static Peer = Peer // hack to be able to access Peer from outside this module
+  static STALLFIX = STALLFIX // OPENMAYHEM-STALLFIX marker
+`
+        ],
+        // 7. Replicator state.
+        [
+            String.raw`    this._notDownloadingLinger = notDownloadingLinger
+    this._downloadingTimer = null
+`,
+            String.raw`    this._notDownloadingLinger = notDownloadingLinger
+    this._downloadingTimer = null
+
+    // OPENMAYHEM-STALLFIX state (the _sf* numbers are only changed by tests)
+    this._sfTimer = null
+    this._sfCursor = 0
+    this._sfAllCooling = false
+    this._sfMaxAge = STALLFIX_REQUEST_MAX_AGE
+    this._sfInterval = STALLFIX_SWEEP_INTERVAL
+    this._sfCooldownMin = STALLFIX_COOLDOWN_MIN
+    this._sfStats = { expired: 0, notAvailableSent: 0, notAvailableSuppressed: 0 }
+`
+        ],
+        // 8. Valid block data from a peer ends its cool-down.
+        [
+            String.raw`  _ondata (peer, req, data) {
+    req.elapsed = Date.now() - req.timestamp
+    if (data.block !== null) {
+`,
+            String.raw`  _ondata (peer, req, data) {
+    req.elapsed = Date.now() - req.timestamp
+    if (data.block !== null) {
+      peer._sfTimeouts = 0 // OPENMAYHEM-STALLFIX
+      peer._sfCoolUntil = 0
+`
+        ],
+        // 9. The sweep (bounded work, one timer per replicator).
+        [
+            String.raw`  _maybeUpdate () {
+    return this._upgrade !== null && this._upgrade.inflight.length === 0
+  }
+`,
+            String.raw`  _maybeUpdate () {
+    return this._upgrade !== null && this._upgrade.inflight.length === 0
+  }
+
+  // OPENMAYHEM-STALLFIX: one unref'd timer per replicator, armed by _send, re-armed only while requests are
+  // in flight or a peer is cooling down. Looks at <= STALLFIX_MAX_SCAN slots, expires <= STALLFIX_MAX_EXPIRE.
+  _sfArm () {
+    if (this.destroyed || this._sfTimer !== null) return
+    this._sfTimer = setTimeout(sfSweepLater, this._sfInterval, this)
+    if (this._sfTimer.unref) this._sfTimer.unref()
+  }
+
+  _sfSweep () {
+    this._sfTimer = null
+    if (this.destroyed) return
+
+    const now = Date.now()
+    const reqs = this._inflight._requests
+    const scan = Math.min(reqs.length, STALLFIX_MAX_SCAN)
+    let live = 0
+    let expired = 0
+
+    for (let i = 0; i < scan && expired < STALLFIX_MAX_EXPIRE; i++) {
+      if (this._sfCursor >= reqs.length) this._sfCursor = 0
+      const req = reqs[this._sfCursor++]
+      if (!req) continue
+      live++
+      if (now - req.timestamp < this._sfMaxAge) continue
+      if (req.peer._sfExpireRequest(req, now) === true) expired++
+    }
+
+    let cooling = 0
+    let usable = 0
+    let cooled = false
+    for (const peer of this.peers) {
+      if (peer._sfCoolUntil !== 0 && peer._sfCoolUntil <= now) {
+        peer._sfCoolUntil = 0
+        cooled = true
+      }
+      if (peer._sfCoolUntil !== 0) cooling++
+      else if (peer.isActive()) usable++
+    }
+    // if no other peer is usable, a cooling peer may still be asked (no deadlock with a single peer)
+    this._sfAllCooling = cooling > 0 && usable === 0
+
+    if (expired > 0) this._sfStats.expired += expired
+    if (expired > 0 || cooled) this.updateAll()
+
+    if (live > 0 || cooling > 0 || scan < reqs.length) this._sfArm()
+  }
+
+  _sfCoolingDown (peer) {
+    return peer._sfCoolUntil !== 0 && this._sfAllCooling === false && peer._sfCoolUntil > Date.now()
+  }
+`
+        ],
+        // 10. Hotswap does not pick a cooling peer.
+        [
+            String.raw`    if (!peer.isActive() || peer.inflight >= maxHotswaps) return
+`,
+            String.raw`    if (!peer.isActive() || peer.inflight >= maxHotswaps || this._sfCoolingDown(peer)) return
+`
+        ],
+        // 11. _updatePeer: skip a cooling peer + backport upstream 8cf8ad9 fix 2 (stale queue entry must not
+        //     consume the peer's update).
+        [
+            String.raw`  _updatePeer (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight()) {
+      return false
+    }
+`,
+            String.raw`  _updatePeer (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight() || this._sfCoolingDown(peer)) {
+      return false
+    }
+`
+        ],
+        [
+            String.raw`    for (const b of blks) {
+      if (b.queued === false || peer._requestBlock(b) === true) {
+        b.queued = false
+        blks.dequeue()
+        return true
+      }
+    }
+`,
+            String.raw`    for (const b of blks) {
+      if (b.queued === false) {
+        // upstream 8cf8ad9 (#896) fix 2, backported by OPENMAYHEM-STALLFIX
+        blks.dequeue()
+        continue
+      }
+      if (peer._requestBlock(b) === true) {
+        b.queued = false
+        blks.dequeue()
+        return true
+      }
+    }
+`
+        ],
+        // 12. _updatePeerNonPrimary: skip a cooling peer.
+        [
+            String.raw`  _updatePeerNonPrimary (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight()) {
+      return false
+    }
+`,
+            String.raw`  _updatePeerNonPrimary (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight() || this._sfCoolingDown(peer)) {
+      return false
+    }
+`
+        ],
+        // 13. destroy() clears the sweep timer.
+        [
+            String.raw`    if (this._downloadingTimer) {
+      clearTimeout(this._downloadingTimer)
+      this._downloadingTimer = null
+    }
+
+    while (this.peers.length) {
+`,
+            String.raw`    if (this._downloadingTimer) {
+      clearTimeout(this._downloadingTimer)
+      this._downloadingTimer = null
+    }
+
+    if (this._sfTimer !== null) {
+      clearTimeout(this._sfTimer) // OPENMAYHEM-STALLFIX
+      this._sfTimer = null
+    }
+
+    while (this.peers.length) {
+`
+        ],
+        // 14. Timer trampoline.
+        [
+            String.raw`function setDownloadingLater (repl, downloading, session) {
+`,
+            String.raw`function sfSweepLater (repl) {
+  repl._sfSweep() // OPENMAYHEM-STALLFIX
+}
+
+function setDownloadingLater (repl, downloading, session) {
+`
+        ],
+        // Upstream #911, in addition to the existing stall fixes.
+        closedStreamGuard
+    ]
+}

package/src/config/config.js

@@ -224,6 +224,11 @@ export class Config {
         return this.#config.confirmationStallTimeout
     }
 
+    get replicationStallTimeout() {
+        if (this.#isOverriden('replicationStallTimeout')) return this.#options.replicationStallTimeout
+        return this.#config.replicationStallTimeout
+    }
+
     get maxPendingRequestsInPendingRequestsService() {
         return this.#config.maxPendingRequestsInPendingRequestsService
     }

package/src/config/env.js

@@ -58,6 +58,7 @@ const configData = {
         validatorHealthCheckInterval: 5 * 60 * 1000, // How often to check validator health (ms)
         validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
         confirmationStallTimeout: 120_000, // Force a fast-forward when the confirmed length has not moved for this long while unconfirmed entries wait (ms, 0 = off)
+        replicationStallTimeout: 600_000, // Close all MSB connections once (max 1 per 10 min) when signed system data stays unapplied and the confirmed length has not moved for this long (ms, 0 = off)
         storesDirectory: 'stores/',
         storeName: 'testnet',
     },
@@ -109,6 +110,7 @@ const configData = {
         validatorHealthCheckInterval: 5 * 60 * 1000, // How often to check validator health (ms)
         validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
         confirmationStallTimeout: 120_000, // Force a fast-forward when the confirmed length has not moved for this long while unconfirmed entries wait (ms, 0 = off)
+        replicationStallTimeout: 600_000, // Close all MSB connections once (max 1 per 10 min) when signed system data stays unapplied and the confirmed length has not moved for this long (ms, 0 = off)
         storesDirectory: 'stores/',
         storeName: 'mainnet',
     },
@@ -160,6 +162,7 @@ const configData = {
         validatorHealthCheckInterval: 1_000, // How often to check validator health (ms)
         validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
         confirmationStallTimeout: 120_000, // Force a fast-forward when the confirmed length has not moved for this long while unconfirmed entries wait (ms, 0 = off)
+        replicationStallTimeout: 600_000, // Close all MSB connections once (max 1 per 10 min) when signed system data stays unapplied and the confirmed length has not moved for this long (ms, 0 = off)
         storesDirectory: 'stores/',
         storeName: 'development',
     }

package/src/core/network/services/ReplicationStallGuard.js

@@ -0,0 +1,177 @@
+const DEFAULT_CHECK_INTERVAL = 15_000;
+const DEFAULT_MIN_GAP = 600_000;
+
+/**
+ * Safety net for the Hypercore replication stall (block requests that a peer never answers; the install-time
+ * patch in scripts/hypercore-stallfix/ is the actual fix). Signal: at every check for `timeout` ms this node
+ * knew signed system data it had not applied (base.core.core.state.length > base.core.length) while its
+ * confirmed length (base.view.core.signedLength) did not move. Then it closes ALL connections of the MSB swarm
+ * once, pinned validator links included, and lets discovery reconnect: the transport reset a restart does,
+ * without a restart. At most once per `minGap`, and only after a new full `timeout` window.
+ *
+ * Never runs on an indexer. It only closes connections: roles, transactions, balances and state are untouched.
+ */
+export class ReplicationStallGuard {
+    #base;
+    #getSwarm;
+    #refresh;
+    #timeout;
+    #interval;
+    #minGap;
+    #now;
+    #log;
+    #timer = null;
+    #mark = null;
+    #stalledSince = null;
+    #lastReconnectAt = null;
+    #lastClosed = 0;
+    #reconnects = 0;
+    #last = null;
+
+    /**
+     * @param {object} base Autobase instance of MSB
+     * @param {() => object|null} getSwarm returns the MSB Hyperswarm (or null)
+     * @param {object} options
+     */
+    constructor(base, getSwarm, {
+        timeout,
+        interval = DEFAULT_CHECK_INTERVAL,
+        minGap = DEFAULT_MIN_GAP,
+        refresh = null,
+        now = Date.now,
+        log = console.log
+    } = {}) {
+        this.#base = base;
+        this.#getSwarm = getSwarm;
+        this.#refresh = refresh;
+        this.#timeout = timeout;
+        this.#interval = Number.isSafeInteger(timeout) && timeout > 0 ? Math.min(interval, timeout) : interval;
+        this.#minGap = minGap;
+        this.#now = now;
+        this.#log = log;
+    }
+
+    get enabled() {
+        return Number.isSafeInteger(this.#timeout) && this.#timeout > 0;
+    }
+
+    start() {
+        if (!this.enabled || this.#timer !== null) return;
+        this.#timer = setInterval(() => { this.check(); }, this.#interval);
+        this.#timer.unref?.();
+    }
+
+    stop() {
+        if (this.#timer !== null) clearInterval(this.#timer);
+        this.#timer = null;
+    }
+
+    status() {
+        const last = this.#last;
+        let state = 'ok';
+        if (this.#stalledSince !== null) state = 'stalled';
+        else if (this.#mark !== null) state = 'behind';
+        return {
+            enabled: this.enabled,
+            state,
+            confirmedLength: last?.confirmed ?? null,
+            systemLength: last?.systemLength ?? null,
+            systemSignedLength: last?.systemSignedLength ?? null,
+            behindSince: this.#mark?.since ?? null,
+            stalledSince: this.#stalledSince,
+            reconnects: this.#reconnects,
+            lastReconnectAt: this.#lastReconnectAt,
+            lastReconnectClosed: this.#lastClosed
+        };
+    }
+
+    #emit(event, fields) {
+        try { this.#log(`[MSB] ${JSON.stringify({ event, ...fields })}`); } catch {}
+    }
+
+    #sample() {
+        const base = this.#base;
+        const confirmed = base?.view?.core?.signedLength;
+        const systemLength = base?.core?.length;
+        const systemSignedLength = base?.core?.core?.state?.length;
+        if (![confirmed, systemLength, systemSignedLength].every(Number.isSafeInteger)) return null;
+        return { confirmed, systemLength, systemSignedLength };
+    }
+
+    /** @returns {boolean} true when the connections were closed */
+    check() {
+        const base = this.#base;
+        if (!this.enabled || !base || base.closing || base.closed || base.isIndexer) {
+            this.#mark = null;
+            return false;
+        }
+        const s = this.#sample();
+        if (s === null) return false;
+        this.#last = s;
+
+        const now = this.#now();
+        const ahead = s.systemSignedLength > s.systemLength;
+        const moved = this.#mark !== null && this.#mark.confirmed !== s.confirmed;
+
+        if (this.#stalledSince !== null && (!ahead || moved)) {
+            this.#emit('msb_replication_recovered', {
+                confirmedLength: s.confirmed, systemLength: s.systemLength,
+                systemSignedLength: s.systemSignedLength, stalledMs: now - this.#stalledSince
+            });
+            this.#stalledSince = null;
+        }
+        if (!ahead) {
+            this.#mark = null;
+            return false;
+        }
+        if (this.#mark === null || moved || now < this.#mark.since) {
+            this.#mark = { confirmed: s.confirmed, since: now };
+            return false;
+        }
+
+        const elapsed = now - this.#mark.since;
+        if (elapsed < this.#timeout) return false;
+
+        if (this.#stalledSince === null) {
+            this.#stalledSince = this.#mark.since;
+            this.#emit('msb_replication_stalled', {
+                confirmedLength: s.confirmed, systemLength: s.systemLength,
+                systemSignedLength: s.systemSignedLength, stalledMs: elapsed
+            });
+        }
+        if (this.#lastReconnectAt !== null && now - this.#lastReconnectAt < this.#minGap) return false;
+
+        const closed = this.#reconnectAll();
+        this.#lastReconnectAt = now;
+        this.#lastClosed = closed;
+        this.#reconnects++;
+        this.#emit('msb_replication_reconnect_all', {
+            closedConnections: closed, confirmedLength: s.confirmed, systemLength: s.systemLength,
+            systemSignedLength: s.systemSignedLength, stalledMs: now - this.#stalledSince, reconnects: this.#reconnects
+        });
+        // the next action needs a new full window
+        this.#mark = { confirmed: s.confirmed, since: now };
+        return true;
+    }
+
+    #reconnectAll() {
+        const swarm = this.#getSwarm?.();
+        if (!swarm || swarm.destroyed || swarm.suspended || !swarm.connections) return 0;
+        let closed = 0;
+        for (const connection of [...swarm.connections]) {
+            if (connection.destroyed || connection.destroying) continue;
+            try {
+                connection.destroy();
+                closed++;
+            } catch {}
+        }
+        if (this.#refresh !== null) {
+            try {
+                Promise.resolve(this.#refresh()).catch(() => {});
+            } catch {}
+        }
+        return closed;
+    }
+}
+
+export default ReplicationStallGuard;

package/src/index.js

@@ -29,6 +29,8 @@ import {safeDecodeApplyOperation, safeEncodeApplyOperation} from "./utils/protob
 import PartialTransactionValidator from "./core/network/protocols/shared/validators/PartialTransactionValidator.js";
 import PartialTransferValidator from "./core/network/protocols/shared/validators/PartialTransferValidator.js";
 import { BroadcastError, ValidationError } from "./utils/errors.js";
+import { ReplicationStallGuard } from "./core/network/services/ReplicationStallGuard.js";
+import { hypercoreStallfixLine, hypercoreStallfixStatus } from "./utils/hypercoreStallfix.js";
 
 export class MainSettlementBus extends ReadyResource {
     #store;
@@ -37,6 +39,7 @@ export class MainSettlementBus extends ReadyResource {
     #state;
     #isClosing = false;
     #config
+    #replicationGuard = null;
 
     /**
      * @param {import("./config/config.js").Config} config
@@ -87,6 +90,13 @@ export class MainSettlementBus extends ReadyResource {
             this.#wallet,
         );
 
+        console.log(hypercoreStallfixLine('MSB', this.#state.base.core));
+        this.#replicationGuard = new ReplicationStallGuard(this.#state.base, () => this.#network?.swarm ?? null, {
+            timeout: this.#config.replicationStallTimeout,
+            refresh: () => this.#network?.swarm?.status?.(this.#config.channel)?.refresh?.()
+        });
+        this.#replicationGuard.start();
+
         const adminEntry = await this.#state.getAdminEntry();
         await this.#setUpRoleAutomatically(adminEntry);
 
@@ -106,6 +116,7 @@ export class MainSettlementBus extends ReadyResource {
         console.log("Closing everything gracefully... This may take a moment.");
 
         this.#isClosing = true;
+        this.#replicationGuard?.stop();
         await this.#network.close();
 
         await sleep(100);
@@ -121,6 +132,18 @@ export class MainSettlementBus extends ReadyResource {
         await sleep(100);
     }
 
+    /**
+     * Replication health: whether the loaded Hypercore carries the stall fix, and the stall guard state
+     * ('ok' | 'behind' | 'stalled'). Read-only.
+     */
+    replicationStatus() {
+        return {
+            hypercoreStallfix: hypercoreStallfixStatus(this.#state?.base?.core, this.#store),
+            stall: this.#replicationGuard?.status() ?? null,
+            connections: this.#network?.swarm?.connections?.size ?? 0
+        };
+    }
+
     async destroy() {
         return await this.close();
     }
@@ -316,6 +339,11 @@ export class MainSettlementBus extends ReadyResource {
             console.log("---------- flags ----------");
             console.log(`isIndexer: ${this.#state?.isIndexer?.() ?? "unset"}`);
             console.log(`isWriter: ${this.#state?.isWritable?.() ?? "unset"}`);
+            const replication = this.replicationStatus();
+            console.log("---------- replication ----------");
+            console.log(`hypercoreStallfix: ${replication.hypercoreStallfix.patched ? replication.hypercoreStallfix.marker : "NOT ACTIVE"}` +
+                ` (expired requests: ${replication.hypercoreStallfix.expiredRequests ?? "n/a"}, timed pauses: ${replication.hypercoreStallfix.invalidOperationPauses ?? "n/a"})`);
+            console.log(`replicationStall: ${replication.stall?.state ?? "off"} (reconnects: ${replication.stall?.reconnects ?? 0})`);
         } catch (error) {
             console.error("Error during DAG monitoring:", error.message);
         }

package/src/utils/hypercoreStallfix.js

@@ -0,0 +1,53 @@
+// Runtime view of the install-time Hypercore patch (scripts/hypercore-stallfix/). It looks at the Replicator class
+// of a live core, so it reports the Hypercore copy that is actually loaded, not the one npm resolution suggests.
+export const HYPERCORE_STALLFIX_MARKER = 'openmayhem-stallfix-v2';
+
+/**
+ * @param {object} core a Hypercore session (for MSB: state.base.core, the system view)
+ * @param {object|null} store optional Corestore; when given, request counters are summed over its open cores
+ */
+export function hypercoreStallfixStatus(core, store = null) {
+    const Replicator = core?.core?.replicator?.constructor ?? null;
+    const marker = typeof Replicator?.STALLFIX === 'string' ? Replicator.STALLFIX : null;
+    const status = {
+        patched: marker === HYPERCORE_STALLFIX_MARKER,
+        marker,
+        expiredRequests: null,
+        notAvailableSent: null,
+        notAvailableSuppressed: null,
+        invalidOperationPauses: null,
+        invalidOperationPermanent: null
+    };
+    if (marker === null) return status;
+
+    let expired = 0;
+    let sent = 0;
+    let suppressed = 0;
+    let opPauses = 0;
+    let opPermanent = 0;
+    const cores = store?.cores?.map;
+    const list = cores && typeof cores.values === 'function' ? cores.values() : [core.core];
+    for (const c of list) {
+        const stats = c?.replicator?._sfStats;
+        if (!stats) continue;
+        expired += stats.expired;
+        sent += stats.notAvailableSent;
+        suppressed += stats.notAvailableSuppressed;
+        opPauses += stats.invalidOperationPauses ?? 0; // 11.18.3 only (11.8.3 never pauses on INVALID_OPERATION)
+        opPermanent += stats.invalidOperationPermanent ?? 0;
+    }
+    status.expiredRequests = expired;
+    status.notAvailableSent = sent;
+    status.notAvailableSuppressed = suppressed;
+    status.invalidOperationPauses = opPauses;
+    status.invalidOperationPermanent = opPermanent;
+    return status;
+}
+
+/** One startup line for logs/alerts. */
+export function hypercoreStallfixLine(label, core) {
+    const { patched, marker } = hypercoreStallfixStatus(core);
+    if (patched) return `[${label}] Hypercore replication stall fix: active (${marker}); closed-stream guard #911 active`;
+    return `[${label}] WARNING: Hypercore replication stall fix NOT active in the loaded hypercore` +
+        ` (marker: ${marker ?? 'none'}). Reinstall without --ignore-scripts or run scripts/hypercore-stallfix/apply.mjs.`;
+}