trac-msb 0.2.24 → 0.2.27

package/scripts/hypercore-stallfix/hypercore-11.8.3.mjs

@@ -0,0 +1,348 @@
+// Exact-text patch set for hypercore 11.8.3 lib/replicator.js (OPENMAYHEM-STALLFIX v2).
+// Same logic as hypercore-11.18.3.mjs, written against the 11.8.3 source (older code style, NOT_AVAILABLE has no
+// reason field on the wire in 11.8.3, INVALID_REQUEST does not exist yet).
+// Each edit is [original, replacement]; the original must occur exactly once in the unpatched file.
+// Keep this file free of backticks and dollar-brace sequences inside the edit texts (String.raw templates).
+
+// Upstream c24f37a6 (#911): a close event that already fired cannot detach a later attachment.
+const closedStreamGuard = [
+    String.raw`  attachTo (protomux) {
+    if (this.core.closed) return
+`,
+    String.raw`  attachTo (protomux) {
+    if (this.core.closed) return
+    if (protomux.stream.destroying) return
+`
+]
+
+export default {
+    version: '11.8.3',
+    file: 'lib/replicator.js',
+    originalSha256: '1b4ddcd0d95d804b84a17a21b9076f990b133f78254b5255e5d32f0951cacf15',
+    patchedSha256: '0456d47322a34852e2c9244a0128971d8358fe7575d584e0f025eca3db5f654c',
+    // Only this exact released v1 file is eligible for an in-place upgrade.
+    previousPatchedSha256: '078afd835dc3e9cb0f7ee294a260528f633c05b54f8bf8276dd3d94aef15248b',
+    upgradeEdits: [
+        ['// OPENMAYHEM-STALLFIX v1 (hypercore 11.8.3).', '// OPENMAYHEM-STALLFIX v2 (hypercore 11.8.3).'],
+        ["const STALLFIX = 'openmayhem-stallfix-v1'", "const STALLFIX = 'openmayhem-stallfix-v2'"],
+        closedStreamGuard
+    ],
+    edits: [
+    // 1. Constants + marker.
+        [
+            String.raw`const MAX_RANGES = 64
+`,
+            String.raw`const MAX_RANGES = 64
+
+// OPENMAYHEM-STALLFIX v2 (hypercore 11.8.3). Install-time patch shipped by trac-msb and trac-peer
+// (scripts/hypercore-stallfix/). Fixes the waiting-side replication stall: an in-flight request that a peer
+// never answers is cancelled after STALLFIX_REQUEST_MAX_AGE and asked elsewhere; that peer gets a cool-down;
+// a peer we paused gets a capped NOT_AVAILABLE instead of silence. Wire format unchanged.
+const STALLFIX = 'openmayhem-stallfix-v2'
+const STALLFIX_REQUEST_MAX_AGE = 30000 // ms an in-flight request may wait for its answer
+const STALLFIX_SWEEP_INTERVAL = 5000 // one unref'd timer per replicator, only while requests are in flight
+const STALLFIX_MAX_SCAN = 4096 // request slots looked at per sweep
+const STALLFIX_MAX_EXPIRE = 64 // requests expired per sweep
+const STALLFIX_COOLDOWN_MIN = 30000 // first cool-down of a peer that let a request expire (doubles, capped)
+const STALLFIX_COOLDOWN_MAX = 240000
+const STALLFIX_NOT_AVAILABLE_WINDOW = 60000
+const STALLFIX_NOT_AVAILABLE_MAX = 16 // NOT_AVAILABLE replies per window to a peer we paused; silent beyond
+`
+        ],
+        // 2. Per-peer state.
+        [
+            String.raw`    this.lastExtensionRecv = ''
+
+    replicator._ifAvailable++
+    replicator._active++
+  }
+`,
+            String.raw`    this.lastExtensionRecv = ''
+
+    this._sfTimeouts = 0 // OPENMAYHEM-STALLFIX: cool-down level of this peer
+    this._sfCoolUntil = 0
+    this._sfNaWindow = 0
+    this._sfNaCount = 0
+
+    replicator._ifAvailable++
+    replicator._active++
+  }
+`
+        ],
+        // 3. Responder: reply "no data" (capped) to a peer we paused instead of dropping the request silently.
+        //    Also covers proof === null for an inactive peer (that case threw a TypeError here).
+        [
+            String.raw`    if (!this.isActive() && proof.block !== null) {
+      return
+    }
+`,
+            String.raw`    if (!this.isActive() && (proof === null || proof.block !== null)) {
+      // OPENMAYHEM-STALLFIX: upstream returns silently here and the remote waits forever.
+      this._sfNotAvailable(req.msg.id)
+      return
+    }
+`
+        ],
+        // 4. Peer helpers: capped "no data" and request expiry.
+        [
+            String.raw`  _cancelRequest (req) {
+    if (req.priority === PRIORITY.CANCELLED) return
+`,
+            String.raw`  // OPENMAYHEM-STALLFIX: at most STALLFIX_NOT_AVAILABLE_MAX replies per window, silent beyond the cap.
+  _sfNotAvailable (id) {
+    if (this.removed === true) return
+    const now = Date.now()
+    if (now - this._sfNaWindow >= STALLFIX_NOT_AVAILABLE_WINDOW) {
+      this._sfNaWindow = now
+      this._sfNaCount = 0
+    }
+    if (this._sfNaCount >= STALLFIX_NOT_AVAILABLE_MAX) {
+      this.replicator._sfStats.notAvailableSuppressed++
+      return
+    }
+    this._sfNaCount++
+    this.replicator._sfStats.notAvailableSent++
+    this.wireNoData.send({ request: id })
+  }
+
+  // OPENMAYHEM-STALLFIX: give up on one request this peer did not answer in time. Bookkeeping as for a
+  // "no data" reply (_clearRequest re-queues a wanted block) plus a wire cancel. The id is only reused
+  // after the next roundtrip (as in _cancelRequest), so a late reply finds no request and is dropped
+  // in ondata/onnodata: it is not invalid data and does not pause the peer.
+  _sfExpireRequest (req, now) {
+    if (req.priority === PRIORITY.CANCELLED) return false
+    req.priority = PRIORITY.CANCELLED
+
+    this.inflight--
+    this.replicator._requestDone(req.id, false)
+
+    if (this.roundtripQueue === null) this.roundtripQueue = new RoundtripQueue()
+    this.roundtripQueue.add(req.id)
+    if (this.removed === false) {
+      this.wireCancel.send({ request: req.id })
+      incrementTx(this.stats.wireCancel, this.replicator.stats.wireCancel)
+    }
+
+    if (this._sfCoolUntil <= now) {
+      if (this._sfTimeouts < 8) this._sfTimeouts++
+      const cooldown = this.replicator._sfCooldownMin * 2 ** (this._sfTimeouts - 1)
+      this._sfCoolUntil = now + Math.min(STALLFIX_COOLDOWN_MAX, cooldown)
+    }
+
+    if (this.replicator._manifestPeer === this && this.core.header.manifest === null) {
+      this.replicator._manifestPeer = null
+    }
+
+    this.replicator._clearRequest(this, req)
+    return true
+  }
+
+  _cancelRequest (req) {
+    if (req.priority === PRIORITY.CANCELLED) return
+`
+        ],
+        // 5. Arm the sweep when a request goes in flight.
+        [
+            String.raw`    this.inflight++
+    this.replicator._inflight.add(req)
+`,
+            String.raw`    this.inflight++
+    this.replicator._inflight.add(req)
+    if (this.replicator._sfTimer === null) this.replicator._sfArm() // OPENMAYHEM-STALLFIX
+`
+        ],
+        // 6. Marker on the Replicator class (runtime check: core.replicator.constructor.STALLFIX).
+        [
+            String.raw`  static Peer = Peer // hack to be able to access Peer from outside this module
+`,
+            String.raw`  static Peer = Peer // hack to be able to access Peer from outside this module
+  static STALLFIX = STALLFIX // OPENMAYHEM-STALLFIX marker
+`
+        ],
+        // 7. Replicator state.
+        [
+            String.raw`    this._notDownloadingLinger = notDownloadingLinger
+    this._downloadingTimer = null
+`,
+            String.raw`    this._notDownloadingLinger = notDownloadingLinger
+    this._downloadingTimer = null
+
+    // OPENMAYHEM-STALLFIX state (the _sf* numbers are only changed by tests)
+    this._sfTimer = null
+    this._sfCursor = 0
+    this._sfAllCooling = false
+    this._sfMaxAge = STALLFIX_REQUEST_MAX_AGE
+    this._sfInterval = STALLFIX_SWEEP_INTERVAL
+    this._sfCooldownMin = STALLFIX_COOLDOWN_MIN
+    this._sfStats = { expired: 0, notAvailableSent: 0, notAvailableSuppressed: 0 }
+`
+        ],
+        // 8. Valid block data from a peer ends its cool-down.
+        [
+            String.raw`  _ondata (peer, req, data) {
+    req.elapsed = Date.now() - req.timestamp
+    if (data.block !== null) {
+`,
+            String.raw`  _ondata (peer, req, data) {
+    req.elapsed = Date.now() - req.timestamp
+    if (data.block !== null) {
+      peer._sfTimeouts = 0 // OPENMAYHEM-STALLFIX
+      peer._sfCoolUntil = 0
+`
+        ],
+        // 9. The sweep (bounded work, one timer per replicator).
+        [
+            String.raw`  _maybeUpdate () {
+    return this._upgrade !== null && this._upgrade.inflight.length === 0
+  }
+`,
+            String.raw`  _maybeUpdate () {
+    return this._upgrade !== null && this._upgrade.inflight.length === 0
+  }
+
+  // OPENMAYHEM-STALLFIX: one unref'd timer per replicator, armed by _send, re-armed only while requests are
+  // in flight or a peer is cooling down. Looks at <= STALLFIX_MAX_SCAN slots, expires <= STALLFIX_MAX_EXPIRE.
+  _sfArm () {
+    if (this.destroyed || this._sfTimer !== null) return
+    this._sfTimer = setTimeout(sfSweepLater, this._sfInterval, this)
+    if (this._sfTimer.unref) this._sfTimer.unref()
+  }
+
+  _sfSweep () {
+    this._sfTimer = null
+    if (this.destroyed) return
+
+    const now = Date.now()
+    const reqs = this._inflight._requests
+    const scan = Math.min(reqs.length, STALLFIX_MAX_SCAN)
+    let live = 0
+    let expired = 0
+
+    for (let i = 0; i < scan && expired < STALLFIX_MAX_EXPIRE; i++) {
+      if (this._sfCursor >= reqs.length) this._sfCursor = 0
+      const req = reqs[this._sfCursor++]
+      if (!req) continue
+      live++
+      if (now - req.timestamp < this._sfMaxAge) continue
+      if (req.peer._sfExpireRequest(req, now) === true) expired++
+    }
+
+    let cooling = 0
+    let usable = 0
+    let cooled = false
+    for (const peer of this.peers) {
+      if (peer._sfCoolUntil !== 0 && peer._sfCoolUntil <= now) {
+        peer._sfCoolUntil = 0
+        cooled = true
+      }
+      if (peer._sfCoolUntil !== 0) cooling++
+      else if (peer.isActive()) usable++
+    }
+    // if no other peer is usable, a cooling peer may still be asked (no deadlock with a single peer)
+    this._sfAllCooling = cooling > 0 && usable === 0
+
+    if (expired > 0) this._sfStats.expired += expired
+    if (expired > 0 || cooled) this.updateAll()
+
+    if (live > 0 || cooling > 0 || scan < reqs.length) this._sfArm()
+  }
+
+  _sfCoolingDown (peer) {
+    return peer._sfCoolUntil !== 0 && this._sfAllCooling === false && peer._sfCoolUntil > Date.now()
+  }
+`
+        ],
+        // 10. Hotswap does not pick a cooling peer.
+        [
+            String.raw`    if (!peer.isActive() || peer.inflight >= maxHotswaps) return
+`,
+            String.raw`    if (!peer.isActive() || peer.inflight >= maxHotswaps || this._sfCoolingDown(peer)) return
+`
+        ],
+        // 11. _updatePeer: skip a cooling peer + backport upstream 8cf8ad9 fix 2 (stale queue entry must not
+        //     consume the peer's update).
+        [
+            String.raw`  _updatePeer (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight()) {
+      return false
+    }
+`,
+            String.raw`  _updatePeer (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight() || this._sfCoolingDown(peer)) {
+      return false
+    }
+`
+        ],
+        [
+            String.raw`    for (const b of blks) {
+      if (b.queued === false || peer._requestBlock(b) === true) {
+        b.queued = false
+        blks.dequeue()
+        return true
+      }
+    }
+`,
+            String.raw`    for (const b of blks) {
+      if (b.queued === false) {
+        // upstream 8cf8ad9 (#896) fix 2, backported by OPENMAYHEM-STALLFIX
+        blks.dequeue()
+        continue
+      }
+      if (peer._requestBlock(b) === true) {
+        b.queued = false
+        blks.dequeue()
+        return true
+      }
+    }
+`
+        ],
+        // 12. _updatePeerNonPrimary: skip a cooling peer.
+        [
+            String.raw`  _updatePeerNonPrimary (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight()) {
+      return false
+    }
+`,
+            String.raw`  _updatePeerNonPrimary (peer) {
+    if (!peer.isActive() || peer.inflight >= peer.getMaxInflight() || this._sfCoolingDown(peer)) {
+      return false
+    }
+`
+        ],
+        // 13. destroy() clears the sweep timer.
+        [
+            String.raw`    if (this._downloadingTimer) {
+      clearTimeout(this._downloadingTimer)
+      this._downloadingTimer = null
+    }
+
+    while (this.peers.length) {
+`,
+            String.raw`    if (this._downloadingTimer) {
+      clearTimeout(this._downloadingTimer)
+      this._downloadingTimer = null
+    }
+
+    if (this._sfTimer !== null) {
+      clearTimeout(this._sfTimer) // OPENMAYHEM-STALLFIX
+      this._sfTimer = null
+    }
+
+    while (this.peers.length) {
+`
+        ],
+        // 14. Timer trampoline.
+        [
+            String.raw`function setDownloadingLater (repl, downloading, session) {
+`,
+            String.raw`function sfSweepLater (repl) {
+  repl._sfSweep() // OPENMAYHEM-STALLFIX
+}
+
+function setDownloadingLater (repl, downloading, session) {
+`
+        ],
+        // Upstream #911, in addition to the existing stall fixes.
+        closedStreamGuard
+    ]
+}

package/src/config/config.js

@@ -219,6 +219,16 @@ export class Config {
         return this.#config.validatorHealthCheckFailureThreshold
     }
 
+    get confirmationStallTimeout() {
+        if (this.#isOverriden('confirmationStallTimeout')) return this.#options.confirmationStallTimeout
+        return this.#config.confirmationStallTimeout
+    }
+
+    get replicationStallTimeout() {
+        if (this.#isOverriden('replicationStallTimeout')) return this.#options.replicationStallTimeout
+        return this.#config.replicationStallTimeout
+    }
+
     get maxPendingRequestsInPendingRequestsService() {
         return this.#config.maxPendingRequestsInPendingRequestsService
     }

package/src/config/env.js

@@ -57,6 +57,8 @@ const configData = {
         txPoolSize: 1000, // size of transaction pool
         validatorHealthCheckInterval: 5 * 60 * 1000, // How often to check validator health (ms)
         validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
+        confirmationStallTimeout: 120_000, // Force a fast-forward when the confirmed length has not moved for this long while unconfirmed entries wait (ms, 0 = off)
+        replicationStallTimeout: 600_000, // Close all MSB connections once (max 1 per 10 min) when signed system data stays unapplied and the confirmed length has not moved for this long (ms, 0 = off)
         storesDirectory: 'stores/',
         storeName: 'testnet',
     },
@@ -107,6 +109,8 @@ const configData = {
         txPoolSize: 1000, // size of transaction pool
         validatorHealthCheckInterval: 5 * 60 * 1000, // How often to check validator health (ms)
         validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
+        confirmationStallTimeout: 120_000, // Force a fast-forward when the confirmed length has not moved for this long while unconfirmed entries wait (ms, 0 = off)
+        replicationStallTimeout: 600_000, // Close all MSB connections once (max 1 per 10 min) when signed system data stays unapplied and the confirmed length has not moved for this long (ms, 0 = off)
         storesDirectory: 'stores/',
         storeName: 'mainnet',
     },
@@ -157,6 +161,8 @@ const configData = {
         txPoolSize: 1000, // size of transaction pool
         validatorHealthCheckInterval: 1_000, // How often to check validator health (ms)
         validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
+        confirmationStallTimeout: 120_000, // Force a fast-forward when the confirmed length has not moved for this long while unconfirmed entries wait (ms, 0 = off)
+        replicationStallTimeout: 600_000, // Close all MSB connections once (max 1 per 10 min) when signed system data stays unapplied and the confirmed length has not moved for this long (ms, 0 = off)
         storesDirectory: 'stores/',
         storeName: 'development',
     }

package/src/core/network/services/ReplicationStallGuard.js

@@ -0,0 +1,177 @@
+const DEFAULT_CHECK_INTERVAL = 15_000;
+const DEFAULT_MIN_GAP = 600_000;
+
+/**
+ * Safety net for the Hypercore replication stall (block requests that a peer never answers; the install-time
+ * patch in scripts/hypercore-stallfix/ is the actual fix). Signal: at every check for `timeout` ms this node
+ * knew signed system data it had not applied (base.core.core.state.length > base.core.length) while its
+ * confirmed length (base.view.core.signedLength) did not move. Then it closes ALL connections of the MSB swarm
+ * once, pinned validator links included, and lets discovery reconnect: the transport reset a restart does,
+ * without a restart. At most once per `minGap`, and only after a new full `timeout` window.
+ *
+ * Never runs on an indexer. It only closes connections: roles, transactions, balances and state are untouched.
+ */
+export class ReplicationStallGuard {
+    #base;
+    #getSwarm;
+    #refresh;
+    #timeout;
+    #interval;
+    #minGap;
+    #now;
+    #log;
+    #timer = null;
+    #mark = null;
+    #stalledSince = null;
+    #lastReconnectAt = null;
+    #lastClosed = 0;
+    #reconnects = 0;
+    #last = null;
+
+    /**
+     * @param {object} base Autobase instance of MSB
+     * @param {() => object|null} getSwarm returns the MSB Hyperswarm (or null)
+     * @param {object} options
+     */
+    constructor(base, getSwarm, {
+        timeout,
+        interval = DEFAULT_CHECK_INTERVAL,
+        minGap = DEFAULT_MIN_GAP,
+        refresh = null,
+        now = Date.now,
+        log = console.log
+    } = {}) {
+        this.#base = base;
+        this.#getSwarm = getSwarm;
+        this.#refresh = refresh;
+        this.#timeout = timeout;
+        this.#interval = Number.isSafeInteger(timeout) && timeout > 0 ? Math.min(interval, timeout) : interval;
+        this.#minGap = minGap;
+        this.#now = now;
+        this.#log = log;
+    }
+
+    get enabled() {
+        return Number.isSafeInteger(this.#timeout) && this.#timeout > 0;
+    }
+
+    start() {
+        if (!this.enabled || this.#timer !== null) return;
+        this.#timer = setInterval(() => { this.check(); }, this.#interval);
+        this.#timer.unref?.();
+    }
+
+    stop() {
+        if (this.#timer !== null) clearInterval(this.#timer);
+        this.#timer = null;
+    }
+
+    status() {
+        const last = this.#last;
+        let state = 'ok';
+        if (this.#stalledSince !== null) state = 'stalled';
+        else if (this.#mark !== null) state = 'behind';
+        return {
+            enabled: this.enabled,
+            state,
+            confirmedLength: last?.confirmed ?? null,
+            systemLength: last?.systemLength ?? null,
+            systemSignedLength: last?.systemSignedLength ?? null,
+            behindSince: this.#mark?.since ?? null,
+            stalledSince: this.#stalledSince,
+            reconnects: this.#reconnects,
+            lastReconnectAt: this.#lastReconnectAt,
+            lastReconnectClosed: this.#lastClosed
+        };
+    }
+
+    #emit(event, fields) {
+        try { this.#log(`[MSB] ${JSON.stringify({ event, ...fields })}`); } catch {}
+    }
+
+    #sample() {
+        const base = this.#base;
+        const confirmed = base?.view?.core?.signedLength;
+        const systemLength = base?.core?.length;
+        const systemSignedLength = base?.core?.core?.state?.length;
+        if (![confirmed, systemLength, systemSignedLength].every(Number.isSafeInteger)) return null;
+        return { confirmed, systemLength, systemSignedLength };
+    }
+
+    /** @returns {boolean} true when the connections were closed */
+    check() {
+        const base = this.#base;
+        if (!this.enabled || !base || base.closing || base.closed || base.isIndexer) {
+            this.#mark = null;
+            return false;
+        }
+        const s = this.#sample();
+        if (s === null) return false;
+        this.#last = s;
+
+        const now = this.#now();
+        const ahead = s.systemSignedLength > s.systemLength;
+        const moved = this.#mark !== null && this.#mark.confirmed !== s.confirmed;
+
+        if (this.#stalledSince !== null && (!ahead || moved)) {
+            this.#emit('msb_replication_recovered', {
+                confirmedLength: s.confirmed, systemLength: s.systemLength,
+                systemSignedLength: s.systemSignedLength, stalledMs: now - this.#stalledSince
+            });
+            this.#stalledSince = null;
+        }
+        if (!ahead) {
+            this.#mark = null;
+            return false;
+        }
+        if (this.#mark === null || moved || now < this.#mark.since) {
+            this.#mark = { confirmed: s.confirmed, since: now };
+            return false;
+        }
+
+        const elapsed = now - this.#mark.since;
+        if (elapsed < this.#timeout) return false;
+
+        if (this.#stalledSince === null) {
+            this.#stalledSince = this.#mark.since;
+            this.#emit('msb_replication_stalled', {
+                confirmedLength: s.confirmed, systemLength: s.systemLength,
+                systemSignedLength: s.systemSignedLength, stalledMs: elapsed
+            });
+        }
+        if (this.#lastReconnectAt !== null && now - this.#lastReconnectAt < this.#minGap) return false;
+
+        const closed = this.#reconnectAll();
+        this.#lastReconnectAt = now;
+        this.#lastClosed = closed;
+        this.#reconnects++;
+        this.#emit('msb_replication_reconnect_all', {
+            closedConnections: closed, confirmedLength: s.confirmed, systemLength: s.systemLength,
+            systemSignedLength: s.systemSignedLength, stalledMs: now - this.#stalledSince, reconnects: this.#reconnects
+        });
+        // the next action needs a new full window
+        this.#mark = { confirmed: s.confirmed, since: now };
+        return true;
+    }
+
+    #reconnectAll() {
+        const swarm = this.#getSwarm?.();
+        if (!swarm || swarm.destroyed || swarm.suspended || !swarm.connections) return 0;
+        let closed = 0;
+        for (const connection of [...swarm.connections]) {
+            if (connection.destroyed || connection.destroying) continue;
+            try {
+                connection.destroy();
+                closed++;
+            } catch {}
+        }
+        if (this.#refresh !== null) {
+            try {
+                Promise.resolve(this.#refresh()).catch(() => {});
+            } catch {}
+        }
+        return closed;
+    }
+}
+
+export default ReplicationStallGuard;

package/src/core/state/ConfirmationWatch.js

@@ -0,0 +1,82 @@
+const DEFAULT_CHECK_INTERVAL = 15_000;
+
+/**
+ * Autobase starts a follower's fast-forward only when the network's signed length is at least
+ * 16 entries past the follower's total length, unconfirmed entries included. After a writer is
+ * removed in the middle of a batch, a follower that already holds unconfirmed entries can stay
+ * above that mark: its confirmed length then stops for good, and a restart does not help.
+ *
+ * This watch forces one fast-forward when the confirmed length has not moved for `timeout` ms
+ * while unconfirmed entries wait. It never runs on an indexer and does not change how entries are
+ * applied.
+ */
+export class ConfirmationWatch {
+    #base;
+    #timeout;
+    #interval;
+    #now;
+    #log;
+    #timer = null;
+    #mark = null;
+    #running = false;
+
+    constructor(base, { timeout, interval = DEFAULT_CHECK_INTERVAL, now = Date.now, log = console.log } = {}) {
+        this.#base = base;
+        this.#timeout = timeout;
+        this.#interval = Math.min(interval, timeout);
+        this.#now = now;
+        this.#log = log;
+    }
+
+    get enabled() {
+        return Number.isFinite(this.#timeout) && this.#timeout > 0;
+    }
+
+    start() {
+        if (!this.enabled || this.#timer !== null) return;
+        this.#timer = setInterval(() => { this.check().catch(() => {}) }, this.#interval);
+        this.#timer.unref?.();
+    }
+
+    stop() {
+        if (this.#timer !== null) clearInterval(this.#timer);
+        this.#timer = null;
+    }
+
+    /** @returns {Promise<boolean>} true when a fast-forward was forced */
+    async check() {
+        const base = this.#base;
+        if (this.#running || base.closing || base.isIndexer) return false;
+        if (base.isFastForwarding?.()) {
+            this.#mark = null;
+            return false;
+        }
+
+        const confirmed = base.view.core.signedLength;
+        const total = base.view.core.length;
+        const now = this.#now();
+
+        if (total <= confirmed) {
+            this.#mark = null;
+            return false;
+        }
+        if (this.#mark === null || this.#mark.confirmed !== confirmed) {
+            this.#mark = { confirmed, since: now };
+            return false;
+        }
+        if (now - this.#mark.since < this.#timeout) return false;
+
+        this.#running = true;
+        try {
+            this.#log(`[MSB] Confirmed length stuck at ${confirmed} (local ${total}) for ${Math.round((now - this.#mark.since) / 1000)} s; forcing a fast-forward`);
+            await base.forceFastForward();
+            this.#log(`[MSB] Fast-forward finished: confirmed ${base.view.core.signedLength}, local ${base.view.core.length}`);
+        } catch (error) {
+            this.#log(`[MSB] Fast-forward failed: ${error?.message ?? error}`);
+        } finally {
+            this.#running = false;
+            this.#mark = { confirmed: base.view.core.signedLength, since: this.#now() };
+        }
+        return true;
+    }
+}

package/src/core/state/State.js

@@ -42,6 +42,7 @@ import { deepCopyBuffer } from '../../utils/buffer.js';
 import { Status } from './utils/transaction.js';
 import remote from 'hypercore/lib/fully-remote-proof.js'
 import PQueue from 'p-queue';
+import { ConfirmationWatch } from './ConfirmationWatch.js';
 
 const OVERSIZED_BATCH_PENALTY_MULTIPLIER = BATCH_SIZE;
 
@@ -56,6 +57,7 @@ class State extends ReadyResource {
     #config
     #wallet
     #activeWriterCountCache = new Map();
+    #confirmationWatch = null;
 
     /**
      * @param {Corestore} store
@@ -96,10 +98,13 @@ class State extends ReadyResource {
         console.log("State initialization...")
         await this.#base.ready();
         this.#writingKey = this.#base.local.key;
+        this.#confirmationWatch = new ConfirmationWatch(this.#base, { timeout: this.#config.confirmationStallTimeout });
+        this.#confirmationWatch.start();
     }
 
     async _close() {
         console.log("State: closing gracefully...");
+        this.#confirmationWatch?.stop();
         if (this.#bee !== null) {
             await this.#bee.close();
         }