trac-msb 0.2.20 → 0.2.23

package/SECURITY.md

@@ -6,13 +6,11 @@ The following table shows which versions of this project are currently receiving
 
 | Version  | Supported          | Pear compatibility |
 | -------- | ------------------ | ------------------ |
-| 0.2.19   | :white_check_mark: | Pear v2 and v3     |
-| 0.2.18   | :white_check_mark: | Pear v2 only       |
-| 0.2.17   | :white_check_mark: | Pear v2 only       |
+| 0.2.21   | :white_check_mark: | Pear v2 and v3     |
 | main     | :warning: unstable | Pear v2 and v3     |
-| < 0.2.17 | :x:               | N/A                |
+| < 0.2.21 | :x:                | N/A                |
 
-Versions older than 0.2.17 are deprecated.
+Versions older than 0.2.21 are deprecated.
 Please upgrade to the latest release to ensure you receive security fixes.
 
 ---

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "trac-msb",
   "main": "msb.mjs",
-  "version": "0.2.20",
+  "version": "0.2.23",
   "pear": {
     "name": "trac-msb",
     "type": "terminal"
@@ -54,7 +54,7 @@
     "hyperbee": "2.26.5",
     "hypercore": "11.18.3",
     "hypercore-crypto": "3.6.1",
-    "hyperdht": "6.27.0",
+    "hyperdht": "6.29.6",
     "hyperswarm": "4.14.2",
     "hyperschema": "1.17.1",
     "lodash": "^4.18.1",

package/src/config/config.js

@@ -214,6 +214,11 @@ export class Config {
         return this.#config.validatorHealthCheckInterval
     }
 
+    get validatorHealthCheckFailureThreshold() {
+        if (this.#isOverriden('validatorHealthCheckFailureThreshold')) return this.#options.validatorHealthCheckFailureThreshold
+        return this.#config.validatorHealthCheckFailureThreshold
+    }
+
     get maxPendingRequestsInPendingRequestsService() {
         return this.#config.maxPendingRequestsInPendingRequestsService
     }

package/src/config/env.js

@@ -52,10 +52,11 @@ const configData = {
         rateLimitConnectionTimeoutMs: 60_000, // Rate limiting constants
         rateLimitMaxTransactionsPerSecond: 50, // Rate limiting constants
         maxPendingRequestsInPendingRequestsService: 50_000, // Maximum number of pending requests in PendingRequestService (This value should not exceed 256MB)
-        pendingRequestTimeout: 3000, // constant after which time the transaction will be considered invalid
-        txCommitTimeout: 2200,
+        pendingRequestTimeout: 10_000, // Deadline for a pending V1 response (ms), including health checks
+        txCommitTimeout: 5_000, // Wait for local append and publication proof (ms); shorter than the response deadline
         txPoolSize: 1000, // size of transaction pool
         validatorHealthCheckInterval: 5 * 60 * 1000, // How often to check validator health (ms)
+        validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
         storesDirectory: 'stores/',
         storeName: 'testnet',
     },
@@ -101,10 +102,11 @@ const configData = {
         rateLimitConnectionTimeoutMs: 60_000, // Rate limiting constants
         rateLimitMaxTransactionsPerSecond: 50, // Rate limiting constants
         maxPendingRequestsInPendingRequestsService: 50_000, // Maximum number of pending requests in PendingRequestService (This value should not exceed 256MB)
-        pendingRequestTimeout: 3000, // constant after which time the transaction will be considered invalid
-        txCommitTimeout: 2200,
+        pendingRequestTimeout: 10_000, // Deadline for a pending V1 response (ms), including health checks
+        txCommitTimeout: 5_000, // Wait for local append and publication proof (ms); shorter than the response deadline
         txPoolSize: 1000, // size of transaction pool
         validatorHealthCheckInterval: 5 * 60 * 1000, // How often to check validator health (ms)
+        validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
         storesDirectory: 'stores/',
         storeName: 'mainnet',
     },
@@ -150,10 +152,11 @@ const configData = {
         rateLimitConnectionTimeoutMs: 60_000, // Rate limiting constants
         rateLimitMaxTransactionsPerSecond: 50, // Rate limiting constants
         maxPendingRequestsInPendingRequestsService: 50_000, // Maximum number of pending requests in PendingRequestService (This value should not exceed 256MB)
-        pendingRequestTimeout: 3000, // constant after which time the transaction will be considered invalid
-        txCommitTimeout: 2200,
+        pendingRequestTimeout: 10_000, // Deadline for a pending V1 response (ms), including health checks
+        txCommitTimeout: 5_000, // Wait for local append and publication proof (ms); shorter than the response deadline
         txPoolSize: 1000, // size of transaction pool
         validatorHealthCheckInterval: 1_000, // How often to check validator health (ms)
+        validatorHealthCheckFailureThreshold: 3, // Consecutive failed health checks before a validator is removed
         storesDirectory: 'stores/',
         storeName: 'development',
     }
@@ -161,4 +164,4 @@ const configData = {
 
 export const createConfig = (environment, options) => {
     return new Config(options, configData[environment])
-}
+}

package/src/core/network/Network.js

@@ -263,7 +263,10 @@ class Network extends ReadyResource {
                         new Error('Connection closed before response')
                     );
                     this.#swarm?.leavePeer(connection.remotePublicKey);
-                    this.#validatorConnectionManager.remove(publicKey);
+                    // only act on the connection this event belongs to
+                    if (this.#validatorConnectionManager.isCurrent(publicKey, connection)) {
+                        this.#validatorConnectionManager.remove(publicKey);
+                    }
                     if (connection.protocolSession) {
                         try {
                             connection.protocolSession.close();
@@ -350,7 +353,8 @@ class Network extends ReadyResource {
         this.#pendingConnections.set(publicKey, { type, timeoutId });
 
         const target = b4a.from(publicKey, 'hex');
-        if (!this.#swarm.peers.has(publicKey)) {
+        // A retained peer may have lost explicit retries after leavePeer().
+        if (!this.#swarm.peers.get(publicKey)?.explicit) {
             this.#swarm.joinPeer(target);
         }
 

package/src/core/network/services/ConnectionManager.js

@@ -6,6 +6,8 @@ import {Logger} from "../../../utils/logger.js";
  * @typedef {import('hyperswarm').Connection} Connection
  */
 
+const DEFAULT_HEALTH_CHECK_FAILURE_THRESHOLD = 3;
+
 export class ConnectionManagerError extends Error {
     constructor(message) {
         super(message);
@@ -18,6 +20,8 @@ class ConnectionManager {
     #maxValidators
     #config
     #healthCheckService
+    #healthCheckFailures
+    #healthCheckFailureThreshold
     #boundedHealthCheckHandler
     #logger
     // Note: #validators is using publicKey (Buffer) as key
@@ -28,8 +32,11 @@ class ConnectionManager {
      **/
     constructor(config) {
         this.#validators = new Map();
+        // consecutive failed health checks per validator
+        this.#healthCheckFailures = new Map();
         this.#config = config
         this.#maxValidators = config.maxValidators
+        this.#healthCheckFailureThreshold = this.#resolveHealthCheckFailureThreshold(config);
         this.#boundedHealthCheckHandler = this.#healthCheckHandler.bind(this);
         this.#logger = new Logger(config)
     }
@@ -98,10 +105,15 @@ class ConnectionManager {
         }
 
         if (!success) {
-            this.#logger.debug(`healthCheck: liveness request failed, removing validator. Address = ${targetAddress}; Request ID = ${requestId}`);
-            this.remove(publicKey);
-            this.#stopHealthCheck(publicKey);
+            const failures = this.#countHealthCheckFailure(publicKey);
+            this.#logger.debug(`healthCheck: liveness request failed. Address = ${targetAddress}; Request ID = ${requestId}; Consecutive failures = ${failures}`);
+            if (failures >= this.#healthCheckFailureThreshold) {
+                this.#logger.debug(`healthCheck: failure threshold reached, removing validator. Address = ${targetAddress}; Request ID = ${requestId}`);
+                this.remove(publicKey);
+                this.#stopHealthCheck(publicKey);
+            }
         } else {
+            this.#clearHealthCheckFailures(publicKey);
             this.#logger.debug(`healthCheck: success. Address = ${targetAddress}; Request ID = ${requestId}`);
         }
     };
@@ -123,6 +135,23 @@ class ConnectionManager {
         }
     }
 
+    #resolveHealthCheckFailureThreshold(config) {
+        const threshold = Number(config.validatorHealthCheckFailureThreshold);
+        if (!Number.isInteger(threshold) || threshold < 1) return DEFAULT_HEALTH_CHECK_FAILURE_THRESHOLD;
+        return threshold;
+    }
+
+    #countHealthCheckFailure(publicKey) {
+        const publicKeyHex = this.#toHexString(publicKey);
+        const failures = (this.#healthCheckFailures.get(publicKeyHex) || 0) + 1;
+        this.#healthCheckFailures.set(publicKeyHex, failures);
+        return failures;
+    }
+
+    #clearHealthCheckFailures(publicKey) {
+        this.#healthCheckFailures.delete(this.#toHexString(publicKey));
+    }
+
     /**
      * Retrieves the Hyperswarm connection object for a given validator public key.
      * @param {String | Buffer} publicKey - The public key (Buffer or hex string) of the validator.
@@ -134,6 +163,18 @@ class ConnectionManager {
         return entry ? entry.connection : undefined;
     }
 
+    /**
+     * Checks whether a connection is the one currently tracked for a validator.
+     * @param {String | Buffer} publicKey - The public key (Buffer or hex string) of the validator.
+     * @param {Object} connection - The connection object to compare with the tracked one.
+     * @returns {Boolean} - Returns true if the tracked connection is the given connection, false otherwise.
+     */
+    isCurrent(publicKey, connection) {
+        const publicKeyHex = this.#toHexString(publicKey);
+        const entry = this.#validators.get(publicKeyHex);
+        return !!entry && entry.connection === connection;
+    }
+
     /**
      * Sends a message through a specific validator without increasing sent messages count.
      * @param {Object} message - The message to send to the validator.
@@ -194,6 +235,7 @@ class ConnectionManager {
         this.#logger.debug(`remove: removing validator ${publicKeyToAddress(publicKey, this.#config)}`);
         const publicKeyHex = this.#toHexString(publicKey);
         this.#stopHealthCheck(publicKeyHex);
+        this.#clearHealthCheckFailures(publicKeyHex);
         if (this.exists(publicKeyHex)) {
             const entry = this.#validators.get(publicKeyHex);
             if (endConnection && entry && entry.connection && typeof entry.connection.end === 'function') {
@@ -326,8 +368,11 @@ class ConnectionManager {
             return;
         }
         this.#validators.set(publicKeyHex, {connection, sent: 0});
+        this.#clearHealthCheckFailures(publicKeyHex);
         connection.on('close', () => {
             this.#logger.debug(`#append: connection closing for validator ${publicKeyToAddress(publicKey, this.#config)}`);
+            // only act on the connection this event belongs to
+            if (!this.isCurrent(publicKeyHex, connection)) return;
             this.remove(publicKeyHex);
             this.#logger.debug(`#append: connection closed for validator ${publicKeyToAddress(publicKey, this.#config)}`);
         });
@@ -350,6 +395,7 @@ class ConnectionManager {
         } else {
             this.#validators.set(publicKeyHex, {connection, sent: 0});
         }
+        this.#clearHealthCheckFailures(publicKeyHex);
     }
 
     #toHexString(publicKey) {

package/src/core/network/services/ValidatorObserverService.js

@@ -122,7 +122,7 @@ class ValidatorObserverService {
      *
      * If the count of fully connected writers exceeds the threshold:
      * - The admin node is excluded from the validator pool (handled in selection phase).
-     * - Non-admin nodes actively disconnect from the admin validator (if currently connected).
+     * - Non-admin nodes detach the admin from the validator pool, preserving replication.
      *
      * NOTE:
      * The admin is not forcibly disconnected from all roles.
@@ -144,7 +144,7 @@ class ValidatorObserverService {
         // Non-admin nodes: ensure the admin is not kept as a validator connection
         const adminWriter = writers.find((w) => w.address === adminEntry?.address);
         if (adminWriter?.publicKey) {
-            this.#network.validatorConnectionManager.remove(adminWriter.publicKey);
+            this.#network.validatorConnectionManager.remove(adminWriter.publicKey, { endConnection: false });
         }
     }
 
@@ -156,7 +156,8 @@ class ValidatorObserverService {
         const manager = this.#network.validatorConnectionManager;
 
         if (manager.connected(publicKey)) {
-            manager.remove(publicKey);
+            // leave the transport to the swarm
+            manager.remove(publicKey, { endConnection: false });
             this.#logger.debug(`Removed stale validator connection: ${b4a.toString(publicKey, "hex")}`);
         }
     }
@@ -373,8 +374,8 @@ class ValidatorObserverService {
      * Rules:
      * - Skips invalid/malformed entries.
      * - Skips non-admin indexers.
-     * - Deduplicates nodes by mapping their latest state to a unique address.
-     * - Removes stale connections for nodes flagged with 'isRemoved'.
+     * - Deduplicates active writers by wallet address.
+     * - Removes stale connections only when no active writer remains for an address.
      * - Includes only valid and active writer nodes.
      * @param {Object} adminEntry - The current admin node entry.
      * @returns {Promise<Array<Object>>} A promise resolving to the deduplicated array of active writers.
@@ -385,13 +386,13 @@ class ValidatorObserverService {
             this.#keyDecodeCache.clear();
         }
 
-        // Map is used for deduplication, ensuring that each node physical address
-        // occupies only a single slot despite the append-only nature of Autobase
+        // Historical writer keys can share an address with an active replacement.
         const activeMap = new Map();
+        const removedMap = new Map();
 
         try {
             for await (const { key, value } of this.#state.base.system.list()) {
-                if (!this.#shouldRun()) break;
+                if (!this.#shouldRun()) return Array.from(activeMap.values());
 
                 if (!key || b4a.byteLength(key) !== WRITER_BYTE_LENGTH) continue;
                 if (!value) continue;
@@ -426,16 +427,20 @@ class ValidatorObserverService {
 
                 if (value.isIndexer && addr !== adminEntry?.address) continue;
 
-                // Handles soft-delete: drops stale connections and removes the node from the pool
                 if (value.isRemoved) {
-                    this.#enforceNoStaleConnections(publicKey);
-                    activeMap.delete(addr);
+                    removedMap.set(addr, publicKey);
                     continue;
                 }
 
-                // Registers or overwrites to maintain a single source of truth per machine
                 activeMap.set(addr, { key, address: addr, publicKey, publicKeyHex });
             }
+
+            // Writer iteration order does not represent key age. Only a complete scan
+            // can establish that a removed writer has no active replacement.
+            for (const [addr, publicKey] of removedMap) {
+                if (!this.#shouldRun()) break;
+                if (!activeMap.has(addr)) this.#enforceNoStaleConnections(publicKey);
+            }
         } catch (err) {
             this.#logger.error(`Autobase writer scan error: ${err.message}`);
         }
@@ -462,4 +467,4 @@ class ValidatorObserverService {
     }
 }
 
-export default ValidatorObserverService;
+export default ValidatorObserverService;

package/src/core/state/State.js

@@ -15,7 +15,7 @@ import {
     TRAC_NAMESPACE,
     CustomEventType
 } from '../../utils/constants.js';
-import { isHexString, sleep, isTransactionRecordPut } from '../../utils/helpers.js';
+import { isHexString, sleep } from '../../utils/helpers.js';
 import tracCryptoApi from 'trac-crypto-api';
 import Check from '../../utils/check.js';
 import { safeDecodeApplyOperation } from '../../utils/protobuf/operationHelpers.js';
@@ -372,18 +372,13 @@ class State extends ReadyResource {
             throw new Error("Invalid hash format");
         }
 
-        const confirmedLength = this.getSignedLength();
-        const historyStream = this.#base.view.createHistoryStream({
-            gte: 0,
-            lte: confirmedLength
-        });
-
-        for await (const entry of historyStream) {
-            if (isTransactionRecordPut(entry) && entry.key === hash) {
-                return entry.seq;
-            }
+        const snapshot = this.#base.view.checkout(this.getSignedLength());
+        try {
+            const entry = await snapshot.get(hash);
+            return entry ? entry.seq : null;
+        } finally {
+            await snapshot.close();
         }
-        return null;
     }
 
     async confirmedTransactionsBetween(startSignedLength, endSignedLength) {