trac-msb 0.2.26 → 0.3.0-beta.2

package/src/core/network/protocols/{v1 → validators/v1}/validators/V1BroadcastTransactionRequest.js

@@ -1,6 +1,6 @@
 import V1BaseOperation from "./V1BaseOperation.js";
 import b4a from "b4a";
-import {MAX_PARTIAL_TX_PAYLOAD_BYTE_SIZE, ResultCode} from '../../../../../utils/constants.js';
+import {MAX_PARTIAL_TX_PAYLOAD_BYTE_SIZE, ResultCode} from '../../../../../../utils/constants.js';
 import {V1ProtocolError} from "../V1ProtocolError.js";
 
 class V1BroadcastTransactionRequest extends V1BaseOperation {
@@ -18,7 +18,7 @@ class V1BroadcastTransactionRequest extends V1BaseOperation {
     isDataPropertySizeValid(payload) {
         if (b4a.byteLength(payload.broadcast_transaction_request.data) > MAX_PARTIAL_TX_PAYLOAD_BYTE_SIZE) {
             throw new V1ProtocolError(
-                ResultCode.INVALID_PAYLOAD,
+                ResultCode.SCHEMA_VALIDATION_FAILED,
                 `The 'data' field exceeds the maximum allowed byte size of ${MAX_PARTIAL_TX_PAYLOAD_BYTE_SIZE}. Actual size: ${b4a.byteLength(payload.broadcast_transaction_request.data)}`);
         }
     }

package/src/core/network/protocols/{v1 → validators/v1}/validators/V1BroadcastTransactionResponse.js

@@ -2,44 +2,44 @@ import Autobase from "autobase";
 import b4a from "b4a";
 import Hypercore from 'hypercore';
 import V1BaseOperation from "./V1BaseOperation.js";
-import {unsafeDecodeApplyOperation} from "../../../../../utils/protobuf/operationHelpers.js";
-import {isDeepEqualApplyPayload} from "../../../../../utils/deepEqualApplyPayload.js";
-import {addressToBuffer, bufferToAddress} from "../../../../state/utils/address.js";
-import {publicKeyToAddress} from "../../../../../utils/helpers.js";
-import Check from "../../../../../utils/check.js";
-import {OperationType, ResultCode} from "../../../../../utils/constants.js";
+import {unsafeDecodeApplyOperation} from "../../../../../../codecs/apply/applyOperationCodec.js";
+import {isDeepEqualApplyPayload} from "../../../../../../utils/deepEqualApplyPayload.js";
+import {addressToBuffer, bufferToAddress} from "../../../../../state/utils/address.js";
+import {publicKeyToAddress} from "../../../../../../utils/helpers.js";
+import StateValidationSchema from "../../../../../state/validators/StateValidationSchema.js";
+import {OperationType, ResultCode} from "../../../../../../utils/constants.js";
 import {V1ProtocolError} from "../V1ProtocolError.js";
 
 const VALIDATOR_METADATA_FIELDS = new Set(["va", "vn", "vs"]);
 
-const stripValidatorMetadata = (value) => {
+const stripValidatorMetadata = (value, omitFields = false) => {
 
     if (value === null || value === undefined) return value;
     if (b4a.isBuffer(value)) return value;
-    if (Array.isArray(value)) return value.map(stripValidatorMetadata);
+    if (Array.isArray(value)) return value.map(item => stripValidatorMetadata(item, omitFields));
     if (typeof value !== "object") return value;
 
     const result = {};
     for (const [key, nestedValue] of Object.entries(value)) {
         if (VALIDATOR_METADATA_FIELDS.has(key)) {
-            result[key] = null;
+            if (!omitFields) result[key] = null;
             continue;
         }
-        result[key] = stripValidatorMetadata(nestedValue);
+        result[key] = stripValidatorMetadata(nestedValue, omitFields);
     }
     return result;
 };
 
 class V1BroadcastTransactionResponse extends V1BaseOperation {
     #config;
-    #check;
+    #stateValidationSchema;
     #state;
 
     constructor(state, config) {
         super(config);
         this.#state = state;
         this.#config = config;
-        this.#check = new Check(config);
+        this.#stateValidationSchema = new StateValidationSchema(config);
     }
 
     async validate(payload, connection, pendingRequestServiceEntry) {
@@ -90,19 +90,31 @@ class V1BroadcastTransactionResponse extends V1BaseOperation {
 
         let isValid = false;
         switch (type) {
+            case OperationType.HTLC_LOCK:
+                isValid = this.#stateValidationSchema.validateHtlcLockOperation(validatorDecodedTx) &&
+                    [...VALIDATOR_METADATA_FIELDS].every(field => b4a.isBuffer(validatorDecodedTx.hlo[field]));
+                break;
+            case OperationType.HTLC_CLAIM:
+                isValid = this.#stateValidationSchema.validateHtlcClaimOperation(validatorDecodedTx) &&
+                    [...VALIDATOR_METADATA_FIELDS].every(field => b4a.isBuffer(validatorDecodedTx.hco?.[field]));
+                break;
+            case OperationType.HTLC_REFUND:
+                isValid = this.#stateValidationSchema.validateHtlcRefundOperation(validatorDecodedTx) &&
+                    [...VALIDATOR_METADATA_FIELDS].every(field => b4a.isBuffer(validatorDecodedTx.hro?.[field]));
+                break;
             case OperationType.ADD_WRITER:
             case OperationType.REMOVE_WRITER:
             case OperationType.ADMIN_RECOVERY:
-                isValid =  this.#check.validateRoleAccessOperation(validatorDecodedTx);
+                isValid =  this.#stateValidationSchema.validateRoleAccessOperation(validatorDecodedTx);
                 break;
             case OperationType.BOOTSTRAP_DEPLOYMENT:
-                isValid =  this.#check.validateBootstrapDeploymentOperation(validatorDecodedTx);
+                isValid =  this.#stateValidationSchema.validateBootstrapDeploymentOperation(validatorDecodedTx);
                 break;
             case OperationType.TX:
-                isValid = this.#check.validateTransactionOperation(validatorDecodedTx);
+                isValid = this.#stateValidationSchema.validateTransactionOperation(validatorDecodedTx);
                 break;
             case OperationType.TRANSFER:
-                isValid = this.#check.validateTransferOperation(validatorDecodedTx);
+                isValid = this.#stateValidationSchema.validateTransferOperation(validatorDecodedTx);
                 break;
             default:
                 throw new V1ProtocolError(
@@ -138,7 +150,11 @@ class V1BroadcastTransactionResponse extends V1BaseOperation {
 
         const stateTxDecodedFromRequest = unsafeDecodeApplyOperation(stateTxEncodedFromRequest);
         const stateTxDecodedFromResponse = unsafeDecodeApplyOperation(stateTxEncodedFromResponse);
-        const strippedValidatorPayload = stripValidatorMetadata(stateTxDecodedFromResponse);
+        const strippedValidatorPayload = stripValidatorMetadata(
+            stateTxDecodedFromResponse,
+            stateTxDecodedFromRequest.type === OperationType.HTLC_CLAIM ||
+                stateTxDecodedFromRequest.type === OperationType.HTLC_REFUND
+        );
 
         if (!isDeepEqualApplyPayload(stateTxDecodedFromRequest, strippedValidatorPayload)) {
             throw new V1ProtocolError(
@@ -168,12 +184,12 @@ class V1BroadcastTransactionResponse extends V1BaseOperation {
     }
 
     async validateValidatorCorrectness(validatorDecodedTx, connectionRemotePublicKey, writerKeyFromManifest, validatorAddressCorrelatedWithManifest) {
-        const validatorAddressFromTx = extractRequiredVaFromDecodedTx(validatorDecodedTx);
         const validatorAddressFromConnectionPublicKey = addressToBuffer(
             publicKeyToAddress(connectionRemotePublicKey, this.#config),
             this.#config.addressPrefix
         );
 
+        const validatorAddressFromTx = extractRequiredVaFromDecodedTx(validatorDecodedTx);
         if (!b4a.equals(validatorAddressFromTx, validatorAddressFromConnectionPublicKey)) {
             throw new V1ProtocolError(
                 ResultCode.VALIDATOR_ADDRESS_MISMATCH,

package/src/core/network/protocols/{v1 → validators/v1}/validators/V1ValidationSchema.js

@@ -5,10 +5,10 @@ import {
     NONCE_BYTE_LENGTH,
     SIGNATURE_BYTE_LENGTH,
     MAX_PARTIAL_TX_PAYLOAD_BYTE_SIZE,
-    ResultCode
-} from '../../../../../utils/constants.js';
+    NetworkResultCode
+} from '../../../../../../utils/constants.js';
 
-const ALLOWED_RESULT_CODES = Object.values(ResultCode);
+const ALLOWED_RESULT_CODES = Object.values(NetworkResultCode);
 
 class V1ValidationSchema {
     #validator;

package/src/core/network/services/IndexerConnectionManager.js

@@ -0,0 +1,88 @@
+import tracCryptoApi from 'trac-crypto-api'
+import { PeerConnectionManager } from '../shared/PeerConnectionManager.js'
+import b4a from "b4a"
+import { bufferToAddress } from '../../state/utils/address.js'
+
+class IndexerConnectionManager extends PeerConnectionManager {
+    #messages
+    #state
+    #network
+    #wallet
+
+
+    constructor(maxIndexers, config, logger, messages, state, network, wallet) {
+        super(maxIndexers, config, logger);
+        this.#messages = messages;
+        this.#state = state;
+        this.#network = network;
+        this.#wallet = wallet;
+    }
+
+    async add(publicKey, connection) {
+        await this.#messages.attachChannel(connection);
+        if (connection.destroyed) return;
+        if (!connection.protocolSessions?.indexer || connection.protocolSessions.indexer.closed) return;
+        this._add(publicKey, connection)
+    }
+
+    async connect() {
+        const entries = await this.#state.getIndexersEntry();
+        // Promotion events can precede Autobase's membership update. Refresh the
+        // capacity from the same snapshot used for discovery on every retry.
+        this.setMax(Math.max(0, entries.length - 1));
+
+        for (const entry of entries) {
+            const writerKeyHex = b4a.toString(entry.key, 'hex');
+            const addressBuffer = await this.#state.getRegisteredWriterKey(writerKeyHex);
+            if (!addressBuffer) continue;
+
+            const addr = bufferToAddress(addressBuffer, this._config.addressPrefix);
+            if (!addr || addr === this.#wallet.address) continue;
+
+            const publicKey = tracCryptoApi.address.decode(addr);
+            const publicKeyHex = b4a.toString(publicKey, 'hex');
+
+            const isConnected = this.connected(publicKey);
+            const isPending = this.#network.isConnectionPending(publicKeyHex);
+            if (isConnected || isPending) continue;
+
+            await this.#network.tryConnect(publicKeyHex, "indexer");
+        }
+    }
+
+    remove(publicKey, connection = null) {
+        const key = this._toHexString(publicKey);
+        const entry = this._connections.get(key);
+        if (!entry) return;
+        if (connection && entry.connection !== connection) return;
+
+        const targetConnection = connection ?? entry.connection;
+        this._connections.delete(key);
+        targetConnection.protocolSessions.indexer?.close();
+    }
+
+    async send(publicKey, message) {
+        const connection = this.getConnection(publicKey);
+        if (!connection) {
+            throw new Error(`PeerConnectionManager: no session for ${this._toHexString(publicKey)}`);
+        }
+        if (!connection.connected) {
+            this.remove(publicKey, connection);
+            throw new Error(`PeerConnectionManager: stale connection for ${this._toHexString(publicKey)}`);
+        }
+        return connection.protocolSessions.indexer.send(message);
+    }
+
+    sendAndForget(publicKey, message) {
+        const connection = this.getConnection(publicKey);
+        if (!connection) return;
+        connection.protocolSessions.indexer.sendAndForget(message);
+    }
+
+    prettyPrint() {
+        console.log(`Connection count: ${this._connections.size}`);
+        console.log(`Indexer map keys:\n${Array.from(this._connections.keys()).join('\n')}`);
+    }
+}
+
+export default IndexerConnectionManager;

package/src/core/network/services/TransactionPoolService.js

@@ -1,6 +1,6 @@
 // PoolService.js
 import { BATCH_SIZE } from '../../../utils/constants.js';
-import Scheduler from '../../../utils/Scheduler.js';
+import SchedulableService from '../../../utils/scheduler/SchedulableService.js';
 import Denque from "denque";
 import b4a from "b4a";
 
@@ -43,13 +43,12 @@ export class TransactionPoolAlreadyQueuedError extends Error {
     }
 }
 
-class TransactionPoolService {
+class TransactionPoolService extends SchedulableService {
     #state;
     #address;
     #config;
     #txPool = new Denque();
     #transactionCommitService
-    #scheduler = null;
     #queuedTxHashes
 
     /**
@@ -59,6 +58,7 @@ class TransactionPoolService {
      * @param {Config} config
      **/
     constructor(state, address, transactionCommitService, config) {
+        super();
         this.#state = state;
         this.#address = address;
         this.#transactionCommitService = transactionCommitService;
@@ -79,16 +79,15 @@ class TransactionPoolService {
             console.info('TransactionPoolService can not start. Wallet is not enabled');
             return;
         }
-        if (this.#scheduler && this.#scheduler.isRunning) {
-            console.info('TransactionPoolService is already started');
-            return;
-        }
 
-        this.#scheduler = this.#createScheduler();
-        this.#scheduler.start();
+        return super.start();
+    }
+
+    getScheduleInterval() {
+        return this.#config.processIntervalMs;
     }
 
-    async #worker(next) {
+    async worker(next) {
         try {
             await this.#processTransactions();
             if (this.#txPool.size() > 0) {
@@ -101,10 +100,6 @@ class TransactionPoolService {
         }
     }
 
-    #createScheduler() {
-        return new Scheduler((next) => this.#worker(next), this.#config.processIntervalMs);
-    }
-
     async #processTransactions() {
         const canValidate = await this.#checkValidationPermissions();
         if (!canValidate || this.#txPool.size() === 0) return;
@@ -187,10 +182,8 @@ class TransactionPoolService {
         this.txPool.push(txData);
     }
 
-    async stopPool(waitForCurrent = true) {
-        if (!this.#scheduler) return;
-        await this.#scheduler.stop(waitForCurrent);
-        this.#scheduler = null;
+    async stop(waitForCurrent = true) {
+        await super.stop(waitForCurrent);
         this.#queuedTxHashes.clear();
         this.#txPool.clear();
         console.info('TransactionPoolService: closing gracefully...');

package/src/core/network/services/TransactionRateLimiterService.js

@@ -1,5 +1,5 @@
 import b4a from 'b4a';
-import {V1ProtocolError} from "../protocols/v1/V1ProtocolError.js";
+import {V1ProtocolError} from "../protocols/validators/v1/V1ProtocolError.js";
 import {ResultCode} from "../../../utils/constants.js";
 import {publicKeyToAddress} from "../../../utils/helpers.js";
 

package/src/core/network/services/ValidatorConnectionManager.js

@@ -0,0 +1,163 @@
+import {EventType, ResultCode} from '../../../utils/constants.js';
+import {publicKeyToAddress} from "../../../utils/helpers.js";
+import { PeerConnectionManager, PeerConnectionManagerError } from '../shared/PeerConnectionManager.js'
+import ValidatorHealthCheckService from './ValidatorHealthCheckService.js'
+
+const DEFAULT_HEALTH_CHECK_FAILURE_THRESHOLD = 3;
+/**
+ * @typedef {import('hyperswarm').Connection} Connection
+ */
+
+class ValidatorConnectionManager extends PeerConnectionManager {
+    #healthCheckService
+    #messages
+    #healthCheckFailures = new Map();
+    #healthCheckFailureThreshold;
+    // Note: _connections is using publicKey (Buffer) as key
+    // As Buffers are objects, we will rely on internal conversions done by JS to compare them.
+    // It would be better to handle these conversions manually by using hex strings as keys to avoid issues
+    /**
+     * @param {Config} config
+     **/
+    constructor(maxValidators, config, logger, messages) {
+        super(maxValidators, config, logger, messages);
+        this.#messages = messages
+        this.#healthCheckService = new ValidatorHealthCheckService(config);
+        const threshold = Number(config.validatorHealthCheckFailureThreshold);
+        this.#healthCheckFailureThreshold = Number.isInteger(threshold) && threshold > 0
+            ? threshold : DEFAULT_HEALTH_CHECK_FAILURE_THRESHOLD;
+    }
+
+    async _open() {
+        await this.#healthCheckService.ready();
+        this.#subscribeToHealthChecks();
+    }
+
+    async _close() {
+        await this.#healthCheckService.close();
+    }
+
+    /**
+     * Subscribes to periodic validator health checks.
+     * @param {ReadyResource} healthCheckService
+     */
+    // TODO: We should consider moving this to ValidatorObserver instead.
+    // Keep here only if we forsee having health checks for non-validator connections in the future. 
+    // For now, it seems that it would be better to keep this logic here.
+    #subscribeToHealthChecks() {
+        this.#healthCheckService.on(EventType.VALIDATOR_HEALTH_CHECK, async publicKey => {
+            if (!this.exists(publicKey) || !this.connected(publicKey)) {
+                this.#stopHealthCheck(publicKey);
+                return;
+            }
+
+            const connection = this.getConnection(publicKey);
+            if (!connection) {
+                this.#stopHealthCheck(publicKey);
+                this.remove(publicKey);
+                return;
+            }
+
+            let success = false;
+            try {
+                const resultCode = await connection.protocolSessions.validator.sendHealthCheck();
+                success = resultCode === ResultCode.OK;
+            } catch {
+                success = false;
+            }
+
+            // An in-flight health check must not affect a replacement connection.
+            if (this.getConnection(publicKey) !== connection) return;
+            const publicKeyHex = this._toHexString(publicKey);
+            if (!success) {
+                const failures = (this.#healthCheckFailures.get(publicKeyHex) ?? 0) + 1;
+                this.#healthCheckFailures.set(publicKeyHex, failures);
+                if (failures >= this.#healthCheckFailureThreshold) this.remove(publicKey, connection);
+            } else {
+                this.#healthCheckFailures.delete(publicKeyHex);
+            }
+        });
+    }
+
+    #stopHealthCheck(publicKeyHex) {
+        try {
+            if (this.#healthCheckService.has(publicKeyHex)) {
+                this.#healthCheckService.stop(publicKeyHex);
+            }
+        } catch {}
+    }
+
+    async add(publicKey, connection) {
+        this.#messages.attachChannel(connection);
+        try {
+            if (!connection.protocolSessions.validator.isProbed()) {
+                await connection.protocolSessions.validator.probe();
+            }
+        } catch (err) {
+            this._logger.error(`failed to probe peer with publicKey ${publicKey}: ${err?.message ?? err}`);
+        }        
+        if (this._add(publicKey, connection)) {
+            this.#healthCheckFailures.delete(this._toHexString(publicKey));
+        }
+        
+        if (connection.protocolSessions.validator.isHealthCheckSupported()) {
+            this.#healthCheckService.start(publicKey);
+        } else {
+            this.#healthCheckService.stop(publicKey);
+        }
+    }
+
+    remove(publicKey, connection = null) {
+        const publicKeyHex = this._toHexString(publicKey);
+        const entry = this._connections.get(publicKeyHex);
+        if (!entry) return;
+        if (connection && entry.connection !== connection) return;
+
+        this.#stopHealthCheck(publicKeyHex);
+        this.#healthCheckFailures.delete(publicKeyHex);
+        this._connections.delete(publicKeyHex);
+    }
+
+    async sendSingleMessage(message, publicKey) {
+        const publicKeyHex = this._toHexString(publicKey);
+        if (!this.connected(publicKeyHex)) {
+            throw new PeerConnectionManagerError(
+                `Cannot send message: peer ${publicKeyToAddress(publicKey, this._config)} is not connected.`
+            );
+        }
+
+        const connection = this.getConnection(publicKeyHex);
+        if (!connection) {
+            throw new PeerConnectionManagerError(
+                `Cannot send message: no valid connection found for peer ${publicKeyToAddress(publicKeyHex, this._config)}.`
+            );
+        }
+
+        return connection.protocolSessions.validator.send(message);
+    }
+
+    sendAndForget(publicKey, message) {
+        const connection = this.getConnection(publicKey);
+        if (!connection) return;
+        connection.protocolSessions.validator.sendAndForget(message);
+    }
+
+    async send(publicKey, message) {
+        const connection = this.getConnection(publicKey);
+        if (!connection) {
+            throw new Error(`PeerConnectionManager: no session for ${this._toHexString(publicKey)}`);
+        }
+        return connection.protocolSessions.validator.send(message);
+    }
+
+    prettyPrint() {
+        this._logger.info(`Connection count: ${this.connectionCount()}`);
+        this._logger.info(`Validator map keys count: ${this._connections.size}`);
+        this._logger.info(`Validator map keys:\n${Array.from(this._connections.entries()).map(([publicKey, val]) => {
+            const protocols = val.connection?.protocolSessions?.validator?.preferredProtocol || 'none';
+            return `${publicKeyToAddress(publicKey, this._config)}: ${protocols}`;
+        }).join('\n')}`);
+    }
+}
+
+export default ValidatorConnectionManager;