trac-msb 0.1.76 → 0.1.77-msb-r2-dev.1

package/.github/workflows/CI.yml

@@ -0,0 +1,40 @@
+name: MSB-CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        node-version: [lts/*]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+      - name: Install dependencies
+        run: npm ci
+      - name: Run node tests
+        run: npm run test:node
+      - name: Install bare
+        run: npm i -g bare
+      - name: Run bare tests
+        run: npm run test:bare
+      - name: Run rpc acceptance tests
+        run: npm run test:acceptance

package/README.md

@@ -20,9 +20,21 @@ While the MSB supports native node-js, it is encouraged to use Pear:
 cd main_settlement_bus
 npm install -g pear
 npm install
+```
+
+You can run the node in two modes:
+1. Regular node (validator/indexer):
+```js
 pear run . store1
 ```
 
+2. Admin node (access to administrative commands):
+```js
+pear run . admin
+```
+
+The admin mode provides access to additional commands such as `/add_admin`, `/add_whitelist`, `/balance_migration`, `/disable_initialization`, `/add_indexer`, `/remove_indexer`, and `/ban_writer`. These commands are only visible and available when running in admin mode.
+
 **Deploy Bootstrap (admin):**
 
 - Choose option 1)

package/docs/TIP/TIP-000.md

@@ -0,0 +1,49 @@
+# TIP-NNN: _Title_
+
+## Metadata
+- **TIP**: NNN
+- **Title**: _Title_
+- **Author(s)**: _Author_
+- **Type**: CORE | STATE | CONSENSUS | NETWORK | SECURITY | VALIDATION | API | OPERATIONS | TOOLS | META
+- **Status**: Draft | Review | Accepted | Rejected | Final | Superseded
+- **Created**: YYYY-MM-DD
+- **Last Updated**: YYYY-MM-DD
+
+## Summary
+_Provide a clear and concise explanation of the proposal_
+
+## Motivation
+_Explain the why behind this change_
+
+## Specification
+_Detailed technical specification_
+
+## Reference Implementation
+_Code examples or links_
+
+## Benefits
+_List expected benefits_
+
+## Compatibility
+_Describe compatibility concerns_
+
+## Validation / Testing Plan
+_Describe how this will be tested_
+
+## Security Considerations
+_List security implications_
+
+## References
+_List related documentation_
+
+[//]: # (Type Legend:)
+[//]: # (CORE: Fundamental changes to system architecture)
+[//]: # (STATE: Changes to state management and storage mechanisms)
+[//]: # (CONSENSUS: Changes to consensus mechanisms)
+[//]: # (NETWORK: Changes to network protocol and communication)
+[//]: # (SECURITY: Security and validation improvements)
+[//]: # (VALIDATION: Changes to operation validation logic)
+[//]: # (API: Changes to API interfaces)
+[//]: # (OPERATIONS: Changes to business operations)
+[//]: # (TOOLS: Developer and administrative tools)
+[//]: # (META: Changes to processes, documentation, or standards)

package/docs/TIP/TIP-001.md

@@ -0,0 +1,220 @@
+# TRAC-IMPROVMENT-PROPOSALS-001: State handlers refactoring
+
+## Status
+- **TIP**: 001
+- **Title**: State handlers refactoring
+- **Authors**: b-garbacz
+- **Type**: Technical design proposal 
+- **Status**: Draft
+- **Created**: 2023-09-26
+- **Last Updated**: 2023-09-26
+
+## Summary
+Proposal to refactor the current monolithic State handlers into a modular, maintainable architecture using the Handler pattern.
+
+## Motivation
+Current implementation in `State.js` faces several challenges:
+- Large file size (~3000 lines) making it hard to maintain
+- Duplicate code across handlers
+- Difficult to test individual handler logic
+- Complex error handling spread across methods
+- Limited code reuse between similar operations
+
+## Specification
+
+> **Note**: The following code snippets are abstract examples illustrating the refactoring concept. Any feedback for their improvement is crucial for creating more maintainable code.
+
+### 1. Core Components
+
+#### 1.1 Base Handler
+```javascript
+export class BaseHandler {
+    #state;
+    #enableTxLogs;
+
+    constructor(state, enableTxLogs = true) {
+        this.#state = state;
+        this.#enableTxLogs = enableTxLogs;
+    }
+
+    async handle(op, view, base, node, batch) {
+        return null;
+    }
+
+    #safeLog(operationType, message, validatorWk) {
+        if (this.#enableTxLogs) {
+            this.#state.safeLogApply(operationType, message, validatorWk);
+        }
+    }
+
+    async #validateAddressFormat(address, operationType, nodeKey) {
+        const addressString = addressUtils.bufferToAddress(address);
+        if (addressString === null) {
+            this.#safeLog(operationType, "Invalid address format.", nodeKey);
+            return null;
+        }
+        return addressString;
+    }
+}
+```
+
+#### 1.2 Handler Factory
+```javascript
+export class HandlerFactory {
+    #handlers;
+
+    constructor(state) {
+        this.#handlers = new Map();
+        this.initializeHandlers(state);
+    }
+
+    getHandler(operationType) {
+        const handler = this.#handlers.get(operationType);
+        if (!handler) {
+            console.error(`No handler found for operation type: ${operationType}`);
+            return null;
+        }
+        return handler;
+    }
+}
+```
+
+### 2. Directory Structure
+```
+src/core/state/
+├── handlers/
+│   ├── base/
+│   │   ├── BaseHandler.js
+│   ├── addAdminContract/
+│   │   ├── AddAdminHandler.js
+│   ├── adminRecoveryContract/
+│   │   └── AdminRecoveryHandler.js
+│   ├── addWriterContract/
+│   │   ├── AddWriterHandler.js
+│   ├── addRemoveContract/
+│   │   └── RemoveWriterHandler.js
+│   ├── addIndexercontract/
+│   │   ├── AddIndexerHandler.js
+│   ├── removeIndexercontract/
+│   │   └── RemoveIndexerHandler.js // and other contracts
+│   └── HandlerFactory.js
+├── utils/
+│   ├── balance.js // and other utils
+├── validation/
+│   └── BalanceValidator.js // we can separate implement specific validators which can be used everywhere
+└── State.js
+```
+
+### 3. Integration with State Class
+
+```javascript
+const HandlerFactory = require('./handlers/HandlerFactory');
+
+class State extends ReadyResource {
+    constructor() {
+        super();
+        this.handlerFactory = new HandlerFactory(this);
+    }
+
+    #getApplyOperationHandler(type) {
+        return this.handlerFactory.getHandler(type);
+    }
+
+    async #apply(nodes, view, base) {
+        const batch = this.#setupHyperbee(base);
+        
+        for (const node of nodes) {
+            const operation = node.value;
+            const handler = this.#getApplyOperationHandler(operation.type);
+            await handler.handle(operation, view, base, node, batch);
+        }
+        
+        await batch.flush();
+        await batch.clode();
+    }
+    
+}
+```
+
+### 4. Handler Implementation Example
+```javascript
+export class AddWriterHandler extends BaseHandler {
+    async handle(op, view, base, node, batch) {
+        const schemaValidation = await this.#validateRoleAccessOperation(op);
+        if (!schemaValidation) { validatorPunishment(); return;}
+
+        const addressData = await this.#validateAddress(op, node);
+        if (!addressData) { validatorPunishment(); return;}
+
+        const signatureData = await this.#validateSignatures(op, base, batch);
+        if (!signatureData) { validatorPunishment(); return;}
+
+        await this.#addWriter(
+            op,
+            base,
+            node,
+            batch,
+            ...
+        );
+    }
+```
+
+### 5. Common Functionality Modules
+```javascript
+export class BalanceValidator {
+    #safeLog;
+
+    constructor(safeLog) {
+        this.#safeLog = safeLog;
+    }
+
+    validateFeePayment(nodeEntry, balance, fee, validatorWk) {
+        if (!balance.greaterThanOrEquals(fee)) {
+            this.#safeLog("Balance", "Insufficient balance for fee payment", validatorWk);
+            return null;
+        }
+        return {
+            newBalance: balance.sub(fee)
+        };
+    }
+}
+```
+
+## Benefits
+
+1. **Maintainability**
+   - Smaller, focused files
+   - Clear separation of concerns
+   - Easier to understand and modify individual handlers
+
+2. **Testability**
+   - Isolated handler testing
+   - Mockable dependencies
+
+3. **Code Reuse**
+   - Shared validation logic
+
+## Compatibility Requirements
+
+1. **Behavioral Compatibility**
+   - Refactored code MUST behave identically to the current implementation
+   - All validations and errors MUST be returned at the same points
+   - Operation execution order MUST remain unchanged
+   - Business logic MUST be identical, only code structure changes
+
+2. **State Consistency**
+   - DAG in refactored node MUST be byte-by-byte identical with other nodes
+   - All state operations MUST produce the same results
+   - State replication between nodes MUST work without changes
+   - Batch processing MUST maintain the same atomicity
+
+3. **Network Compatibility**
+   - Nodes with old and new implementation MUST work together in the network
+   - Message format and communication MUST remain unchanged
+   - No hard fork or protocol changes should be required
+
+4. **Validation Points**
+   - Byte-by-byte comparison of all state operations
+   - Verification of DAG identity after replication
+   - Testing node cooperation in testnet
+   - Verification of batch processing consistency

package/msb.mjs

@@ -1,14 +1,45 @@
 import {MainSettlementBus} from './src/index.js';
 
+const isPear = typeof Pear !== 'undefined';
+const args = isPear ? Pear.config.args : process.argv.slice(2);
+
 const opts = {
-    stores_directory : 'stores2/',
+    stores_directory : 'stores/',
     store_name : typeof process !== "undefined" ? process.argv[2] : Pear.config.args[0],
-    bootstrap: 'a4951e5f744e2a9ceeb875a7965762481dab0a7bb0531a71568e34bf7abd2c53',
-    channel: '0002tracnetworkmainsettlementbus',
+    bootstrap: '602d5443c19014e36a01254923afb1df56099d559f282761d70370a0da5d1d8a',
+    channel: 'TESTNET0002tracnetworkmainsettlementbus',
+    enable_role_requester: false,
+    enable_auto_transaction_consent: false,
+    enable_wallet: true,
+    enable_validator_observer: true,
+    enable_interactive_mode: true,
+    disable_rate_limit: true,
+    enable_txlogs: true,
 };
 
-const msb = new MainSettlementBus(opts);
+const rpc_opts = {
+    ...opts,
+    enable_txlogs: false,
+}
+
+const msb = new MainSettlementBus(args.includes('--rpc') ? rpc_opts : opts);
+
+msb.ready().then(async () => {
+    const runRpc = args.includes('--rpc');
+
+    if (runRpc) {
+        console.log('Starting RPC server...');
+        const portIndex = args.indexOf('--port');
+        const port = (portIndex !== -1 && args[portIndex + 1]) ? parseInt(args[portIndex + 1], 10) : 5000;
+        const hostIndex = args.indexOf('--host');
+        const host = (hostIndex !== -1 && args[hostIndex + 1]) ? args[hostIndex + 1] : 'localhost';
+
+        const {startRpcServer} = await import('./rpc/rpc_server.mjs');
+        startRpcServer(msb, host, port);
+    } else {
+        console.log('RPC server will not be started.');
+    }
 
-msb.ready().then(() => {
     msb.interactiveMode();
-});
+});
+

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "trac-msb",
   "main": "msb.mjs",
-  "version": "0.1.76",
+  "version": "0.1.77-msb-r2-dev.1",
   "pear": {
     "name": "trac-msb",
     "type": "terminal"
@@ -9,32 +9,56 @@
   "type": "module",
   "scripts": {
     "dev": "pear run -d .",
-    "test:node": "brittle test/all.test.js",
-    "test:bare": "bare test/all.test.js"
+    "dev-rpc": "pear run -d . ${npm_config_store} --rpc --port ${npm_config_port}",
+    "prod-rpc": "pear run . ${npm_config_store} --rpc --host ${npm_config_host} --port ${npm_config_port}",
+    "protobuf": "node scripts/generate-protobufs.js",
+    "test:acceptance": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testTimeout=200000 test/acceptance/",
+    "test:node": "brittle-node -t 120000 test/all.test.js",
+    "test:bare": "brittle-bare -t 120000 test/all.test.js",
+    "test:node:cov": "brittle-node -c -t 120000 test/all.test.js",
+    "test:bare:cov": "brittle-bare -c -t 120000 test/all.test.js"
   },
   "dependencies": {
-    "autobase": "7.6.3",
-    "hypercore": "^11.6.3",
-    "corestore": "^7.4.2",
+    "@tracsystems/blake3": "^0.0.13",
+    "autobase": "7.16.1",
     "b4a": "1.6.7",
+    "bare-crypto": "^1.11.2",
+    "bare-fs": "^4.1.6",
+    "bare-http1": "^4.0.4",
     "bare-readline": "1.0.7",
     "bare-tty": "5.0.2",
+    "bech32": "^2.0.0",
+    "brittle": "^3.17.1",
     "compact-encoding": "^2.16.0",
+    "corestore": "^7.4.2",
+    "crypto": "npm:bare-node-crypto",
     "fastest-validator": "1.19.0",
+    "http": "npm:bare-node-http",
     "hyperbee": "^2.24.2",
+    "hypercore": "^11.6.3",
     "hypercore-crypto": "^3.4.0",
     "hyperdht": "^6.20.5",
     "hyperswarm": "^4.11.5",
+    "protocol-buffers-encodings": "^1.2.0",
     "protomux": "^3.10.1",
     "protomux-wakeup": "2.4.0",
     "readline": "npm:bare-node-readline",
     "ready-resource": "1.1.2",
-    "trac-wallet": "0.0.43",
+    "trac-wallet": "0.0.43-msb-r2.4",
     "tty": "npm:bare-node-tty"
   },
+  "devDependencies": {
+    "bare-os": "^3.6.1",
+    "bip39-mnemonic": "^2.4.0",
+    "esmock": "^2.7.2",
+    "jest": "^30.1.3",
+    "protocol-buffers": "^4.2.0",
+    "sinon": "^21.0.0",
+    "supertest": "^7.1.4",
+    "trac-crypto-api": "^0.0.10"
+  },
   "publishConfig": {
     "registry": "https://registry.npmjs.org",
-    "access": "public",
-    "brittle": "^3.16.2"
+    "access": "public"
   }
 }

package/proto/applyOperations.proto

@@ -0,0 +1,117 @@
+syntax = "proto3";
+
+package apply.operations;
+
+//DO NOT CHANGE THE ORDER AFTER RELEASE - ONLY APPEND NEW ONES
+enum OperationType {
+    UNKNOWN                = 0;
+    ADD_ADMIN              = 1; // complete
+    DISABLE_INITIALIZATION = 2; // complete
+    BALANCE_INITIALIZATION = 3; // complete
+    APPEND_WHITELIST       = 4; // complete
+    ADD_WRITER             = 5; // partial
+    REMOVE_WRITER          = 6; // partial
+    ADMIN_RECOVERY         = 7; // partial
+    ADD_INDEXER            = 8; // complete
+    REMOVE_INDEXER         = 9; // complete
+    BAN_VALIDATOR          = 10; // complete
+    BOOTSTRAP_DEPLOYMENT   = 11; // partial
+    TX                     = 12; // partial
+    TRANSFER               = 13; // partial
+}
+
+// add admin operation, disable_initialization (complete by default)
+message CoreAdminOperation {
+    bytes tx     = 1; // Transaction hash (unique identifier for the transaction)
+    bytes txv    = 2; // Transaction expiration
+    bytes iw     = 3;  // incoming writer key (admin)
+    bytes in     = 4; // Nonce of the invoker (admin)
+    bytes is     = 5; // Signature of the invoker (admin)
+}
+
+// append_whitelist, add_indexer, remove_indexer, ban_writer (complete by default)
+message AdminControlOperation {
+    bytes tx     = 1; // Transaction hash (unique identifier for the transaction)
+    bytes txv    = 2; // Transaction expiration
+    bytes in     = 3; // Nonce of the invoker (admin)
+    bytes ia     = 4; // selected address to specific operation.
+    bytes is     = 5; // Signature of the invoker (admin)
+}
+
+// balance_initialization (complete by default)
+message BalanceInitializationOperation {
+    bytes tx   = 1; // Transaction hash (unique identifier for the transaction)
+    bytes txv  = 2; // Transaction expiration
+    bytes in   = 3; // Nonce of the contract invoker (admin)
+    bytes ia   = 4; // selected address to specific operation.
+    bytes am   = 5; // Initial balance to be set for the address
+    bytes is   = 6; // Signature of the contract invoker (admin)
+}
+// token_transfer (partial - need to be signed by another writer to be complete)
+message TransferOperation {
+    bytes tx      = 1; // Transaction hash (unique identifier for the transaction)
+    bytes txv     = 2; // Transaction expiration
+    bytes in      = 3; // Nonce of the invoker (any account)
+    bytes to      = 4; // Address of the recipient (any account)
+    bytes am      = 5; // Amount of tokens to transfer
+    bytes is      = 6; // Signature of the invoker (any account)
+    bytes va      = 7; // Validator's address (optional)
+    bytes vn      = 8; // Validator's nonce  (optional)
+    bytes vs      = 9; // Validator's signature (optional)
+}
+
+// add_writer, remove_writer, admin_recovery (partial - need to be signed by another writer to be complete)
+message RoleAccessOperation  {
+    bytes tx      = 1;   // Transaction hash (unique identifier for the transaction)
+    bytes txv     = 2;   // Transaction expiration
+    bytes iw      = 3;  // Writing key of the invoker (whitelisted node, admin to be recovered, Trac Network)
+    bytes in      = 4;  // Nonce of the invoker (whitelisted node, admin to be recovered, Trac Network)
+    bytes is      = 5;  // Signature of the invoker (whitelisted node, admin to be recovered)
+    bytes va      = 6; // Validator's address (optional)
+    bytes vn      = 7; // Validator's nonce  (optional)
+    bytes vs      = 8; // Validator's signature (optional)
+}
+
+// transaction - (partial - need to be signed by another writer to be complete)
+message TxOperation {
+    bytes tx   = 1;  // Transaction hash (unique identifier for the transaction)
+    bytes txv  = 2;   // Transaction expiration
+    bytes iw   = 3;  // Writing key of the requesting node (external subnetwork)
+    bytes in   = 4;  // Nonce of the requesting node
+    bytes ch   = 5;  // Content hash (hash of the transaction's data)
+    bytes is   = 6;  // Requester's signature
+    bytes bs   = 7;  // External bootstrap contract
+    bytes mbs  = 8;  // MSB bootstrap key
+    bytes va   = 9; // Validator's address (optional)
+    bytes vn   = 10; // Validator's nonce  (optional)
+    bytes vs   = 11; // Validator's signature (optional)
+}
+
+// bootstrap deployment - (partial - need to be signed by another writer to be complete)
+message BootstrapDeploymentOperation {
+    bytes tx   = 1; // Transaction hash (unique identifier for the transaction)
+    bytes txv  = 2; // Transaction expiration
+    bytes bs   = 3; // Bootstrap contract address
+    bytes ic   = 4; // Channel on which the contract will operate
+    bytes in   = 5; // Nonce of the contract invoker
+    bytes is   = 6; // Signature of the contract invoker
+    bytes va   = 7; // Validator's address (optional)
+    bytes vn   = 8; // Validator's nonce (optional)
+    bytes vs   = 9; // Validator's signature (optional)
+}
+
+// Main structure for the operations where messages are
+message Operation {
+    OperationType type = 1;
+    bytes address      = 2; // address of the invoker
+
+    oneof value {
+        CoreAdminOperation                cao  = 3;
+        AdminControlOperation             aco  = 4;
+        BalanceInitializationOperation    bio  = 5;
+        TransferOperation                 tro  = 6;
+        RoleAccessOperation               rao  = 7;
+        BootstrapDeploymentOperation      bdo  = 8;
+        TxOperation                       txo  = 9;
+    }
+}

package/rpc/constants.mjs

@@ -0,0 +1,2 @@
+export const MAX_SIGNED_LENGTH = 1000
+export const SIGNED_LENGTH_OFFSET = 20

package/rpc/cors.mjs

@@ -0,0 +1,15 @@
+export function applyCors(req, res) {
+    // Set CORS headers
+    res.setHeader('Access-Control-Allow-Origin', '*'); // Allow all origins, or specify a domain
+    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+    res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+    // Handle preflight OPTIONS request
+    if (req.method === 'OPTIONS') {
+        res.writeHead(204);
+        res.end();
+        return true; // Return true to indicate the request was handled
+    }
+
+    return false; // Return false to indicate the request should be handled by the main router
+}

package/rpc/create_server.mjs

@@ -0,0 +1,39 @@
+// rpc_server.mjs
+import http from 'http'
+import { applyCors } from './cors.mjs';
+import { routes } from './router.mjs'; // Import the new routes array
+
+export const createServer = (msbInstance) => {
+  const server = http.createServer({}, async (req, res) => {
+    if (applyCors(req, res)) return;
+
+    const respond = (code, paylaod) => {
+      res.writeHead(code, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify(paylaod));
+    }
+
+    // Find the matching route
+    let foundRoute = false;
+    for (const route of routes) {
+        // Simple path matching
+        if (req.method === route.method && req.url.startsWith(route.path)) {
+            foundRoute = true;
+            try {
+                await route.handler({ req, res, respond, msbInstance });
+            } catch (error) {
+                console.error(`Error on ${route.path}:`, error);
+                res.writeHead(500, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({ error: 'An error occurred processing the request.' }));
+            }
+            break;
+        }
+    }
+
+    if (!foundRoute) {
+        res.writeHead(404, { 'Content-Type': 'text/plain' });
+        res.end('Not Found');
+    }
+  });
+
+  return server
+}