apsimo 1.3.0__py3-none-any.whl

apsimo/chain/local_keys.py

@@ -0,0 +1,187 @@
+"""Colony Local Key Manager — PEM-based Ed25519 signing for Phase 1/2.
+
+Provides the same interface as ColonyKeyManager (sign, public_key_hex, rotate)
+but stores the key as a simple PEM file. Designed to be swapped out for the
+Shamir-backed ColonyKeyManager when Phase 3/4/5 ship.
+
+Key storage:
+    {state_dir}/colony-keys/
+        private.pem    — Ed25519 private key (optionally encrypted)
+        public.pem     — Ed25519 public key (always readable)
+
+Migration path:
+    Phase 1/2: LocalKeyManager (PEM, optional passphrase)
+    Phase 3:   LocalKeyManager with passphrase (colony key set-passphrase)
+    Phase 4/5: ColonyKeyManager (Shamir-split, colony key migrate-to-shamir)
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Callable, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class LocalKeyManager:
+    """PEM-based Ed25519 key manager for single-node Colony deployments."""
+
+    def __init__(
+        self,
+        keys_dir: str | Path,
+        colony_id: str,
+        passphrase: Optional[bytes] = None,
+    ) -> None:
+        self._keys_dir = Path(keys_dir)
+        self.colony_id = colony_id
+        self._passphrase = passphrase
+        self._keys_dir.mkdir(parents=True, exist_ok=True)
+
+    # ------------------------------------------------------------------
+    # Public interface (matches ColonyKeyManager)
+    # ------------------------------------------------------------------
+
+    def sign(self, payload: bytes) -> str:
+        """Sign payload bytes, returning hex-encoded Ed25519 signature."""
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        private_key = self._load_private_key()
+        signature = private_key.sign(payload)
+        return signature.hex()
+
+    def public_key_hex(self) -> str:
+        """Return the hex-encoded raw Ed25519 public key."""
+        from cryptography.hazmat.primitives import serialization
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        private_key = self._load_private_key()
+        pub = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw,
+        )
+        return pub.hex()
+
+    def public_key_pem(self) -> str:
+        """Return the PEM-encoded public key."""
+        pub_path = self._keys_dir / "public.pem"
+        if pub_path.exists():
+            return pub_path.read_text()
+        # Generate from private key
+        from cryptography.hazmat.primitives import serialization
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        private_key = self._load_private_key()
+        pem = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+        pub_path.write_bytes(pem)
+        return pem.decode()
+
+    def rotate(self, new_passphrase: Optional[bytes] = None) -> None:
+        """Generate a new Ed25519 keypair, replacing the existing one.
+
+        The colony_id does NOT change — only the signing key rotates.
+        """
+        from cryptography.hazmat.primitives import serialization
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        # Backup old key
+        priv_path = self._keys_dir / "private.pem"
+        if priv_path.exists():
+            backup = self._keys_dir / "private.pem.backup"
+            backup.write_bytes(priv_path.read_bytes())
+
+        # Generate new keypair
+        private_key = Ed25519PrivateKey.generate()
+        self._save_keypair(private_key, new_passphrase or self._passphrase)
+        logger.info("Key rotated for colony %s", self.colony_id)
+
+    def set_passphrase(self, new_passphrase: bytes) -> None:
+        """Re-encrypt the private key with a new passphrase."""
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        private_key = self._load_private_key()
+        self._save_keypair(private_key, new_passphrase)
+        self._passphrase = new_passphrase
+        logger.info("Passphrase updated for colony %s", self.colony_id)
+
+    @property
+    def has_keys(self) -> bool:
+        """Check if keypair exists on disk."""
+        return (self._keys_dir / "private.pem").exists()
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _load_private_key(self):
+        from cryptography.hazmat.primitives import serialization
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        priv_path = self._keys_dir / "private.pem"
+        if not priv_path.exists():
+            raise FileNotFoundError(
+                f"No private key found at {priv_path}. Run 'colony init' or 'colony key generate'."
+            )
+
+        pem_data = priv_path.read_bytes()
+        try:
+            return serialization.load_pem_private_key(
+                pem_data,
+                password=self._passphrase,
+            )
+        except (ValueError, TypeError):
+            # Wrong passphrase or unencrypted key with passphrase given
+            if self._passphrase:
+                raise ValueError("Wrong passphrase for private key") from None
+            # Try without passphrase
+            return serialization.load_pem_private_key(pem_data, password=None)
+
+    def _save_keypair(self, private_key, passphrase: Optional[bytes] = None) -> None:
+        from cryptography.hazmat.primitives import serialization
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        # Save private key
+        encryption: Any = serialization.NoEncryption()
+        if passphrase:
+            encryption = serialization.BestAvailableEncryption(passphrase)
+
+        priv_pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=encryption,
+        )
+        (self._keys_dir / "private.pem").write_bytes(priv_pem)
+
+        # Save public key
+        pub_pem = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+        (self._keys_dir / "public.pem").write_bytes(pub_pem)
+
+        # Restrict permissions
+        os.chmod(self._keys_dir / "private.pem", 0o600)
+
+    @classmethod
+    def generate(
+        cls,
+        keys_dir: str | Path,
+        colony_id: str,
+        passphrase: Optional[bytes] = None,
+    ) -> "LocalKeyManager":
+        """Generate a new Ed25519 keypair and return a LocalKeyManager."""
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+        mgr = cls(keys_dir=keys_dir, colony_id=colony_id, passphrase=passphrase)
+        private_key = Ed25519PrivateKey.generate()
+        mgr._save_keypair(private_key, passphrase)
+        logger.info("Generated new Ed25519 keypair for colony %s", colony_id)
+        return mgr
+
+
+# Fix type annotation for Python 3.11 compat
+from typing import Any

apsimo/chain/manager.py

@@ -0,0 +1,290 @@
+"""ChainManager — main coordinator tying all chain components together.
+
+Responsibilities:
+- Owns the ChainStore, ChainStateMachine, and Mempool
+- Chain sync protocol (pull blocks from Sentinels)
+- New block announcement broadcasting
+- Transaction submission and nonce tracking
+- Integration hook for the federation module
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import sqlite3
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable, Coroutine, Optional
+
+from apsimo import get_state_dir
+
+from .block import Block
+from .state_machine import ChainStateMachine
+from .storage import ChainStore
+from .transactions import ChainState, Transaction, TxType
+from .validation import BlockValidator, TransactionValidator, ValidationResult
+from .protocol import (
+    ChainAnnouncement,
+    ChainStateResponse,
+    ChainSyncRequest,
+    ChainSyncResponse,
+    TxSubmitResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+_CHECKPOINT_INTERVAL = 100
+
+_chain_manager_instance: Optional["ChainManager"] = None
+
+
+@dataclass
+class SubmitResult:
+    accepted: bool
+    tx_id: str
+    reason: str = ""
+
+
+class ChainManager:
+    """Main coordinator for ColonyChain.
+
+    Manages the local chain replica, handles sync with Sentinels,
+    validates and queues transactions, and broadcasts new blocks.
+    """
+
+    def __init__(
+        self,
+        db_path: Path | str,
+        colony_id: str,
+        send_message: Callable[[str, dict[str, Any]], Coroutine] | None = None,
+    ) -> None:
+        self.colony_id = colony_id
+        self.store = ChainStore(db_path)
+        self.state_machine = ChainStateMachine(self.store)
+        self._send_message = send_message
+        self._state: ChainState | None = None
+        self._state_lock = asyncio.Lock()
+        self._block_validator = BlockValidator()
+
+    # ── State access ────────────────────────────────────────────────────────
+
+    async def get_state(self) -> ChainState:
+        """Return current chain state (thread-safe cached)."""
+        async with self._state_lock:
+            if self._state is None:
+                self._state = self.state_machine.get_current_state()
+            return self._state
+
+    def _invalidate_state(self) -> None:
+        self._state = None
+
+    # ── Transaction submission ───────────────────────────────────────────────
+
+    async def submit_transaction(self, tx: Transaction) -> SubmitResult:
+        """Validate and add a transaction to the mempool."""
+        state = await self.get_state()
+
+        # Nonce check (against persisted chain)
+        if tx.from_colony_id != "system":
+            persisted_nonce = self.store.get_nonce(tx.from_colony_id)
+            if tx.nonce <= persisted_nonce:
+                return SubmitResult(
+                    accepted=False,
+                    tx_id=tx.tx_id,
+                    reason=f"nonce {tx.nonce} <= last accepted {persisted_nonce}",
+                )
+
+        validator = TransactionValidator(state, chain_store=self.store)
+        result = validator.validate(tx)
+        if not result.ok:
+            return SubmitResult(accepted=False, tx_id=tx.tx_id, reason=result.reason)
+
+        self.store.add_to_mempool(tx)
+        return SubmitResult(accepted=True, tx_id=tx.tx_id)
+
+    # ── Block commitment ─────────────────────────────────────────────────────
+
+    async def commit_block(self, block: Block) -> None:
+        """Commit a block to the chain and update state."""
+        latest = self.store.get_latest_block()
+        bv = self._block_validator
+        result = bv.validate_block(block, latest)
+        if not result.ok:
+            raise ValueError(f"Block validation failed: {result.reason}")
+
+        self.store.append_block(block)
+        self._invalidate_state()
+
+        # Save checkpoint every 100 blocks
+        if block.index > 0 and block.index % _CHECKPOINT_INTERVAL == 0:
+            state = await self.get_state()
+            self.store.save_checkpoint(state, block.index)
+
+        # Announce to peers
+        announcement = ChainAnnouncement(
+            block_index=block.index,
+            block_hash=block.block_hash,
+            producer_id=block.producer_id,
+            tx_count=len(block.transactions),
+        )
+        await self._broadcast(announcement.to_dict())
+
+        logger.info(
+            "Block %d committed (%d txs, hash %s…)",
+            block.index,
+            len(block.transactions),
+            block.block_hash[:16],
+        )
+
+    # ── Chain sync ───────────────────────────────────────────────────────────
+
+    async def sync_from_sentinel(
+        self,
+        sentinel_id: str,
+        from_height: int | None = None,
+    ) -> int:
+        """Pull blocks from a Sentinel to catch up. Returns blocks added."""
+        if from_height is None:
+            from_height = self.store.get_height() + 1
+
+        req = ChainSyncRequest(
+            from_index=from_height,
+            to_index=from_height + 499,  # batch up to 500 blocks
+            requester_id=self.colony_id,
+        )
+        if self._send_message:
+            await self._send_message(sentinel_id, req.to_dict())
+        return 0
+
+    async def handle_chain_sync_response(self, data: dict[str, Any]) -> int:
+        """Process a CHAIN_SYNC_RESPONSE message. Returns blocks committed."""
+        blocks_data = data.get("blocks", [])
+        committed = 0
+        for block_dict in blocks_data:
+            block = Block.from_dict(block_dict)
+            try:
+                await self.commit_block(block)
+                committed += 1
+            except ValueError as exc:
+                logger.warning("Skipping block %d: %s", block.index, exc)
+                break
+        return committed
+
+    async def handle_chain_announce(self, data: dict[str, Any]) -> None:
+        """Handle a CHAIN_ANNOUNCE — trigger sync if we're behind."""
+        announced_height = data.get("block_index", 0)
+        our_height = self.store.get_height()
+        if announced_height > our_height + 1:
+            sender_id = data.get("producer_id", "")
+            if sender_id:
+                await self.sync_from_sentinel(sender_id, from_height=our_height + 1)
+
+    async def handle_tx_submit(
+        self, data: dict[str, Any]
+    ) -> TxSubmitResponse:
+        """Handle a TX_SUBMIT message from another colony."""
+        tx_json = data.get("tx_json", "{}")
+        try:
+            tx_dict = json.loads(tx_json)
+            tx = Transaction.from_dict(tx_dict)
+        except Exception as exc:
+            return TxSubmitResponse(tx_id="", accepted=False, reason=str(exc))
+
+        result = await self.submit_transaction(tx)
+        return TxSubmitResponse(
+            tx_id=result.tx_id,
+            accepted=result.accepted,
+            reason=result.reason,
+        )
+
+    async def get_chain_state_summary(
+        self, responder_id: str | None = None
+    ) -> ChainStateResponse:
+        """Build a ChainStateResponse summary."""
+        state = await self.get_state()
+        return ChainStateResponse(
+            height=state.height,
+            last_block_hash=state.last_block_hash,
+            n_colonies=len(state.colony_registry),
+            n_sentinels=len(state.active_sentinels()),
+            n_suspended=len(state.suspended_colonies),
+            network_id=state.network_id,
+            responder_id=responder_id or self.colony_id,
+        )
+
+    # ── Mempool helpers ──────────────────────────────────────────────────────
+
+    def drain_mempool(self, max_txs: int = 500) -> list[Transaction]:
+        """Drain pending transactions for block production."""
+        return self.store.get_mempool(limit=max_txs)
+
+    def evict_included(self, tx_ids: list[str]) -> None:
+        self.store.remove_from_mempool(tx_ids)
+
+    # ── Internal helpers ─────────────────────────────────────────────────────
+
+    async def _broadcast(self, msg: dict[str, Any]) -> None:
+        """Broadcast a message to all known peers (best-effort)."""
+        # In a real deployment this would iterate discovered peers.
+        # Here we call the send_message hook if provided.
+        if self._send_message:
+            state = await self.get_state()
+            for sentinel in state.active_sentinels():
+                try:
+                    await self._send_message(sentinel.sentinel_id, msg)
+                except Exception as exc:
+                    logger.debug(
+                        "Failed to send to sentinel %s: %s", sentinel.sentinel_id, exc
+                    )
+
+    def get_height(self) -> int:
+        return self.store.get_height()
+
+    @classmethod
+    def get_instance(cls) -> "ChainManager":
+        """Return the process-wide singleton backed by ~/.colony/data/chain.db."""
+        global _chain_manager_instance
+        if _chain_manager_instance is None:
+            colony_home = get_state_dir()
+            db_path = colony_home / "chain.db"
+            # Derive colony_id from genesis block cryptographic identity
+            colony_id = "local"
+            if db_path.exists():
+                try:
+                    conn = sqlite3.connect(str(db_path))
+                    row = conn.execute(
+                        "SELECT raw_json FROM chain_blocks WHERE block_index = 0"
+                    ).fetchone()
+                    if row:
+                        genesis_data = json.loads(row[0])
+                        # Use the cryptographic colony_id (sha256 of Ed25519 pubkey)
+                        # from genesis block metadata — not producer_id which is
+                        # just a human-readable label and trivially spoofable.
+                        colony_id = (
+                            genesis_data.get("metadata", {}).get("genesis_colony_id")
+                            or genesis_data.get("producer_id", "local")
+                        )
+                    conn.close()
+                except Exception:
+                    pass
+            _chain_manager_instance = cls(db_path=db_path, colony_id=colony_id)
+        return _chain_manager_instance
+
+    def get_status(self) -> dict:
+        """Return chain status as a plain dict for the API router."""
+        height = self.store.get_height()
+        latest = self.store.get_latest_block()
+        mempool = self.store.get_mempool(limit=10000)
+        return {
+            "height": height,
+            "last_block_hash": latest.block_hash if latest else "0" * 64,
+            "last_block_timestamp": latest.timestamp if latest else None,
+            "colony_id": self.colony_id,
+            "mempool_size": len(mempool),
+            "protocol_version": "1.0.0",
+            "sync_status": "synced",
+        }

apsimo/chain/node.py

@@ -0,0 +1,163 @@
+"""Colony Node Identity — per-device identity within a Colony.
+
+Each physical device running a Colony instance is a Node. Nodes have:
+    node_id   — unique UUID per device, generated on first `colony start`
+    node keypair — Ed25519, independent from Colony keypair
+    node cert — signed by Colony private key, proving membership
+
+The Colony is the logical identity. Nodes are the physical instances.
+One Colony can run on many devices — each gets its own node_id.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+NODE_ID_FILE = "node-id"
+NODE_KEYS_DIR = "node-keys"
+NODE_CERT_FILE = "node-cert.json"
+
+
+def get_or_create_node_id(state_dir: str | Path) -> str:
+    """Get or create the node_id for this device."""
+    id_path = Path(state_dir) / NODE_ID_FILE
+    if id_path.exists():
+        node_id = id_path.read_text().strip()
+        if node_id:
+            return node_id
+
+    node_id = str(uuid.uuid4())
+    id_path.parent.mkdir(parents=True, exist_ok=True)
+    id_path.write_text(node_id)
+    logger.info("Generated new node_id: %s", node_id)
+    return node_id
+
+
+def ensure_node_keypair(state_dir: str | Path) -> "LocalKeyManager":
+    """Ensure this device has a node keypair. Generate if missing.
+
+    Returns a LocalKeyManager for the node keys.
+    """
+    from apsimo.chain.local_keys import LocalKeyManager
+
+    node_id = get_or_create_node_id(state_dir)
+    keys_dir = Path(state_dir) / NODE_KEYS_DIR
+    keys_dir.mkdir(parents=True, exist_ok=True)
+
+    if (keys_dir / "private.pem").exists():
+        return LocalKeyManager(keys_dir=keys_dir, colony_id=node_id)
+
+    km = LocalKeyManager.generate(keys_dir=keys_dir, colony_id=node_id)
+    logger.info("Generated node keypair for node %s", node_id)
+    return km
+
+
+def create_node_certificate(
+    state_dir: str | Path,
+    colony_key_manager: Optional["LocalKeyManager"] = None,
+) -> dict:
+    """Create a node certificate signed by the Colony's private key.
+
+    The certificate binds: colony_id, node_id, node_public_key, issued_at.
+    It proves this device belongs to the Colony.
+    """
+    from apsimo.chain.local_keys import LocalKeyManager
+    from apsimo.chain.identity import get_or_create_colony_id
+
+    state_dir = Path(state_dir)
+    colony_id = get_or_create_colony_id(state_dir)
+    node_id = get_or_create_node_id(state_dir)
+    node_km = ensure_node_keypair(state_dir)
+    node_pubkey = node_km.public_key_hex()
+
+    cert = {
+        "colony_id": colony_id,
+        "node_id": node_id,
+        "node_public_key_ed25519": node_pubkey,
+        "issued_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+    # Sign with Colony private key
+    if colony_key_manager is None:
+        colony_keys_dir = state_dir / "colony-keys"
+        passphrase = None  # Caller should handle passphrase
+        if (colony_keys_dir / "private.pem").exists():
+            colony_key_manager = LocalKeyManager(keys_dir=colony_keys_dir, colony_id=colony_id, passphrase=passphrase)
+
+    if colony_key_manager is not None:
+        # Deterministic payload for signing (exclude signature field)
+        payload = json.dumps(cert, sort_keys=True, separators=(",", ":")).encode("utf-8")
+        cert["signature"] = colony_key_manager.sign(payload)
+        logger.info("Node certificate signed by Colony key")
+    else:
+        logger.warning("No Colony key manager available — node certificate is unsigned")
+
+    cert_path = state_dir / NODE_CERT_FILE
+    cert_path.write_text(json.dumps(cert, indent=2) + "\n")
+    return cert
+
+
+def verify_node_certificate(cert: dict, colony_public_key_hex: str) -> bool:
+    """Verify a node certificate's signature against a Colony public key."""
+    from apsimo.chain.identity import _verify_ed25519_signature
+
+    signature = cert.get("signature")
+    if not signature:
+        return False
+
+    payload = {k: v for k, v in cert.items() if k != "signature"}
+    payload_bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return _verify_ed25519_signature(colony_public_key_hex, payload_bytes, signature)
+
+
+def load_node_certificate(state_dir: str | Path) -> Optional[dict]:
+    """Load the node certificate from disk."""
+    cert_path = Path(state_dir) / NODE_CERT_FILE
+    if not cert_path.exists():
+        return None
+    try:
+        return json.loads(cert_path.read_text())
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def get_node_info(state_dir: str | Path) -> dict:
+    """Get full node identity info: node_id, public key, cert status."""
+    state_dir = Path(state_dir)
+
+    node_id = None
+    node_pubkey = None
+    certified = False
+    issued_at = None
+
+    node_id_path = state_dir / NODE_ID_FILE
+    if node_id_path.exists():
+        node_id = node_id_path.read_text().strip() or None
+
+    node_keys_dir = state_dir / NODE_KEYS_DIR
+    if (node_keys_dir / "private.pem").exists():
+        try:
+            from apsimo.chain.local_keys import LocalKeyManager
+            km = LocalKeyManager(keys_dir=node_keys_dir, colony_id=node_id or "unknown")
+            node_pubkey = km.public_key_hex()
+        except Exception:
+            pass
+
+    cert = load_node_certificate(state_dir)
+    if cert and cert.get("signature"):
+        certified = True
+        issued_at = cert.get("issued_at")
+
+    return {
+        "node_id": node_id,
+        "node_public_key": node_pubkey,
+        "certified": certified,
+        "issued_at": issued_at,
+    }