brawny 0.1.13__py3-none-any.whl → 0.1.22__py3-none-any.whl

brawny/tx/nonce.py

@@ -1,7 +1,7 @@
 """Centralized nonce manager for transaction execution.
 
 Implements the nonce management strategy from SPEC 8:
-- Reserve nonce with SERIALIZABLE isolation
+- Reserve nonce with SQLite BEGIN EXCLUSIVE locking
 - Nonce status transitions (reserved → in_flight → released/orphaned)
 - Reconciliation loop for startup and periodic sync
 - SQLite-specific locking for development
@@ -11,19 +11,25 @@ Jobs NEVER allocate or set nonces - the nonce manager owns all nonce operations.
 
 from __future__ import annotations
 
+import time
 from contextlib import contextmanager
 from typing import TYPE_CHECKING, Generator
 from uuid import UUID
 
+from cachetools import TTLCache
 from web3 import Web3
 
 from brawny.logging import LogEvents, get_logger
 from brawny.model.enums import NonceStatus
+from brawny.timeout import Deadline
+
+# Rate limit gap detection logs to once per 60 seconds per signer
+GAP_LOG_RATE_LIMIT_SECONDS = 60
 
 if TYPE_CHECKING:
     from brawny.db.base import Database
     from brawny.model.types import NonceReservation
-    from brawny._rpc.manager import RPCManager
+    from brawny._rpc.clients import ReadClient
 
 logger = get_logger(__name__)
 
@@ -40,7 +46,7 @@ class NonceManager:
     def __init__(
         self,
         db: Database,
-        rpc: RPCManager,
+        rpc: ReadClient,
         chain_id: int,
     ) -> None:
         """Initialize nonce manager.
@@ -53,11 +59,16 @@ class NonceManager:
         self._db = db
         self._rpc = rpc
         self._chain_id = chain_id
+        # Rate limiting for gap detection logs: signer_address -> last_log_time
+        # Single-threaded access (from executor) - no lock needed
+        # Low cardinality keys (signer addresses): maxsize=1000, ttl=1h
+        self._gap_log_times: TTLCache[str, float] = TTLCache(maxsize=1000, ttl=3600)
 
     def reserve_nonce(
         self,
         signer_address: str,
         intent_id: UUID | None = None,
+        deadline: Deadline | None = None,
     ) -> int:
         """Reserve the next available nonce for a signer.
 
@@ -82,7 +93,9 @@ class NonceManager:
 
         try:
             chain_nonce = self._rpc.get_transaction_count(
-                Web3.to_checksum_address(signer_address), block_identifier="pending"
+                Web3.to_checksum_address(signer_address),
+                block_identifier="pending",
+                deadline=deadline,
             )
         except Exception as e:
             logger.warning(
@@ -92,6 +105,34 @@ class NonceManager:
             )
             chain_nonce = None
 
+        signer_state = self._db.get_signer_state(self._chain_id, signer_address)
+        if (
+            signer_state
+            and chain_nonce is not None
+            and signer_state.last_synced_chain_nonce is not None
+            and chain_nonce < signer_state.last_synced_chain_nonce
+        ):
+            self._db.set_signer_quarantined(
+                self._chain_id,
+                signer_address,
+                reason="stale_chain_nonce",
+                source="nonce_reserve",
+            )
+            logger.warning(
+                "nonce.stale_chain_nonce",
+                signer=signer_address,
+                chain_nonce=chain_nonce,
+                last_synced_chain_nonce=signer_state.last_synced_chain_nonce,
+            )
+            raise RuntimeError("Stale chain nonce detected; signer quarantined")
+        if signer_state and signer_state.quarantined_at:
+            logger.warning(
+                "nonce.signer_quarantined",
+                signer=signer_address,
+                reason=signer_state.quarantine_reason,
+            )
+            raise RuntimeError("Signer is quarantined; nonce reservation blocked")
+
         nonce = self._db.reserve_nonce_atomic(
             chain_id=self._chain_id,
             address=signer_address,
@@ -138,6 +179,8 @@ class NonceManager:
         self,
         signer_address: str,
         nonce: int,
+        reason: str | None = None,
+        source: str | None = None,
     ) -> bool:
         """Release a nonce reservation (after confirm/fail/abandon).
 
@@ -150,7 +193,11 @@ class NonceManager:
         """
         signer_address = signer_address.lower()
         return self._db.release_nonce_reservation(
-            self._chain_id, signer_address, nonce
+            self._chain_id,
+            signer_address,
+            nonce,
+            reason=reason,
+            source=source,
         )
 
     @contextmanager
@@ -265,14 +312,18 @@ class NonceManager:
         ]
 
     def reconcile(self, signer_address: str | None = None) -> dict[str, int]:
-        """Reconcile nonce reservations with chain state.
+        """Reconcile nonce reservations using SAFE operations only.
+
+        SAFETY INVARIANTS:
+        - NEVER mutates signers.next_nonce (use force_reset() for that)
+        - Only releases reservations provable from DB state (confirmed intents)
+        - Gap detection is observability-only (log + metric, no action)
 
         Run at startup and periodically to:
-        - Reset next_nonce when gap detected (CRITICAL for recovery)
-        - Update signer's synced chain nonce
-        - Mark stale reservations as released or orphaned
-        - Clean up confirmed/used nonces
-        - Release gap reservations (nonces >= chain_nonce with no tx in mempool)
+        - Update signer's synced chain nonce (observability)
+        - Detect nonce gaps and emit alerts (no auto-reset)
+        - Release reservations for DB-confirmed intents
+        - Clean up old orphaned reservations (time-based)
 
         Args:
             signer_address: Optional specific signer to reconcile.
@@ -281,13 +332,14 @@ class NonceManager:
         Returns:
             Dictionary with reconciliation stats
         """
+        from brawny.metrics import NONCE_GAP_DETECTED, get_metrics
+
         stats = {
             "signers_checked": 0,
             "nonces_released": 0,
-            "nonces_orphaned": 0,
             "orphans_cleaned": 0,
-            "next_nonce_reset": 0,
-            "gap_reservations_released": 0,
+            "gaps_detected": 0,
+            "orphans_marked": 0,
         }
 
         if signer_address:
@@ -296,89 +348,70 @@ class NonceManager:
         else:
             signers = self._db.get_all_signers(self._chain_id)
 
+        metrics = get_metrics()
+
         for signer in signers:
             stats["signers_checked"] += 1
 
             try:
-                # Get current chain nonce
+                # Get current chain nonce (for observability only)
                 chain_nonce = self._rpc.get_transaction_count(
                     Web3.to_checksum_address(signer.signer_address), block_identifier="pending"
                 )
 
-                # Update signer's synced chain nonce
+                # Update signer's synced chain nonce (observability only)
                 self._db.update_signer_chain_nonce(
                     self._chain_id, signer.signer_address, chain_nonce
                 )
 
-                # CRITICAL FIX: Reset next_nonce when gap detected
-                # Without this, reserve_nonce_atomic() keeps returning stale nonces
+                # Gap detection: log + metric only, NO auto-reset
+                # Reason: single-endpoint "pending nonce" is not a truth source.
+                # Auto-reset based on RPC can brick the system during RPC incidents.
+                # Recovery requires explicit operator action via force_reset().
                 if chain_nonce < signer.next_nonce:
                     gap_size = signer.next_nonce - chain_nonce
-                    logger.warning(
-                        "nonce.gap_reset",
-                        signer=signer.signer_address,
-                        old_next_nonce=signer.next_nonce,
-                        chain_nonce=chain_nonce,
-                        gap_size=gap_size,
-                    )
-                    self._db.update_signer_next_nonce(
-                        self._chain_id, signer.signer_address, chain_nonce
-                    )
-                    stats["next_nonce_reset"] += 1
+                    stats["gaps_detected"] += 1
 
-                    # Release all non-released reservations >= chain_nonce
-                    # These are "gap" reservations whose txs are no longer in mempool
-                    gap_reservations = self._db.get_reservations_for_signer(
-                        self._chain_id, signer.signer_address
+                    # Emit metric for alerting (always - metrics are cheap)
+                    metrics.counter(NONCE_GAP_DETECTED).inc(
+                        chain_id=self._chain_id,
+                        signer=signer.signer_address[:10],  # Truncate for cardinality
                     )
-                    for reservation in gap_reservations:
-                        if reservation.status == NonceStatus.RELEASED:
-                            continue
-                        if reservation.nonce >= chain_nonce:
-                            # This reservation is in the gap - tx doesn't exist
-                            self.release(signer.signer_address, reservation.nonce)
-                            stats["gap_reservations_released"] += 1
-                            logger.debug(
-                                "nonce.gap_reservation_released",
-                                signer=signer.signer_address,
-                                nonce=reservation.nonce,
-                                intent_id=str(reservation.intent_id) if reservation.intent_id else None,
-                            )
-
-                # Get stale reservations (nonce < chain_nonce)
-                stale_reservations = self._db.get_reservations_below_nonce(
-                    self._chain_id, signer.signer_address, chain_nonce
-                )
-
-                for reservation in stale_reservations:
-                    if reservation.status == NonceStatus.RELEASED:
-                        # Already released, skip
-                        continue
 
-                    if reservation.intent_id:
-                        # Has associated intent - check if confirmed
-                        attempt = self._db.get_latest_attempt_for_intent(
-                            reservation.intent_id
+                    # Rate-limited log warning (per signer)
+                    now = time.monotonic()
+                    last_log = self._gap_log_times.get(signer.signer_address, 0)
+                    if now - last_log >= GAP_LOG_RATE_LIMIT_SECONDS:
+                        self._gap_log_times[signer.signer_address] = now
+                        logger.warning(
+                            "nonce.gap_detected",
+                            signer=signer.signer_address,
+                            chain_id=self._chain_id,
+                            db_next_nonce=signer.next_nonce,
+                            chain_pending_nonce=chain_nonce,
+                            gap_size=gap_size,
+                            action="none",
+                            recovery=f"Run 'brawny signer force-reset {signer.signer_address[:10]}...'",
                         )
-                        if attempt and attempt.status.value == "confirmed":
-                            # Confirmed - release the reservation
-                            self.release(signer.signer_address, reservation.nonce)
-                            stats["nonces_released"] += 1
-                        else:
-                            # Not confirmed but nonce is used - orphaned
-                            self.mark_orphaned(signer.signer_address, reservation.nonce)
-                            stats["nonces_orphaned"] += 1
-                    else:
-                        # No intent - just release
-                        self.release(signer.signer_address, reservation.nonce)
-                        stats["nonces_released"] += 1
+
+                # SAFE CLEANUP: Only release reservations provable from DB state
+                # We iterate all non-released reservations and check DB for confirmation
+                released_count = self._release_confirmed_reservations(signer.signer_address)
+                stats["nonces_released"] += released_count
+
+                # Mark reservations below chain nonce with no intent as orphaned.
+                # This handles external transactions consuming a nonce.
+                orphans_marked = self._mark_orphaned_below_chain_nonce(
+                    signer.signer_address, chain_nonce
+                )
+                stats["orphans_marked"] += orphans_marked
 
                 logger.info(
                     LogEvents.NONCE_RECONCILE,
                     signer=signer.signer_address,
                     chain_nonce=chain_nonce,
-                    stale_count=len(stale_reservations),
-                    next_nonce_was_reset=chain_nonce < signer.next_nonce,
+                    released_count=released_count,
+                    orphans_marked=orphans_marked,
                 )
 
             except Exception as e:
@@ -388,11 +421,57 @@ class NonceManager:
                     error=str(e),
                 )
 
-        # Cleanup old orphaned reservations (24+ hours old)
+        # Cleanup old orphaned reservations (24+ hours old, time-based)
         stats["orphans_cleaned"] = self.cleanup_orphaned()
 
         return stats
 
+    def _mark_orphaned_below_chain_nonce(self, signer_address: str, chain_nonce: int) -> int:
+        """Mark reservations below chain nonce with no intent as orphaned.
+
+        Uses chain nonce as a lower bound to prevent reusing externally consumed nonces.
+        """
+        signer_address = signer_address.lower()
+        reservations = self._db.get_reservations_below_nonce(
+            self._chain_id, signer_address, chain_nonce
+        )
+        marked = 0
+        for reservation in reservations:
+            if reservation.status in (NonceStatus.RELEASED, NonceStatus.ORPHANED):
+                continue
+            if reservation.intent_id is not None:
+                continue
+            if self.mark_orphaned(signer_address, reservation.nonce):
+                marked += 1
+        return marked
+
+    def _release_confirmed_reservations(self, signer_address: str) -> int:
+        """Release reservations for intents that are DB-confirmed.
+
+        SAFE: Only uses DB state, never RPC.
+        """
+        signer_address = signer_address.lower()
+        reservations = self._db.get_reservations_for_signer(
+            self._chain_id, signer_address
+        )
+
+        released = 0
+        for reservation in reservations:
+            if reservation.status == NonceStatus.RELEASED:
+                continue
+
+            if not reservation.intent_id:
+                # No intent attached - skip (could be pre-broadcast)
+                continue
+
+            # Check if intent is confirmed IN THE DATABASE
+            attempt = self._db.get_latest_attempt_for_intent(reservation.intent_id)
+            if attempt and attempt.status.value == "confirmed":
+                self.release(signer_address, reservation.nonce)
+                released += 1
+
+        return released
+
     def cleanup_orphaned(self, older_than_hours: int = 24) -> int:
         """Delete orphaned nonce reservations older than specified hours.
 
@@ -447,35 +526,55 @@ class NonceManager:
 
         return chain_nonce
 
-    def force_reset(self, signer_address: str) -> int:
-        """Force reset nonce state to match chain. Returns new next_nonce.
+    def force_reset(
+        self,
+        signer_address: str,
+        source: str = "unknown",
+        reason: str | None = None,
+        target_nonce: int | None = None,
+    ) -> int:
+        """Force reset nonce state. Returns new next_nonce.
 
-        USE WITH CAUTION: May cause issues if dropped txs later mine.
+        USE WITH CAUTION: Destructive operation that may cause issues if
+        dropped txs later mine. This requires explicit operator action.
 
         This will:
-        - Query current chain pending nonce
-        - Reset local next_nonce to match chain
-        - Release all reservations with nonce >= chain_pending_nonce
+        - Query current chain pending nonce (or use target_nonce if provided)
+        - Reset local next_nonce to match
+        - Release all reservations with nonce >= target
         - Clear gap tracking
+        - Emit audit log and metric
 
         Args:
             signer_address: Ethereum address of the signer
+            source: Where this reset originated ("cli", "executor", "api")
+            reason: Human-readable reason for the reset
+            target_nonce: Optional explicit target. If None, uses chain pending nonce.
 
         Returns:
-            The new next_nonce (equal to chain pending nonce)
+            The new next_nonce
         """
+        from brawny.metrics import NONCE_FORCE_RESET, get_metrics
+
         signer_address = signer_address.lower()
-        chain_nonce = self._rpc.get_transaction_count(
-            Web3.to_checksum_address(signer_address), block_identifier="pending"
-        )
 
-        # Release all reservations at or above chain nonce
+        # Get target nonce
+        if target_nonce is None:
+            target_nonce = self._rpc.get_transaction_count(
+                Web3.to_checksum_address(signer_address), block_identifier="pending"
+            )
+
+        # Get current state for audit logging
+        current_state = self._db.get_signer_state(self._chain_id, signer_address)
+        old_next_nonce = current_state.next_nonce if current_state else None
+
+        # Release all reservations at or above target nonce
         reservations = self._db.get_reservations_for_signer(
             self._chain_id, signer_address
         )
         released_count = 0
         for r in reservations:
-            if r.nonce >= chain_nonce and r.status in (
+            if r.nonce >= target_nonce and r.status in (
                 NonceStatus.RESERVED,
                 NonceStatus.IN_FLIGHT,
             ):
@@ -483,16 +582,39 @@ class NonceManager:
                 released_count += 1
 
         # Reset next_nonce
-        self._db.update_signer_next_nonce(self._chain_id, signer_address, chain_nonce)
+        self._db.update_signer_next_nonce(self._chain_id, signer_address, target_nonce)
 
         # Clear gap tracking
         self._db.clear_gap_started_at(self._chain_id, signer_address)
 
+        # Emit metric for observability
+        metrics = get_metrics()
+        metrics.counter(NONCE_FORCE_RESET).inc(
+            chain_id=self._chain_id,
+            signer=signer_address[:10],  # Truncate for cardinality
+            source=source,
+        )
+
+        # Explicit audit log - this is a destructive operation
         logger.warning(
             "nonce.force_reset",
             signer=signer_address,
-            new_next_nonce=chain_nonce,
+            old_next_nonce=old_next_nonce,
+            new_next_nonce=target_nonce,
             released_reservations=released_count,
+            source=source,
+            reason=reason or "not provided",
         )
 
-        return chain_nonce
+        # Durable audit record in DB (survives log rotation)
+        self._db.record_nonce_reset_audit(
+            chain_id=self._chain_id,
+            signer_address=signer_address,
+            old_next_nonce=old_next_nonce,
+            new_next_nonce=target_nonce,
+            released_reservations=released_count,
+            source=source,
+            reason=reason,
+        )
+
+        return target_nonce

brawny/tx/replacement.py

@@ -19,6 +19,7 @@ from web3 import Web3
 from brawny.logging import LogEvents, get_logger
 from brawny.metrics import TX_REPLACED, get_metrics
 from brawny.model.enums import AttemptStatus, IntentStatus
+from brawny.timeout import Deadline
 from brawny.tx.intent import transition_intent
 from brawny.tx.utils import normalize_tx_dict
 from brawny.model.types import GasParams
@@ -29,11 +30,14 @@ if TYPE_CHECKING:
     from brawny.keystore import Keystore
     from brawny.lifecycle import LifecycleDispatcher
     from brawny.model.types import TxAttempt, TxIntent
-    from brawny._rpc.manager import RPCManager
+    from brawny._rpc.clients import BroadcastClient
     from brawny.tx.nonce import NonceManager
+    from brawny.runtime_controls import RuntimeControls
 
 logger = get_logger(__name__)
 
+REPLACER_TICK_TIMEOUT_SECONDS = 10.0
+
 
 @dataclass
 class ReplacementResult:
@@ -60,11 +64,12 @@ class TxReplacer:
     def __init__(
         self,
         db: Database,
-        rpc: RPCManager,
+        rpc: BroadcastClient,
         keystore: Keystore,
         nonce_manager: NonceManager,
         config: Config,
         lifecycle: "LifecycleDispatcher | None" = None,
+        controls: "RuntimeControls | None" = None,
     ) -> None:
         """Initialize transaction replacer.
 
@@ -81,6 +86,7 @@ class TxReplacer:
         self._nonce_manager = nonce_manager
         self._config = config
         self._lifecycle = lifecycle
+        self._controls = controls
 
     def calculate_replacement_fees(self, old_params: GasParams) -> GasParams:
         """Calculate bumped fees for replacement transaction.
@@ -114,7 +120,12 @@ class TxReplacer:
         attempts = self._db.get_attempts_for_intent(intent_id)
         return sum(1 for a in attempts if a.replaces_attempt_id is not None)
 
-    def should_replace(self, intent: TxIntent, attempt: TxAttempt) -> bool:
+    def should_replace(
+        self,
+        intent: TxIntent,
+        attempt: TxAttempt,
+        deadline: Deadline | None,
+    ) -> bool:
         """Check if a transaction should be replaced.
 
         Args:
@@ -154,21 +165,30 @@ class TxReplacer:
             return False
 
         # Check if still pending (no receipt)
-        receipt = self._rpc.get_transaction_receipt(attempt.tx_hash)
+        if deadline is not None and deadline.expired():
+            return False
+        receipt = self._rpc.get_transaction_receipt(attempt.tx_hash, deadline=deadline)
         if receipt is not None:
             # Has receipt - don't replace
             return False
 
         # Check blocks elapsed
         try:
-            current_block = self._rpc.get_block_number()
+            current_block = self._rpc.get_block_number(deadline=deadline)
             blocks_since = current_block - attempt.broadcast_block
 
             required_blocks = self._config.stuck_tx_blocks * wait_multiplier
             if blocks_since < required_blocks:
                 return False
-        except Exception:
-            pass
+        except Exception as e:
+            logger.debug(
+                "replacement.block_number_failed",
+                intent_id=str(intent.intent_id),
+                attempt_id=str(attempt.attempt_id),
+                tx_hash=attempt.tx_hash,
+                error=str(e)[:200],
+            )
+            return False
 
         return True
 
@@ -176,6 +196,7 @@ class TxReplacer:
         self,
         intent: TxIntent,
         attempt: TxAttempt,
+        deadline: Deadline | None,
     ) -> ReplacementResult:
         """Create a replacement transaction with bumped fees.
 
@@ -190,7 +211,7 @@ class TxReplacer:
         """
         if attempt.tx_hash:
             try:
-                receipt = self._rpc.get_transaction_receipt(attempt.tx_hash)
+                receipt = self._rpc.get_transaction_receipt(attempt.tx_hash, deadline=deadline)
             except Exception as e:
                 logger.warning(
                     "replacement.receipt_check_failed",
@@ -266,16 +287,22 @@ class TxReplacer:
             intent_id=intent.intent_id,
             nonce=attempt.nonce,  # Same nonce
             gas_params_json=new_gas_params.to_json(),
-            status=AttemptStatus.SIGNED.value,
+            status=AttemptStatus.PENDING_SEND.value,
             replaces_attempt_id=attempt.attempt_id,
+            actor=intent.job_id,
+            reason="replacement_attempt",
+            source="replacer",
         )
 
         try:
             # Broadcast replacement
-            tx_hash, _endpoint_url = self._rpc.send_raw_transaction(signed_tx.raw_transaction)
+            tx_hash, _endpoint_url = self._rpc.send_raw_transaction(
+                signed_tx.raw_transaction,
+                deadline=deadline,
+            )
 
             # Update new attempt with tx_hash
-            current_block = self._rpc.get_block_number()
+            current_block = self._rpc.get_block_number(deadline=deadline)
             self._db.update_attempt_status(
                 new_attempt_id,
                 AttemptStatus.BROADCAST.value,
@@ -403,16 +430,32 @@ class TxReplacer:
             "checked": 0,
             "replaced": 0,
             "abandoned": 0,
+            "paused": 0,
             "errors": 0,
         }
 
+        if self._controls and self._controls.is_active("pause_replacements"):
+            logger.warning("replacement.paused_globally")
+            return results
+
         # Get pending intents
         pending_intents = self._db.get_intents_by_status(
             IntentStatus.PENDING.value,
             chain_id=self._config.chain_id,
         )
 
+        deadline = Deadline.from_seconds(REPLACER_TICK_TIMEOUT_SECONDS)
+
         for intent in pending_intents:
+            if deadline.expired():
+                logger.warning("replacement.tick_timeout", pending_remaining=len(pending_intents))
+                break
+            signer_state = self._db.get_signer_state(
+                self._config.chain_id,
+                intent.signer_address,
+            )
+            if signer_state and signer_state.replacements_paused:
+                continue
             attempt = self._db.get_latest_attempt_for_intent(intent.intent_id)
             if not attempt or not attempt.tx_hash:
                 continue
@@ -420,20 +463,28 @@ class TxReplacer:
             results["checked"] += 1
 
             try:
-                if self.should_replace(intent, attempt):
+                if self.should_replace(intent, attempt, deadline):
                     # Check if we've exceeded max replacements
                     replacement_count = self.get_replacement_count(intent.intent_id)
                     if replacement_count >= self._config.max_replacement_attempts:
-                        self.abandon_intent(
-                            intent,
-                            attempt,
-                            f"Max replacement attempts ({self._config.max_replacement_attempts}) exceeded",
+                        self._db.set_replacements_paused(
+                            self._config.chain_id,
+                            intent.signer_address,
+                            True,
+                            reason="replacement_budget_exceeded",
+                            source="replacer",
+                        )
+                        logger.warning(
+                            "replacement.paused_signer",
+                            intent_id=str(intent.intent_id),
+                            signer=intent.signer_address,
+                            count=replacement_count,
                         )
-                        results["abandoned"] += 1
+                        results["paused"] += 1
                         continue
 
                     # Attempt replacement
-                    result = self.replace_transaction(intent, attempt)
+                    result = self.replace_transaction(intent, attempt, deadline)
                     if result.success:
                         results["replaced"] += 1
                     else: