brawny 0.1.13__py3-none-any.whl → 0.1.22__py3-none-any.whl

brawny/config/validation.py

@@ -81,6 +81,16 @@ def dedupe_preserve_order(endpoints: list[str]) -> list[str]:
     return result
 
 
+def canonicalize_endpoints(endpoints: list[str]) -> list[str]:
+    """Canonicalize endpoint list for stable comparison.
+
+    Applies canonicalize_endpoint to each entry, then sorts for deterministic
+    equality checks.
+    """
+    canonical = [canonicalize_endpoint(ep) for ep in endpoints]
+    return sorted(set(canonical))
+
+
 REMOVED_FIELDS = {
     "alerts_dx_enabled",
     "allowed_signers",
@@ -151,20 +161,39 @@ def validate_config(config: "Config") -> None:
     # Required fields
     if not config.database_url:
         errors.append("database_url is required")
-    elif not (
-        config.database_url.startswith("postgresql://")
-        or config.database_url.startswith("postgres://")
-        or config.database_url.startswith("sqlite:///")
-    ):
-        errors.append(
-            "database_url must start with postgresql://, postgres://, or sqlite:///"
-        )
-    elif config.database_url.startswith("sqlite:///") and config.worker_count > 1:
-        errors.append("SQLite does not support worker_count > 1. Use Postgres for production.")
+    elif not config.database_url.startswith("sqlite:///"):
+        errors.append("database_url must start with sqlite:///")
+    elif config.worker_count > 1:
+        errors.append("SQLite does not support worker_count > 1.")
 
     if not config.rpc_groups:
         errors.append("rpc_groups is required (at least one group)")
 
+    if config.guardrails:
+        lint_paths = config.guardrails.lint_paths
+        if not isinstance(lint_paths, list):
+            errors.append("guardrails.lint_paths must be a list")
+        else:
+            for idx, value in enumerate(lint_paths):
+                if not isinstance(value, str) or not value.strip():
+                    errors.append(f"guardrails.lint_paths[{idx}] must be a non-empty string")
+
+    if config.debug and not isinstance(config.debug.allow_console, bool):
+        errors.append("debug.allow_console must be a boolean")
+
+    if config.intent_cooldown:
+        cd = config.intent_cooldown
+        if not isinstance(cd.enabled, bool):
+            errors.append("intent_cooldown.enabled must be a boolean")
+        if cd.default_seconds < 0:
+            errors.append("intent_cooldown.default_seconds cannot be negative")
+        if cd.max_seconds < 0:
+            errors.append("intent_cooldown.max_seconds cannot be negative")
+        if cd.max_seconds < cd.default_seconds:
+            errors.append("intent_cooldown.max_seconds must be >= default_seconds")
+        if cd.prune_older_than_days < 0:
+            errors.append("intent_cooldown.prune_older_than_days cannot be negative")
+
     if config.chain_id <= 0:
         errors.append("chain_id must be positive")
 
@@ -192,6 +221,19 @@ def validate_config(config: "Config") -> None:
     if config.keystore_type == KeystoreType.FILE and not config.keystore_path:
         errors.append("keystore_path is required when keystore_type is 'file'")
 
+    # HTTP allowlist validation
+    for domain in config.http.allowed_domains:
+        if "://" in domain:
+            errors.append(f"http.allowed_domains entries must be hostnames, got: {domain}")
+        if "/" in domain:
+            errors.append(f"http.allowed_domains entries must not include paths: {domain}")
+    if config.http.connect_timeout_seconds <= 0:
+        errors.append("http.connect_timeout_seconds must be positive")
+    if config.http.read_timeout_seconds <= 0:
+        errors.append("http.read_timeout_seconds must be positive")
+    if config.http.max_retries < 0:
+        errors.append("http.max_retries cannot be negative")
+
     if errors:
         raise ConfigError(
             "Configuration validation failed:\n" + "\n".join(f"  - {e}" for e in errors)
@@ -234,11 +276,6 @@ def validate_advanced_config(advanced: "AdvancedConfig") -> None:
     if advanced.rpc_max_retries < 0:
         errors.append("rpc_max_retries cannot be negative")
 
-    if advanced.database_pool_size <= 0:
-        errors.append("database_pool_size must be positive")
-    if advanced.database_pool_max_overflow < 0:
-        errors.append("database_pool_max_overflow cannot be negative")
-
     if errors:
         raise ConfigError(
             "Advanced configuration validation failed:\n"

brawny/daemon/context.py

@@ -12,11 +12,12 @@ from typing import TYPE_CHECKING, Callable
 if TYPE_CHECKING:
     from brawny.config import Config
     from brawny.db.base import Database
-    from brawny._rpc.manager import RPCManager
+    from brawny._rpc.clients import ReadClient
     from brawny.tx.executor import TxExecutor
     from brawny.tx.monitor import TxMonitor
     from brawny.tx.replacement import TxReplacer
     from brawny.tx.nonce import NonceManager
+    from brawny.runtime_controls import RuntimeControls
 
 
 @dataclass
@@ -29,12 +30,13 @@ class DaemonContext:
     config: "Config"
     log: Logger
     db: "Database"
-    rpc: "RPCManager"
+    rpc: "ReadClient"
     executor: "TxExecutor | None"
     monitor: "TxMonitor | None"
     replacer: "TxReplacer | None"
     nonce_manager: "NonceManager | None"
     chain_id: int
+    controls: "RuntimeControls | None" = None
 
     # Health alerts (optional - None means disabled)
     health_send_fn: Callable[..., None] | None = None

brawny/daemon/core.py

@@ -12,13 +12,15 @@ import socket
 import threading
 import time
 from threading import Event, Lock, Thread
-from typing import TYPE_CHECKING, Callable
+from typing import TYPE_CHECKING, Any, Callable
 
 from brawny.alerts.contracts import ContractSystem
 from brawny.alerts.health import health_alert
-from brawny.alerts.send import create_send_health
+from brawny.alerts.send import AlertService, create_send_health, set_alert_service
+from brawny.async_runtime import clear_loop, register_loop, run_sync
 from brawny.daemon.context import DaemonContext, DaemonState, RuntimeOverrides
 from brawny.daemon.loops import run_monitor, run_worker
+from brawny.daemon.supervisor import WorkerSupervisor
 from brawny.db import create_database
 from brawny.db.migrate import Migrator, verify_critical_schema
 from brawny.jobs.discovery import (
@@ -36,7 +38,7 @@ from brawny.metrics import ACTIVE_WORKERS, get_metrics
 from brawny.model.enums import IntentStatus
 from brawny.model.startup import StartupMessage
 from brawny.model.types import BlockInfo
-from brawny._rpc import RPCManager
+from brawny._rpc.clients import ReadClient
 from brawny.scheduler.poller import BlockPoller
 from brawny.scheduler.reorg import ReorgDetector
 from brawny.scheduler.runner import JobRunner
@@ -45,6 +47,7 @@ from brawny.tx.executor import TxExecutor
 from brawny.tx.intent import transition_intent
 from brawny.tx.monitor import TxMonitor
 from brawny.tx.replacement import TxReplacer
+from brawny.runtime_controls import RuntimeControls
 from brawny.validation import validate_job_routing
 from brawny.telegram import TelegramBot
 
@@ -82,13 +85,14 @@ class BrawnyDaemon:
 
         # Components (initialized in start())
         self._db: Database | None = None
-        self._rpc: RPCManager | None = None
+        self._rpc: ReadClient | None = None
         self._keystore: Keystore | None = None
         self._contract_system: ContractSystem | None = None
         self._lifecycle: LifecycleDispatcher | None = None
         self._executor: TxExecutor | None = None
         self._monitor: TxMonitor | None = None
         self._replacer: TxReplacer | None = None
+        self._controls: RuntimeControls | None = None
         self._job_runner: JobRunner | None = None
         self._reorg_detector: ReorgDetector | None = None
         self._poller: BlockPoller | None = None
@@ -111,6 +115,9 @@ class BrawnyDaemon:
         self._monitor_thread: Thread | None = None
         self._monitor_stop = Event()
 
+        # Worker supervision (fail-fast on worker thread failures)
+        self._supervisor = WorkerSupervisor(fail_fast=True)
+
         # Inflight tracking
         self._inflight_lock = Lock()
         self._inflight_count = 0
@@ -124,19 +131,23 @@ class BrawnyDaemon:
 
         # Async event loop (owned by daemon, used by runner for async job.check())
         self._loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
-        asyncio.set_event_loop(self._loop)  # Make it the current loop for this thread
-        self._loop_thread_id: int = threading.get_ident()  # Assert ownership
+        self._loop_thread: Thread | None = None
+        self._loop_started = Event()
+        self._loop_thread_id: int | None = None
+        self._alert_service: AlertService | None = None
 
     @property
     def db(self) -> "Database":
         """Get database connection."""
-        assert self._db is not None, "Daemon not started"
+        if self._db is None:
+            raise RuntimeError("Daemon not started")
         return self._db
 
     @property
-    def rpc(self) -> RPCManager:
+    def rpc(self) -> ReadClient:
         """Get RPC manager."""
-        assert self._rpc is not None, "Daemon not started"
+        if self._rpc is None:
+            raise RuntimeError("Daemon not started")
         return self._rpc
 
     @property
@@ -151,7 +162,8 @@ class BrawnyDaemon:
 
     def _check_schema(self) -> None:
         """Verify critical DB schema columns exist. Hard-fail if not."""
-        assert self._db is not None
+        if self._db is None:
+            raise RuntimeError("Database not initialized")
 
         try:
             verify_critical_schema(self._db)
@@ -160,20 +172,46 @@ class BrawnyDaemon:
             self._log.critical(
                 "schema.validation_failed",
                 error=error_msg,
-                table="tx_intents",
+                table="critical_schema",
             )
             health_alert(
                 component="brawny.startup.schema",
                 chain_id=self.config.chain_id,
                 error=error_msg,
                 level="critical",
-                action="Run: brawny migrate",
+                action="See error for remediation",
                 db_dialect=self._db.dialect,
                 force_send=True,
                 send_fn=self._health_send_fn,
                 health_chat_id=self._health_chat_id,
             )
-            raise SystemExit(f"DB schema mismatch: {error_msg}. Run: brawny migrate") from exc
+            raise SystemExit(f"DB schema mismatch: {error_msg}") from exc
+
+    def _start_async_loop(self) -> None:
+        if self._loop_thread and self._loop_thread.is_alive():
+            return
+
+        def _run_loop() -> None:
+            asyncio.set_event_loop(self._loop)
+            self._loop_thread_id = threading.get_ident()
+            register_loop(self._loop, self._loop_thread_id)
+            self._loop_started.set()
+            self._loop.run_forever()
+            self._loop.close()
+
+        self._loop_started.clear()
+        self._loop_thread = Thread(target=_run_loop, name="brawny-async-loop", daemon=True)
+        self._loop_thread.start()
+        self._loop_started.wait(timeout=5.0)
+        if not self._loop_started.is_set():
+            raise RuntimeError("Async loop failed to start")
+
+    def _stop_async_loop(self) -> None:
+        if self._loop and not self._loop.is_closed():
+            self._loop.call_soon_threadsafe(self._loop.stop)
+        if self._loop_thread:
+            self._loop_thread.join(timeout=5.0)
+        clear_loop()
 
     def _make_claim_token(self, worker_id: int) -> str:
         """Generate a unique claim token for a worker."""
@@ -202,7 +240,8 @@ class BrawnyDaemon:
 
     def _process_block(self, block: BlockInfo) -> None:
         """Process a single block."""
-        assert self._job_runner is not None
+        if self._job_runner is None:
+            raise RuntimeError("Job runner not initialized")
 
         self._log.info(
             "block.ingest.start",
@@ -320,8 +359,10 @@ class BrawnyDaemon:
 
     def _reconcile_startup(self) -> None:
         """Reconcile state on startup."""
-        assert self._db is not None
-        assert self._monitor is not None or self.overrides.dry_run
+        if self._db is None:
+            raise RuntimeError("Database not initialized")
+        if self._monitor is None and not self.overrides.dry_run:
+            raise RuntimeError("Monitor not initialized")
 
         # Reconcile nonces
         if self._executor and self._executor.nonce_manager:
@@ -343,23 +384,19 @@ class BrawnyDaemon:
                     "startup_recover_sending",
                     chain_id=self.config.chain_id,
                 )
-            else:
-                # No tx_hash means intent never got broadcast - reset to CREATED
-                if attempt and self._executor and self._executor.nonce_manager:
-                    from brawny.model.enums import AttemptStatus
-                    self._db.update_attempt_status(
-                        attempt.attempt_id,
-                        AttemptStatus.FAILED.value,
-                        error_code="startup_stuck",
-                        error_detail="Stuck in SENDING without broadcast",
-                    )
-                    self._executor.nonce_manager.release(intent.signer_address, attempt.nonce)
-                transition_intent(
-                    self._db,
-                    intent.intent_id,
-                    IntentStatus.CREATED,
-                    "startup_recover_sending",
-                    chain_id=self.config.chain_id,
+                continue
+            if not attempt or not attempt.tx_hash:
+                self._db.set_signer_quarantined(
+                    self.config.chain_id,
+                    intent.signer_address,
+                    reason="startup_sending_no_tx_hash",
+                    source="startup_reconcile",
+                )
+                self._log.warning(
+                    "startup.quarantine_sending_no_tx",
+                    intent_id=str(intent.intent_id),
+                    job_id=intent.job_id,
+                    attempt_id=str(attempt.attempt_id) if attempt else None,
                 )
 
         if stuck_sending:
@@ -378,7 +415,7 @@ class BrawnyDaemon:
             )
 
     def _start_workers(self) -> None:
-        """Start worker threads."""
+        """Start worker threads with supervision."""
         if self.overrides.dry_run:
             return
 
@@ -397,6 +434,7 @@ class BrawnyDaemon:
             monitor=self._monitor,
             replacer=self._replacer,
             nonce_manager=self._executor.nonce_manager if self._executor else None,
+            controls=self._controls,
             chain_id=self.config.chain_id,
             health_send_fn=self._health_send_fn,
             health_chat_id=self._health_chat_id,
@@ -409,22 +447,48 @@ class BrawnyDaemon:
             inflight_dec=self._inflight_done,
         )
 
+        # Register workers with supervisor
         for i in range(worker_count):
-            t = Thread(
-                target=run_worker,
-                args=(i, self._stop, self._wakeup_hint, ctx, state, self.overrides.dry_run),
-                daemon=True,
+            self._supervisor.add(
+                f"tx_worker_{i}",
+                lambda worker_id=i: run_worker(
+                    worker_id, self._stop, self._wakeup_hint, ctx, state, self.overrides.dry_run
+                ),
             )
-            t.start()
-            self._worker_threads.append(t)
-
-        # Start monitor thread
-        self._monitor_thread = Thread(
-            target=run_monitor,
-            args=(self._monitor_stop, ctx, self._worker_threads),
-            daemon=True,
+
+        # Register monitor as supervised worker
+        self._supervisor.add(
+            "tx_monitor",
+            lambda: run_monitor(self._monitor_stop, ctx, self._worker_threads),
         )
-        self._monitor_thread.start()
+
+        # Start all supervised workers
+        self._supervisor.start_all()
+
+        # Track worker threads for backward compatibility (used in monitor and shutdown)
+        # The supervisor owns the actual threads, but we need references for metrics
+        with self._supervisor._lock:
+            for name, worker_state in self._supervisor._workers.items():
+                if name.startswith("tx_worker_") and worker_state.thread:
+                    self._worker_threads.append(worker_state.thread)
+                elif name == "tx_monitor" and worker_state.thread:
+                    self._monitor_thread = worker_state.thread
+
+        # Start supervisor watcher - signals daemon stop when supervisor triggers shutdown
+        def _watch_supervisor() -> None:
+            self._supervisor.wait_for_shutdown()
+            if not self._stop.is_set():
+                self._log.critical(
+                    "daemon.supervisor_shutdown",
+                    reason=self._supervisor.fatal_reason(),
+                )
+                self._stop.set()
+                self._wakeup_hint.set()
+                if self._poller:
+                    self._poller.stop(timeout=0.1)
+
+        watcher = Thread(target=_watch_supervisor, name="supervisor-watcher", daemon=True)
+        watcher.start()
 
         # Initial gauge
         metrics = get_metrics()
@@ -467,9 +531,18 @@ class BrawnyDaemon:
         if alive:
             self._log.warning("shutdown.threads_still_alive", count=len(alive))
 
-        # Close event loop
-        if self._loop and not self._loop.is_closed():
-            self._loop.close()
+        # Close HTTP clients to avoid leaked connections
+        # Keep calls qualified to avoid name collision (both modules export close_http_client)
+        from brawny.alerts import abi_resolver
+        from brawny.telegram import close_http_client as close_telegram_http_client
+
+        if self._alert_service is not None:
+            run_sync(self._alert_service.stop(flush_timeout=self.config.shutdown_grace_seconds))
+            set_alert_service(None)
+        abi_resolver.close_http_client()
+        close_telegram_http_client()
+
+        self._stop_async_loop()
 
         self._log.info("daemon.shutdown.complete")
 
@@ -486,13 +559,11 @@ class BrawnyDaemon:
         # Database
         self._db = create_database(
             self.config.database_url,
-            pool_size=self.config.database_pool_size,
-            pool_max_overflow=self.config.database_pool_max_overflow,
-            pool_timeout=self.config.database_pool_timeout_seconds,
             circuit_breaker_failures=self.config.db_circuit_breaker_failures,
             circuit_breaker_seconds=self.config.db_circuit_breaker_seconds,
         )
         self._db.connect()
+        self._controls = RuntimeControls(self._db)
 
         # Migrations
         migrator = Migrator(self._db)
@@ -502,7 +573,7 @@ class BrawnyDaemon:
             migrator.migrate()
 
         # RPC
-        self._rpc = RPCManager.from_config(self.config)
+        self._rpc = ReadClient.from_config(self.config)
 
         self._log.info(
             "startup.finality_policy",
@@ -554,7 +625,10 @@ class BrawnyDaemon:
 
         # Cache TelegramBot instance (if configured)
         if self.config.telegram.bot_token:
-            self._telegram_bot = TelegramBot(token=self.config.telegram.bot_token)
+            self._telegram_bot = TelegramBot(
+                token=self.config.telegram.bot_token,
+                default_parse_mode=self.config.telegram.parse_mode or "Markdown",
+            )
 
         # Initialize health alerting
         tg = self.config.telegram
@@ -575,6 +649,16 @@ class BrawnyDaemon:
         if tg:
             self._health_cooldown = tg.health_cooldown_seconds
 
+        from brawny.alerts import send as alerts_send
+        self._alert_service = AlertService(
+            maxsize=alerts_send.ALERT_QUEUE_MAXSIZE,
+            max_attempts=alerts_send.ALERT_SEND_MAX_ATTEMPTS,
+            backoff_base_seconds=alerts_send.ALERT_SEND_BACKOFF_BASE_SECONDS,
+            backoff_max_seconds=alerts_send.ALERT_SEND_BACKOFF_MAX_SECONDS,
+            health_max_oldest_age_seconds=self.config._advanced_or_default().alerts_health_max_oldest_age_seconds,
+        )
+        set_alert_service(self._alert_service)
+
         # Validate schema (after health is set up so we can alert on failure)
         self._check_schema()
 
@@ -604,7 +688,8 @@ class BrawnyDaemon:
             )
             self._replacer = TxReplacer(
                 self._db, self._rpc, self._keystore, self._executor.nonce_manager, self.config,
-                lifecycle=self._lifecycle
+                lifecycle=self._lifecycle,
+                controls=self._controls,
             )
 
         # Job runner
@@ -616,7 +701,7 @@ class BrawnyDaemon:
             lifecycle=self._lifecycle,
             contract_system=self._contract_system,
             loop=self._loop,
-            loop_thread_id=self._loop_thread_id,
+            controls=self._controls,
         )
         self._job_runner._on_intent_created = self._on_intent_created
 
@@ -649,22 +734,31 @@ class BrawnyDaemon:
 
         return validation_errors, routing_errors, startup_messages
 
-    def run(self, blocking: bool = True) -> None:
-        """Run the daemon.
+    def run(self, blocking: bool = True) -> int:
+        """Run the daemon. Returns exit code (0=clean, 1=failure).
+
+        Caller should: sys.exit(daemon.run())
 
         Args:
             blocking: If True, block until shutdown. If False, return immediately.
+
+        Returns:
+            Exit code: 0 for clean shutdown, 1 for worker failure
         """
-        assert self._poller is not None, "Daemon not initialized"
+        if self._poller is None:
+            raise RuntimeError("Daemon not initialized")
+
+        # Start async loop and services
+        self._start_async_loop()
+        if self._alert_service is not None:
+            run_sync(self._alert_service.start())
 
         # Startup reconciliation
         self._reconcile_startup()
 
         # Warm gas cache before workers start (eliminates cold-start race)
         try:
-            self._loop.run_until_complete(
-                asyncio.wait_for(self._rpc.gas_quote(), timeout=5.0)
-            )
+            run_sync(asyncio.wait_for(self._rpc.gas_quote(), timeout=5.0))
             self._log.debug("startup.gas_cache_warmed")
         except Exception as e:
             self._log.warning("startup.gas_cache_warm_failed", error=str(e))
@@ -685,6 +779,34 @@ class BrawnyDaemon:
         finally:
             self._shutdown()
 
+        # Return non-zero exit code if supervisor triggered shutdown due to worker failure
+        if self._supervisor.fatal_reason():
+            return 1
+        return 0
+
+    def health_check(self) -> dict[str, Any]:
+        """Return daemon health status.
+
+        Uses all_healthy() as primary health indicator. This ensures:
+        - fail_fast=True + worker fails → shutdown_requested()=True → healthy=False
+        - fail_fast=False + worker fails → all_healthy()=False → healthy=False
+
+        Either way, health checks report unhealthy when workers fail.
+        """
+        worker_snapshot = self._supervisor.snapshot()
+        workers_ok = self._supervisor.all_healthy()
+
+        from brawny.alerts import send as alerts_send
+        alert_health = alerts_send.get_alert_worker_health()
+        alerts_ok = bool(alert_health.get("healthy", True))
+
+        return {
+            "healthy": workers_ok and alerts_ok and not self._supervisor.shutdown_requested(),
+            "workers": worker_snapshot,
+            "fatal_reason": self._supervisor.fatal_reason(),
+            "alerts": alert_health,
+        }
+
     def stop(self, timeout: float = 5.0) -> None:
         """Stop the daemon.