brawny 0.1.13__py3-none-any.whl

brawny/daemon/loops.py

@@ -0,0 +1,327 @@
+"""Worker and monitor loops for brawny daemon.
+
+Provides the main loop functions for intent execution and transaction monitoring.
+"""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timedelta
+from threading import Event
+from typing import TYPE_CHECKING
+
+from brawny.metrics import (
+    ACTIVE_WORKERS,
+    INTENT_CLAIMED,
+    INTENT_RELEASED,
+    INTENT_SENDING_STUCK,
+    INTENTS_BACKING_OFF,
+    get_metrics,
+)
+from brawny.model.enums import AttemptStatus, IntentStatus
+from brawny.tx.intent import transition_intent
+
+if TYPE_CHECKING:
+    from threading import Thread
+    from brawny.daemon.context import DaemonContext, DaemonState
+
+from brawny.alerts.health import cleanup_stale_fingerprints, health_alert
+
+
+def run_worker(
+    worker_id: int,
+    stop_event: Event,
+    wakeup_hint: Event,
+    ctx: "DaemonContext",
+    state: "DaemonState",
+    dry_run: bool = False,
+) -> None:
+    """Worker thread for executing intents.
+
+    Args:
+        worker_id: Worker identifier for logging
+        stop_event: Event signaling shutdown
+        wakeup_hint: Event for immediate wakeup on new intents
+        ctx: Daemon context with shared components
+        state: Daemon state with callbacks
+        dry_run: If True, claim and release without executing
+    """
+    assert ctx.executor is not None or dry_run, "run_worker requires executor unless dry_run"
+
+    ctx.log.debug("worker.started", worker_id=worker_id)
+
+    while not stop_event.is_set():
+        released = ctx.db.release_stale_intent_claims(
+            max_age_seconds=ctx.config.claim_timeout_seconds
+        )
+        if released > 0:
+            ctx.log.info(
+                "worker.stale_claims_released",
+                worker_id=worker_id,
+                released=released,
+            )
+            metrics = get_metrics()
+            metrics.counter(INTENT_RELEASED).inc(
+                released,
+                chain_id=ctx.chain_id,
+                reason="stale_claim",
+            )
+
+        claim_token = state.make_claim_token(worker_id)
+        claimed_by = state.make_claimed_by(worker_id)
+        intent = ctx.db.claim_next_intent(claim_token, claimed_by=claimed_by)
+
+        if intent is None:
+            wakeup_hint.wait(timeout=1.0)
+            wakeup_hint.clear()
+            continue
+
+        ctx.log.info(
+            "intent.claimed",
+            intent_id=str(intent.intent_id),
+            job_id=intent.job_id,
+            claim_token=claim_token,
+            claimed_by=claimed_by,
+            worker_id=worker_id,
+        )
+        metrics = get_metrics()
+        metrics.counter(INTENT_CLAIMED).inc(
+            chain_id=ctx.chain_id,
+        )
+
+        if dry_run:
+            ctx.log.info("worker.dry_run", intent_id=str(intent.intent_id))
+            released = ctx.db.release_intent_claim(intent.intent_id)
+            if not released:
+                ctx.log.warning(
+                    "worker.dry_run_release_failed",
+                    intent_id=str(intent.intent_id),
+                )
+            else:
+                metrics = get_metrics()
+                metrics.counter(INTENT_RELEASED).inc(
+                    chain_id=ctx.chain_id,
+                    reason="dry_run",
+                )
+            continue
+
+        state.inflight_inc()
+        try:
+            outcome = ctx.executor.execute(intent)
+            ctx.log.info(
+                "worker.executed",
+                intent_id=str(intent.intent_id),
+                job_id=intent.job_id,
+                result=outcome.result.value,
+            )
+        except Exception as e:
+            ctx.log.error(
+                "worker.execute_exception",
+                intent_id=str(intent.intent_id),
+                job_id=intent.job_id,
+                error=str(e)[:200],
+                exc_info=True,
+            )
+            health_alert(
+                component="brawny.tx.executor",
+                chain_id=ctx.chain_id,
+                error=e,
+                job_id=intent.job_id,
+                intent_id=str(intent.intent_id),
+                claim_token=intent.claim_token,
+                status=intent.status.value if hasattr(intent.status, "value") else str(intent.status),
+                action="Check logs; intent will retry or timeout",
+                db_dialect=ctx.db.dialect,
+                send_fn=ctx.health_send_fn,
+                health_chat_id=ctx.health_chat_id,
+                cooldown_seconds=ctx.health_cooldown,
+            )
+
+            try:
+                attempts = ctx.db.get_attempts_for_intent(intent.intent_id)
+            except Exception as query_err:
+                ctx.log.warning(
+                    "worker.exception_attempts_lookup_failed",
+                    intent_id=str(intent.intent_id),
+                    job_id=intent.job_id,
+                    error=str(query_err)[:200],
+                )
+                attempts = None
+
+            if attempts == []:
+                if not intent.claim_token:
+                    ctx.log.warning(
+                        "worker.claim_token_missing",
+                        intent_id=str(intent.intent_id),
+                        job_id=intent.job_id,
+                    )
+                else:
+                    try:
+                        released = ctx.db.release_intent_claim_if_token(
+                            intent.intent_id,
+                            intent.claim_token,
+                        )
+                        if released:
+                            ctx.log.info(
+                                "worker.claim_released_on_error",
+                                intent_id=str(intent.intent_id),
+                            )
+                            metrics = get_metrics()
+                            metrics.counter(INTENT_RELEASED).inc(
+                                chain_id=ctx.chain_id,
+                                reason="pre_attempt_exception",
+                            )
+                    except Exception:
+                        ctx.log.exception(
+                            "worker.claim_release_failed",
+                            intent_id=str(intent.intent_id),
+                        )
+            else:
+                ctx.log.warning(
+                    "worker.exception_with_attempts",
+                    intent_id=str(intent.intent_id),
+                    attempt_count=(len(attempts) if attempts is not None else None),
+                    hint="Not releasing claim; monitor/replacer should handle",
+                )
+        finally:
+            state.inflight_dec()
+
+    ctx.log.debug("worker.stopped", worker_id=worker_id)
+
+
+def run_monitor(
+    stop_event: Event,
+    ctx: "DaemonContext",
+    worker_threads: list["Thread"],
+) -> None:
+    """Background loop for monitoring pending transactions.
+
+    Args:
+        stop_event: Event signaling shutdown
+        ctx: Daemon context with shared components
+        worker_threads: List of worker threads for gauge reporting
+    """
+    assert ctx.monitor is not None, "run_monitor requires monitor"
+    assert ctx.replacer is not None, "run_monitor requires replacer"
+    assert ctx.nonce_manager is not None, "run_monitor requires nonce_manager"
+
+    ctx.log.debug("monitor.started")
+    last_reconcile = time.time()
+    last_rpc_health = 0.0
+    last_worker_gauge = 0.0
+    last_sending_recover = 0.0
+    last_log_cleanup = 0.0
+
+    while not stop_event.is_set():
+        try:
+            ctx.monitor.monitor_all_pending()
+            ctx.replacer.process_stuck_transactions()
+
+            now = time.time()
+            if now - last_reconcile >= ctx.config.nonce_reconcile_interval_seconds:
+                ctx.nonce_manager.reconcile()
+                last_reconcile = now
+
+            if now - last_rpc_health >= 30:
+                ctx.rpc.get_health()
+                last_rpc_health = now
+
+            if now - last_worker_gauge >= 10:
+                metrics = get_metrics()
+                active = sum(1 for t in worker_threads if t.is_alive())
+                metrics.gauge(ACTIVE_WORKERS).set(
+                    active,
+                    chain_id=ctx.chain_id,
+                )
+                backing_off = ctx.db.get_backing_off_intent_count(chain_id=ctx.chain_id)
+                metrics.gauge(INTENTS_BACKING_OFF).set(
+                    backing_off,
+                    chain_id=ctx.chain_id,
+                )
+                last_worker_gauge = now
+
+            if now - last_sending_recover >= 30:
+                _recover_stuck_sending(ctx)
+                last_sending_recover = now
+
+            # Job log cleanup (hourly)
+            if now - last_log_cleanup >= 3600:
+                try:
+                    from brawny.db.ops import logs as log_ops
+                    cutoff = datetime.utcnow() - timedelta(days=ctx.config.log_retention_days)
+                    deleted = log_ops.delete_old_logs(ctx.db, ctx.chain_id, cutoff)
+                    if deleted > 0:
+                        ctx.log.info("job_logs.cleanup", deleted=deleted)
+                except Exception as cleanup_err:
+                    ctx.log.warning("job_logs.cleanup_failed", error=str(cleanup_err))
+
+                # Health alert fingerprint cleanup (also hourly)
+                try:
+                    removed = cleanup_stale_fingerprints(ctx.health_cooldown)
+                    if removed > 0:
+                        ctx.log.debug("health_fingerprints.cleanup", removed=removed)
+                except Exception as cleanup_err:
+                    ctx.log.warning("health_fingerprints.cleanup_failed", error=str(cleanup_err))
+
+                last_log_cleanup = now
+        except Exception as e:
+            ctx.log.error("monitor.error", error=str(e)[:200])
+            health_alert(
+                component="brawny.tx.monitor",
+                chain_id=ctx.chain_id,
+                error=e,
+                action="Check DB/RPC connectivity",
+                db_dialect=ctx.db.dialect,
+                send_fn=ctx.health_send_fn,
+                health_chat_id=ctx.health_chat_id,
+                cooldown_seconds=ctx.health_cooldown,
+            )
+
+        stop_event.wait(timeout=ctx.config.poll_interval_seconds * 2)
+
+    ctx.log.debug("monitor.stopped")
+
+
+def _recover_stuck_sending(ctx: "DaemonContext") -> None:
+    """Recover intents stuck in SENDING state.
+
+    Args:
+        ctx: Daemon context with shared components
+    """
+    assert ctx.nonce_manager is not None, "_recover_stuck_sending requires nonce_manager"
+
+    stuck_sending = ctx.db.list_sending_intents_older_than(
+        max_age_seconds=ctx.config.claim_timeout_seconds,
+        chain_id=ctx.chain_id,
+    )
+    for intent in stuck_sending:
+        attempt = ctx.db.get_latest_attempt_for_intent(intent.intent_id)
+        if attempt and attempt.tx_hash:
+            transition_intent(
+                ctx.db,
+                intent.intent_id,
+                IntentStatus.PENDING,
+                "sending_recover",
+                chain_id=ctx.chain_id,
+            )
+        else:
+            if attempt:
+                ctx.db.update_attempt_status(
+                    attempt.attempt_id,
+                    AttemptStatus.FAILED.value,
+                    error_code="sending_stuck",
+                    error_detail="Intent stuck in sending without broadcast",
+                )
+                ctx.nonce_manager.release(intent.signer_address, attempt.nonce)
+            transition_intent(
+                ctx.db,
+                intent.intent_id,
+                IntentStatus.CREATED,
+                "sending_stuck",
+                chain_id=ctx.chain_id,
+            )
+        metrics = get_metrics()
+        metrics.counter(INTENT_SENDING_STUCK).inc(
+            chain_id=ctx.chain_id,
+            age_bucket=">claim_timeout",
+        )

brawny/db/__init__.py

@@ -0,0 +1,78 @@
+"""Database layer with support for PostgreSQL (production) and SQLite (development)."""
+
+from brawny.db.base import (
+    ABICacheEntry,
+    BlockHashEntry,
+    BlockState,
+    Database,
+    IsolationLevel,
+    ProxyCacheEntry,
+)
+from brawny.db.migrate import Migrator, discover_migrations, get_pending_migrations
+try:
+    from brawny.db.postgres import PostgresDatabase
+except ModuleNotFoundError:
+    PostgresDatabase = None  # type: ignore
+from brawny.db.sqlite import SQLiteDatabase
+
+__all__ = [
+    # Base classes
+    "Database",
+    "IsolationLevel",
+    # Data classes
+    "BlockState",
+    "BlockHashEntry",
+    "ABICacheEntry",
+    "ProxyCacheEntry",
+    # Implementations
+    "SQLiteDatabase",
+    "PostgresDatabase",
+    # Migration
+    "Migrator",
+    "discover_migrations",
+    "get_pending_migrations",
+    # Factory
+    "create_database",
+]
+
+
+def create_database(database_url: str, **kwargs: object) -> Database:
+    """Factory function to create a database instance based on URL.
+
+    Args:
+        database_url: Database connection URL
+            - sqlite:///path/to/db.sqlite
+            - postgresql://user:pass@host:port/dbname
+        **kwargs: Additional arguments passed to the database constructor
+
+    Returns:
+        Database instance (SQLiteDatabase or PostgresDatabase)
+
+    Raises:
+        ValueError: If database URL scheme is not supported
+    """
+    circuit_breaker_failures = int(kwargs.pop("circuit_breaker_failures", 5))
+    circuit_breaker_seconds = int(kwargs.pop("circuit_breaker_seconds", 30))
+    if database_url.startswith("sqlite:///"):
+        return SQLiteDatabase(
+            database_url,
+            circuit_breaker_failures=circuit_breaker_failures,
+            circuit_breaker_seconds=circuit_breaker_seconds,
+        )
+    elif database_url.startswith(("postgresql://", "postgres://")):
+        if PostgresDatabase is None:
+            raise ValueError(
+                "Postgres support requires psycopg and psycopg-pool. "
+                "Install with: pip install psycopg[binary] psycopg-pool"
+            )
+        return PostgresDatabase(  # type: ignore
+            database_url,
+            circuit_breaker_failures=circuit_breaker_failures,
+            circuit_breaker_seconds=circuit_breaker_seconds,
+            **kwargs,
+        )
+    else:
+        raise ValueError(
+            f"Unsupported database URL: {database_url}. "
+            "Must start with 'sqlite:///', 'postgresql://', or 'postgres://'"
+        )