brawny 0.1.13__py3-none-any.whl

brawny/obs/__init__.py

@@ -0,0 +1,67 @@
+"""Observability module for brawny.
+
+Provides structured logging, liveness heartbeats, and readiness health checks.
+
+See LOGGING_METRICS_PLAN.md for design rationale and usage patterns.
+
+Quick Reference:
+    # Logging via emit() gateway
+    from brawny.obs import emit, get_logger, bind_intent
+
+    log = get_logger(worker_id=1, chain_id=1)
+    log = bind_intent(log, intent_id=str(intent.intent_id), job_id=intent.job_id)
+    emit(log, level="info", event="tx", result="broadcast", tx_hash=hash)
+
+    # Heartbeat for liveness
+    from brawny.obs import get_heartbeat
+
+    heartbeat = get_heartbeat("block_poller")
+    heartbeat.beat()  # Call in loop
+
+    # Health state for readiness
+    from brawny.obs import get_health_state
+
+    health = get_health_state()
+    health.update_db(db.ping())
+    if not health.is_ready():
+        return 503
+"""
+
+from brawny.obs.emit import (
+    ALLOWED,
+    RUN_ID,
+    bind_attempt,
+    bind_intent,
+    emit,
+    get_logger,
+)
+from brawny.obs.health import (
+    HealthState,
+    get_health_state,
+    reset_health_state,
+)
+from brawny.obs.heartbeat import (
+    Heartbeat,
+    all_heartbeat_ages,
+    any_stale,
+    get_heartbeat,
+)
+
+__all__ = [
+    # emit.py
+    "ALLOWED",
+    "RUN_ID",
+    "bind_attempt",
+    "bind_intent",
+    "emit",
+    "get_logger",
+    # health.py
+    "HealthState",
+    "get_health_state",
+    "reset_health_state",
+    # heartbeat.py
+    "Heartbeat",
+    "all_heartbeat_ages",
+    "any_stale",
+    "get_heartbeat",
+]

brawny/obs/emit.py

@@ -0,0 +1,158 @@
+"""Structured logging gateway for brawny.
+
+The emit() function is the single enforcement choke point for all logging.
+It validates event/result pairs, normalizes error fields, and controls trace inclusion.
+
+See LOGGING_METRICS_PLAN.md for design rationale.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+from typing import TYPE_CHECKING
+
+import structlog
+
+if TYPE_CHECKING:
+    from typing import Any
+
+# Run ID for correlating logs across restarts
+RUN_ID = os.environ.get("BRAWNY_RUN_ID") or f"run_{uuid.uuid4().hex[:12]}"
+
+# Allowed event families and their valid results
+# Any (event, result) pair not in this dict will raise ValueError
+ALLOWED: dict[str, set[str]] = {
+    "job.check": {"triggered", "skipped", "timeout", "error"},
+    "intent": {"created", "claimed", "executed", "failed", "status"},
+    "tx": {"signed", "broadcast", "confirmed", "failed", "replaced"},
+    "rpc": {"ok", "error", "timeout"},
+    "nonce": {"reserved", "released", "reconciled"},
+    "block": {"processed", "reorg"},
+    "system": {"started", "draining", "shutdown"},
+}
+
+# Error message truncation limit
+ERROR_MESSAGE_MAX_LENGTH = 500
+
+
+def emit(
+    log: structlog.stdlib.BoundLogger,
+    *,
+    level: str,
+    event: str,
+    result: str,
+    err: Exception | None = None,
+    is_terminal: bool = False,
+    **fields: Any,
+) -> None:
+    """Emit a structured log event.
+
+    This is the single enforcement choke point for all logging in brawny.
+    It validates event/result pairs, normalizes error fields, and controls
+    stack trace inclusion.
+
+    Args:
+        log: Bound logger instance
+        level: Log level ("debug", "info", "warning", "error")
+        event: Event family (e.g., "tx", "intent", "job.check")
+        result: Event result (e.g., "confirmed", "failed", "triggered")
+        err: Optional exception for error events
+        is_terminal: If True and err is provided, include stack trace
+        **fields: Additional context fields
+
+    Raises:
+        ValueError: If (event, result) pair is not in ALLOWED
+
+    Example:
+        emit(log, level="info", event="tx", result="broadcast", tx_hash=hash)
+        emit(log, level="error", event="tx", result="failed", err=e, is_terminal=True)
+    """
+    # Validate event/result pair
+    if event not in ALLOWED:
+        raise ValueError(f"Invalid event family: {event!r}. Must be one of: {sorted(ALLOWED.keys())}")
+    if result not in ALLOWED[event]:
+        raise ValueError(
+            f"Invalid result {result!r} for event {event!r}. Must be one of: {sorted(ALLOWED[event])}"
+        )
+
+    # Normalize error fields
+    if err is not None:
+        msg = str(err)
+        fields["error_type"] = type(err).__name__
+        # Truncate long error messages
+        if len(msg) > ERROR_MESSAGE_MAX_LENGTH:
+            fields["error"] = msg[:ERROR_MESSAGE_MAX_LENGTH] + "..."
+        else:
+            fields["error"] = msg
+
+    # Get the logging function for this level
+    log_fn = getattr(log, level.lower())
+
+    # Dispatch - exc_info must be a kwarg to logger, not a field
+    if err is not None and is_terminal:
+        log_fn(event, result=result, exc_info=True, **fields)
+    else:
+        log_fn(event, result=result, **fields)
+
+
+def get_logger(**bind: Any) -> structlog.stdlib.BoundLogger:
+    """Get a logger with run_id bound.
+
+    Use this at component boundaries to get a base logger.
+
+    Args:
+        **bind: Additional fields to bind (e.g., worker_id, chain_id)
+
+    Returns:
+        Bound logger with run_id and any additional fields
+
+    Example:
+        log = get_logger(worker_id=1, chain_id=1)
+    """
+    return structlog.get_logger("brawny").bind(run_id=RUN_ID, **bind)
+
+
+def bind_intent(
+    log: structlog.stdlib.BoundLogger,
+    *,
+    intent_id: str,
+    job_id: str,
+) -> structlog.stdlib.BoundLogger:
+    """Bind intent context to a logger.
+
+    Use when processing a specific intent.
+
+    Args:
+        log: Base logger
+        intent_id: Intent UUID as string
+        job_id: Job identifier
+
+    Returns:
+        Logger with intent context bound
+    """
+    return log.bind(intent_id=intent_id, job_id=job_id)
+
+
+def bind_attempt(
+    log: structlog.stdlib.BoundLogger,
+    *,
+    attempt_id: str,
+    nonce: int | None = None,
+) -> structlog.stdlib.BoundLogger:
+    """Bind attempt context to a logger.
+
+    Use when processing a specific transaction attempt.
+
+    Args:
+        log: Base logger (typically with intent context already bound)
+        attempt_id: Attempt UUID as string
+        nonce: Transaction nonce (optional, but include when known)
+
+    Returns:
+        Logger with attempt context bound
+    """
+    # Use 'is not None' to correctly handle nonce=0
+    if nonce is not None:
+        return log.bind(attempt_id=attempt_id, nonce=nonce)
+    return log.bind(attempt_id=attempt_id)

brawny/obs/health.py

@@ -0,0 +1,175 @@
+"""Cached health state for readiness probes.
+
+Readiness probes (/readyz) must be fast and never block on slow checks.
+This module provides cached health state that's updated by background loops.
+
+See LOGGING_METRICS_PLAN.md Section 4.1.3 for design rationale.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from threading import Lock
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from threading import Thread
+
+
+@dataclass
+class HealthState:
+    """Cached health state for readiness checks.
+
+    Updated by background loops, read by /readyz endpoint.
+    All fields are protected by a lock for thread safety.
+
+    NOT READY when:
+    - shutdown_requested is True (draining)
+    - db_ok is False
+    - rpc_ok is False
+    - workers_ok is False
+
+    Usage:
+        # Background loop updates state
+        health_state.update_db(db.ping())
+        health_state.update_rpc(rpc.any_healthy())
+        health_state.update_workers(worker_threads)
+
+        # /readyz reads cached state (fast, never blocks)
+        if not health_state.is_ready():
+            return Response(status_code=503)
+        return Response(status_code=200)
+    """
+
+    # Cached component health
+    db_ok: bool = field(default=True)
+    rpc_ok: bool = field(default=True)
+    workers_ok: bool = field(default=True)
+
+    # Draining state
+    shutdown_requested: bool = field(default=False)
+
+    # Last update timestamps (for staleness detection)
+    last_db_check: float = field(default=0.0)
+    last_rpc_check: float = field(default=0.0)
+    last_workers_check: float = field(default=0.0)
+
+    # Thread safety
+    _lock: Lock = field(default_factory=Lock, repr=False)
+
+    def update_db(self, ok: bool) -> None:
+        """Update database health state."""
+        with self._lock:
+            self.db_ok = ok
+            self.last_db_check = time.time()
+
+    def update_rpc(self, ok: bool) -> None:
+        """Update RPC health state."""
+        with self._lock:
+            self.rpc_ok = ok
+            self.last_rpc_check = time.time()
+
+    def update_workers(self, threads: list["Thread"]) -> None:
+        """Update worker health state.
+
+        Args:
+            threads: List of worker threads
+        """
+        with self._lock:
+            self.workers_ok = any(t.is_alive() for t in threads) if threads else False
+            self.last_workers_check = time.time()
+
+    def request_shutdown(self) -> None:
+        """Mark the system as draining.
+
+        Call this at the start of graceful shutdown.
+        /readyz will return 503 immediately.
+        """
+        with self._lock:
+            self.shutdown_requested = True
+
+    def is_ready(self) -> bool:
+        """Check if the system is ready to accept work.
+
+        Returns:
+            True if ready, False if not ready (should return 503)
+        """
+        with self._lock:
+            if self.shutdown_requested:
+                return False
+            if not self.db_ok:
+                return False
+            if not self.rpc_ok:
+                return False
+            if not self.workers_ok:
+                return False
+            return True
+
+    def readiness_reasons(self) -> list[str]:
+        """Get human-readable reasons for not being ready.
+
+        Useful for /healthz diagnostics.
+
+        Returns:
+            List of reasons why the system is not ready, empty if ready
+        """
+        reasons = []
+        with self._lock:
+            if self.shutdown_requested:
+                reasons.append("shutdown_requested")
+            if not self.db_ok:
+                reasons.append("db_unhealthy")
+            if not self.rpc_ok:
+                reasons.append("rpc_unhealthy")
+            if not self.workers_ok:
+                reasons.append("no_workers_alive")
+        return reasons
+
+    def to_dict(self) -> dict[str, object]:
+        """Get full health state as a dictionary.
+
+        Useful for /healthz JSON response.
+        """
+        with self._lock:
+            # Compute ready inline to avoid deadlock (is_ready also acquires lock)
+            ready = (
+                not self.shutdown_requested
+                and self.db_ok
+                and self.rpc_ok
+                and self.workers_ok
+            )
+            return {
+                "ready": ready,
+                "shutdown_requested": self.shutdown_requested,
+                "db_ok": self.db_ok,
+                "rpc_ok": self.rpc_ok,
+                "workers_ok": self.workers_ok,
+                "last_db_check": self.last_db_check,
+                "last_rpc_check": self.last_rpc_check,
+                "last_workers_check": self.last_workers_check,
+            }
+
+
+# Global health state singleton
+_health_state: HealthState | None = None
+_health_state_lock = Lock()
+
+
+def get_health_state() -> HealthState:
+    """Get the global health state singleton.
+
+    Creates the singleton on first access.
+    """
+    global _health_state
+    with _health_state_lock:
+        if _health_state is None:
+            _health_state = HealthState()
+        return _health_state
+
+
+def reset_health_state() -> None:
+    """Reset the global health state (for testing)."""
+    global _health_state
+    with _health_state_lock:
+        _health_state = None

brawny/obs/heartbeat.py

@@ -0,0 +1,133 @@
+"""Heartbeat-based liveness for brawny.
+
+Liveness is NOT "can we respond to HTTP?" - it's "is the core loop making progress?"
+
+The Heartbeat class tracks when critical loops last made progress.
+/livez returns 503 when the heartbeat is stale (no progress in 30s).
+
+See LOGGING_METRICS_PLAN.md Section 4.1.2 for design rationale.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from threading import Lock
+
+
+@dataclass
+class Heartbeat:
+    """Track liveness of a critical loop.
+
+    A heartbeat is considered stale if beat() hasn't been called within
+    max_age_seconds. This indicates the loop is stuck or deadlocked.
+
+    Thread-safe: beat() and is_stale() can be called from different threads.
+
+    Usage:
+        heartbeat = Heartbeat()
+
+        # In the critical loop
+        while not stop_event.is_set():
+            heartbeat.beat()
+            # ... do work ...
+
+        # In /livez endpoint
+        if heartbeat.is_stale():
+            return Response(status_code=503)
+        return Response(status_code=200)
+    """
+
+    last_beat_ts: float = field(default=0.0)
+    _lock: Lock = field(default_factory=Lock, repr=False)
+
+    def beat(self) -> None:
+        """Record that the loop is making progress.
+
+        Call this at the start of each loop iteration.
+        """
+        with self._lock:
+            self.last_beat_ts = time.time()
+
+    def is_stale(self, max_age_seconds: float = 30.0) -> bool:
+        """Check if the heartbeat is stale.
+
+        Args:
+            max_age_seconds: Maximum allowed time since last beat.
+                Default 30s is a reasonable choice for most loops.
+
+        Returns:
+            True if:
+            - beat() was never called (last_beat_ts == 0.0), OR
+            - More than max_age_seconds have passed since last beat
+        """
+        with self._lock:
+            if self.last_beat_ts == 0.0:
+                # Never started
+                return True
+            return (time.time() - self.last_beat_ts) > max_age_seconds
+
+    def age_seconds(self) -> float:
+        """Get the age of the last heartbeat in seconds.
+
+        Returns:
+            Seconds since last beat, or float('inf') if never beat.
+        """
+        with self._lock:
+            if self.last_beat_ts == 0.0:
+                return float("inf")
+            return time.time() - self.last_beat_ts
+
+
+# Global heartbeats for critical loops
+_heartbeats: dict[str, Heartbeat] = {}
+_heartbeats_lock = Lock()
+
+
+def get_heartbeat(name: str) -> Heartbeat:
+    """Get or create a named heartbeat.
+
+    Use this to track different critical loops:
+    - "block_poller" for the block processing loop
+    - "monitor" for the transaction monitor loop
+
+    Args:
+        name: Identifier for the heartbeat
+
+    Returns:
+        The Heartbeat instance for this name
+    """
+    with _heartbeats_lock:
+        if name not in _heartbeats:
+            _heartbeats[name] = Heartbeat()
+        return _heartbeats[name]
+
+
+def any_stale(max_age_seconds: float = 30.0) -> bool:
+    """Check if any registered heartbeat is stale.
+
+    Use this in /livez to check overall system liveness.
+
+    Args:
+        max_age_seconds: Maximum allowed time since last beat
+
+    Returns:
+        True if any heartbeat is stale
+    """
+    with _heartbeats_lock:
+        if not _heartbeats:
+            # No heartbeats registered yet - system is starting up
+            return False
+        return any(hb.is_stale(max_age_seconds) for hb in _heartbeats.values())
+
+
+def all_heartbeat_ages() -> dict[str, float]:
+    """Get the age of all registered heartbeats.
+
+    Useful for /healthz diagnostics endpoint.
+
+    Returns:
+        Dict mapping heartbeat name to age in seconds
+    """
+    with _heartbeats_lock:
+        return {name: hb.age_seconds() for name, hb in _heartbeats.items()}

brawny/reconciliation.py

@@ -0,0 +1,108 @@
+"""Startup reconciliation for detecting and repairing inconsistent state.
+
+Phase 1 implementation: runs at startup only.
+Phase 2 will add periodic reconciliation after metrics prove stability.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import TYPE_CHECKING
+
+from brawny.logging import get_logger
+from brawny.metrics import get_metrics
+
+if TYPE_CHECKING:
+    from brawny.db.base import Database
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class ReconciliationStats:
+    """Statistics from a reconciliation run."""
+
+    orphaned_claims_cleared: int = 0
+    orphaned_nonces_released: int = 0
+    pending_without_attempts: int = 0
+    stale_claims: int = 0
+
+
+def reconcile_startup(db: Database, chain_id: int) -> ReconciliationStats:
+    """Run reconciliation checks at startup.
+
+    Repairs:
+    - Orphaned claims (status != claimed but claim_token set, stale)
+    - Orphaned nonces (reserved but intent is terminal and stale)
+
+    Detects (logs only, no repair):
+    - Pending intents without attempts (data integrity issue)
+    - Stale claimed intents (worker may have crashed)
+
+    Args:
+        db: Database connection
+        chain_id: Chain ID to reconcile
+
+    Returns:
+        Statistics from the reconciliation run
+    """
+    stats = ReconciliationStats()
+
+    # Repair: clear orphaned claims (with time guard)
+    stats.orphaned_claims_cleared = db.clear_orphaned_claims(
+        chain_id, older_than_minutes=2
+    )
+    if stats.orphaned_claims_cleared > 0:
+        logger.warning(
+            "reconciliation.orphaned_claims_cleared",
+            count=stats.orphaned_claims_cleared,
+            chain_id=chain_id,
+        )
+
+    # Repair: release orphaned nonces (with time guard)
+    stats.orphaned_nonces_released = db.release_orphaned_nonces(
+        chain_id, older_than_minutes=5
+    )
+    if stats.orphaned_nonces_released > 0:
+        logger.warning(
+            "reconciliation.orphaned_nonces_released",
+            count=stats.orphaned_nonces_released,
+            chain_id=chain_id,
+        )
+
+    # Detect: pending without attempts (log only - needs investigation)
+    stats.pending_without_attempts = db.count_pending_without_attempts(chain_id)
+    if stats.pending_without_attempts > 0:
+        logger.error(
+            "reconciliation.pending_without_attempts",
+            count=stats.pending_without_attempts,
+            chain_id=chain_id,
+            action="manual_investigation_required",
+        )
+
+    # Detect: stale claims (log only - may self-recover or need intervention)
+    stats.stale_claims = db.count_stale_claims(chain_id, older_than_minutes=10)
+    if stats.stale_claims > 0:
+        logger.warning(
+            "reconciliation.stale_claims",
+            count=stats.stale_claims,
+            chain_id=chain_id,
+        )
+
+    # Emit metrics
+    metrics = get_metrics()
+    metrics.gauge("brawny_reconciliation_orphaned_claims").set(
+        stats.orphaned_claims_cleared, chain_id=chain_id
+    )
+    metrics.gauge("brawny_reconciliation_orphaned_nonces").set(
+        stats.orphaned_nonces_released, chain_id=chain_id
+    )
+    metrics.gauge("brawny_reconciliation_pending_no_attempts").set(
+        stats.pending_without_attempts, chain_id=chain_id
+    )
+    metrics.gauge("brawny_reconciliation_stale_claims").set(
+        stats.stale_claims, chain_id=chain_id
+    )
+
+    logger.info("reconciliation.completed", **asdict(stats), chain_id=chain_id)
+    return stats

brawny/scheduler/__init__.py

@@ -0,0 +1,19 @@
+"""Block poller, reorg detection, and job scheduler."""
+
+from brawny.scheduler.poller import BlockPoller, PollResult
+from brawny.scheduler.reorg import ReorgDetector, ReorgResult
+from brawny.scheduler.runner import BlockResult, JobResult, JobRunner
+from brawny.scheduler.shutdown import ShutdownContext, ShutdownHandler, ShutdownStats
+
+__all__ = [
+    "BlockPoller",
+    "PollResult",
+    "ReorgDetector",
+    "ReorgResult",
+    "JobRunner",
+    "JobResult",
+    "BlockResult",
+    "ShutdownHandler",
+    "ShutdownContext",
+    "ShutdownStats",
+]