brawny 0.1.13__py3-none-any.whl → 0.1.22__py3-none-any.whl

brawny/scheduler/runner.py

@@ -10,10 +10,11 @@ Implements the job evaluation logic from SPEC 5.3:
 from __future__ import annotations
 
 import asyncio
+import hashlib
 import inspect
-import threading
 import time
 from uuid import UUID
+from datetime import datetime, timedelta
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import TimeoutError as FuturesTimeout
@@ -27,6 +28,7 @@ from brawny._rpc.context import set_job_context as set_rpc_job_context
 from brawny.jobs.base import Job  # Runtime import for legacy API detection
 from brawny.jobs.kv import DatabaseJobKVStore
 from brawny.logging import LogEvents, get_logger
+from brawny._rpc.errors import RPCError
 from brawny.metrics import (
     INTENTS_CREATED,
     JOB_BUILD_TIMEOUTS,
@@ -34,21 +36,30 @@ from brawny.metrics import (
     JOB_CHECK_TIMEOUTS,
     JOBS_TRIGGERED,
     LAST_INTENT_CREATED_TIMESTAMP,
+    INTENT_COOLDOWN_SKIPPED,
+    INTENT_COOLDOWN_ERRORS,
     get_metrics,
 )
-from brawny.model.contexts import BlockContext, BuildContext, CheckContext
+from brawny.model.contexts import BlockContext, BuildContext, CheckContext, CancellationToken
 from brawny.model.types import BlockInfo, Trigger
+from brawny.model.errors import DatabaseError
+from brawny.http import ApprovedHttpClient
+from brawny.alerts.health import health_alert
+from brawny.async_runtime import run_sync
+from brawny.network_guard import job_network_guard
 
 if TYPE_CHECKING:
     from brawny._rpc.clients import RPCClients
-    from brawny._rpc.manager import RPCManager
+    from brawny._rpc.clients import ReadClient
     from brawny.alerts.contracts import ContractSystem
     from brawny.config import Config
     from brawny.db.base import Database
     from brawny.lifecycle import LifecycleDispatcher
     from brawny.model.types import TxIntent, TxIntentSpec
+    from brawny.runtime_controls import RuntimeControls
 
 logger = get_logger(__name__)
+_MAX_ABANDONED_EXECUTORS = 3
 
 
 @lru_cache(maxsize=1024)
@@ -97,6 +108,7 @@ class JobResult:
     intent_created: bool = False
     skipped: bool = False
     error: Exception | None = None
+    check_token: CancellationToken | None = None
 
 
 @dataclass
@@ -120,14 +132,14 @@ class JobRunner:
     def __init__(
         self,
         db: Database,
-        rpc: RPCManager,
+        rpc: ReadClient,
         config: Config,
         jobs: dict[str, Job],
         on_intent_created: Callable[[str], None] | None = None,
         lifecycle: LifecycleDispatcher | None = None,
         contract_system: ContractSystem | None = None,
         loop: asyncio.AbstractEventLoop | None = None,
-        loop_thread_id: int | None = None,
+        controls: "RuntimeControls | None" = None,
     ) -> None:
         """Initialize job runner.
 
@@ -138,7 +150,6 @@ class JobRunner:
             jobs: Dictionary of job_id -> Job instances
             on_intent_created: Callback when intent is created (for worker scheduling)
             loop: Event loop for async job.check() support
-            loop_thread_id: Thread ID that owns the loop (for assertion)
         """
         self._db = db
         self._rpc = rpc
@@ -149,39 +160,113 @@ class JobRunner:
         self._lifecycle = lifecycle
         self._contract_system = contract_system
         self._loop = loop
-        self._loop_thread_id = loop_thread_id
+        self._http_client = ApprovedHttpClient(config.http)
+        self._controls = controls
 
         # RPC clients cache for per-job read routing
         from brawny._rpc.clients import RPCClients
         self._rpc_clients: RPCClients = RPCClients(config)
 
         # Thread pool for job check timeouts (used for sync jobs only)
-        self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="job_check")
+        self._executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="job_check")
         self._abandoned_executors = 0
+        self._last_cooldown_prune = 0.0
+
+        cooldown_cfg = config.intent_cooldown
+        logger.info(
+            "intent.cooldown.config",
+            enabled=cooldown_cfg.enabled,
+            default_seconds=cooldown_cfg.default_seconds,
+            max_seconds=cooldown_cfg.max_seconds,
+            prune_older_than_days=cooldown_cfg.prune_older_than_days,
+        )
 
     def _recreate_executor_after_timeout(self, operation: str, job_id: str) -> None:
         """Recreate the executor after a timeout to prevent deadlock.
 
         When a job times out, the worker thread continues running but the future
-        is cancelled. With max_workers=1, this blocks all subsequent job operations.
-        We recreate the executor to abandon the stuck thread and continue processing.
+        is cancelled. A stuck worker can starve the pool; recreating the executor
+        abandons the thread and allows new work to proceed.
 
         Args:
             operation: The operation that timed out ("check" or "build")
             job_id: The job that caused the timeout
         """
+        next_count = self._abandoned_executors + 1
         logger.warning(
             "runner.executor_recreated",
             operation=operation,
             job_id=job_id,
             reason="Abandoning stuck thread after timeout",
-            abandoned_executors=self._abandoned_executors + 1,
+            abandoned_executors=next_count,
         )
         # Don't wait for the stuck thread - just abandon it
         self._executor.shutdown(wait=False, cancel_futures=True)
-        self._abandoned_executors += 1
+        self._abandoned_executors = next_count
+        if self._abandoned_executors > _MAX_ABANDONED_EXECUTORS:
+            error = RuntimeError(
+                f"Exceeded abandoned executor cap ({_MAX_ABANDONED_EXECUTORS}); "
+                "possible stuck job execution threads."
+            )
+            health_alert(
+                component="brawny.scheduler.runner",
+                chain_id=self._chain_id,
+                job_id=job_id,
+                error=error,
+                level="critical",
+                action="Investigate stuck job checks/builds; restart daemon.",
+            )
+            raise error
         self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="job_check")
 
+    def _safe_get_attempts_for_intent(
+        self,
+        *,
+        intent_id: str | UUID,
+        job_id: str,
+        signer: str | None,
+        to_address: str | None,
+        existing_status: str | None,
+        existing_claimed_at: datetime | None,
+    ) -> int:
+        """Return attempts count, or -1 if unknown due to DB read failure."""
+        try:
+            intent_uuid = intent_id if isinstance(intent_id, UUID) else UUID(str(intent_id))
+            return len(self._db.get_attempts_for_intent(intent_uuid))
+        except asyncio.CancelledError:
+            raise
+        except DatabaseError:
+            logger.warning(
+                "intent.create.skipped_inflight",
+                job_id=job_id,
+                signer=signer,
+                to_address=to_address,
+                intent_id=str(intent_id),
+                existing_intent_id=str(intent_id),
+                existing_status=existing_status,
+                existing_claimed_at=existing_claimed_at,
+                existing_attempt_count=-1,
+                attempts_read_error=True,
+                chain_id=self._chain_id,
+                exc_info=True,
+            )
+            return -1
+        except Exception as e:
+            logger.error(
+                "intent.create.skipped_inflight_failed",
+                job_id=job_id,
+                signer=signer,
+                to_address=to_address,
+                intent_id=str(intent_id),
+                existing_intent_id=str(intent_id),
+                existing_status=existing_status,
+                existing_claimed_at=existing_claimed_at,
+                chain_id=self._chain_id,
+                error=str(e),
+                exc_info=True,
+            )
+            raise
+
     def process_block(self, block: BlockInfo) -> BlockResult:
         """Process a block by evaluating all enabled jobs.
 
@@ -194,18 +279,61 @@ class JobRunner:
             BlockResult with processing stats
         """
         result = BlockResult(block_number=block.block_number)
+        now_epoch = time.time()
+
+        cooldown_cfg = self._config.intent_cooldown
+        if cooldown_cfg.enabled and cooldown_cfg.prune_older_than_days > 0:
+            if now_epoch - self._last_cooldown_prune >= 24 * 3600:
+                try:
+                    deleted = self._db.prune_job_cooldowns(cooldown_cfg.prune_older_than_days)
+                    if deleted > 0:
+                        logger.info("intent.cooldown_prune", deleted=deleted)
+                except asyncio.CancelledError:
+                    raise
+                except DatabaseError as e:
+                    logger.warning(
+                        "intent.cooldown_prune_failed",
+                        chain_id=self._chain_id,
+                        error=str(e)[:200],
+                    )
+                except Exception as e:
+                    logger.error(
+                        "intent.cooldown_prune_failed",
+                        chain_id=self._chain_id,
+                        error=str(e)[:200],
+                        exc_info=True,
+                    )
+                    raise
+                self._last_cooldown_prune = now_epoch
 
         # Warm gas quote cache at start of block (for executor)
         if self._loop is not None:
             try:
-                self._loop.run_until_complete(
-                    asyncio.wait_for(self._rpc.gas_quote(), timeout=5.0)
+                run_sync(asyncio.wait_for(self._rpc.gas_quote(), timeout=5.0))
+            except asyncio.CancelledError:
+                raise
+            except (asyncio.TimeoutError, RPCError) as e:
+                logger.warning(
+                    "gas.cache_warm_failed",
+                    chain_id=self._chain_id,
+                    error=str(e),
                 )
             except Exception as e:
-                logger.warning("gas.cache_warm_failed", error=str(e))
+                logger.error(
+                    "gas.cache_warm_failed",
+                    chain_id=self._chain_id,
+                    error=str(e),
+                    exc_info=True,
+                )
+                raise
 
         # Get enabled jobs sorted by job_id
         enabled_jobs = self._db.get_enabled_jobs()
+        pause_new_intents = (
+            self._controls.is_active("pause_new_intents")
+            if self._controls is not None
+            else False
+        )
 
         for job_config in enabled_jobs:
             job_id = job_config.job_id
@@ -216,117 +344,144 @@ class JobRunner:
                 # Job in DB but not discovered - skip silently
                 # (orphaned jobs are warned about once at startup)
                 continue
+            try:
 
-            # Check interval
-            last_checked = job_config.last_checked_block_number
-            if last_checked is not None and (block.block_number - last_checked) < job.check_interval_blocks:
-                logger.debug(
-                    LogEvents.JOB_CHECK_SKIP,
-                    job_id=job_id,
-                    block_number=block.block_number,
-                    last_checked=last_checked,
-                    interval=job.check_interval_blocks,
-                )
-                continue
-            backoff_until = self._db.get_job_kv(job_id, "backoff_until_block")
-            if isinstance(backoff_until, int) and block.block_number <= backoff_until:
-                logger.debug(
-                    "job.check_backoff",
-                    job_id=job_id,
-                    block_number=block.block_number,
-                    backoff_until=backoff_until,
-                )
-                continue
-
-            if job.max_in_flight_intents is not None:
-                active_count = self._db.get_active_intent_count(
-                    job_id,
-                    chain_id=self._chain_id,
-                )
-                if active_count >= job.max_in_flight_intents:
-                    logger.warning(
-                        "job.check.backpressure",
+                # Check interval
+                last_checked = job_config.last_checked_block_number
+                if last_checked is not None and (block.block_number - last_checked) < job.check_interval_blocks:
+                    logger.debug(
+                        LogEvents.JOB_CHECK_SKIP,
                         job_id=job_id,
                         block_number=block.block_number,
-                        active_intents=active_count,
-                        limit=job.max_in_flight_intents,
+                        last_checked=last_checked,
+                        interval=job.check_interval_blocks,
                     )
-                    self._db.update_job_checked(
-                        job_id,
-                        block.block_number,
-                        triggered=False,
+                    continue
+                backoff_until = self._db.get_job_kv(job_id, "backoff_until_block")
+                if isinstance(backoff_until, int) and block.block_number <= backoff_until:
+                    logger.debug(
+                        "job.check_backoff",
+                        job_id=job_id,
+                        block_number=block.block_number,
+                        backoff_until=backoff_until,
                     )
                     continue
 
-            # Run job check
-            job_result = self._run_job_check(job, block)
-            result.jobs_checked += 1
-
-            if job_result.error:
-                if self._config.job_error_backoff_blocks > 0:
-                    self._db.set_job_kv(
+                if job.max_in_flight_intents is not None:
+                    active_count = self._db.get_active_intent_count(
                         job_id,
-                        "backoff_until_block",
-                        block.block_number + self._config.job_error_backoff_blocks,
+                        chain_id=self._chain_id,
                     )
-                result.errors.append(f"{job_id}: {job_result.error}")
-                continue
+                    if active_count >= job.max_in_flight_intents:
+                        logger.warning(
+                            "job.check.backpressure",
+                            job_id=job_id,
+                            block_number=block.block_number,
+                            active_intents=active_count,
+                            limit=job.max_in_flight_intents,
+                        )
+                        self._db.update_job_checked(
+                            job_id,
+                            block.block_number,
+                            triggered=False,
+                        )
+                        continue
+
+                # Run job check
+                job_result = self._run_job_check(job, block)
+                result.jobs_checked += 1
+
+                if job_result.error:
+                    if self._config.job_error_backoff_blocks > 0:
+                        self._db.set_job_kv(
+                            job_id,
+                            "backoff_until_block",
+                            block.block_number + self._config.job_error_backoff_blocks,
+                        )
+                    result.errors.append(f"{job_id}: {job_result.error}")
+                    continue
 
-            # Update last checked
-            self._db.update_job_checked(
-                job_id,
-                block.block_number,
-                triggered=job_result.triggered,
-            )
+                # Update last checked
+                self._db.update_job_checked(
+                    job_id,
+                    block.block_number,
+                    triggered=job_result.triggered,
+                )
 
-            if job_result.triggered and job_result.trigger:
-                result.jobs_triggered += 1
+                if job_result.triggered and job_result.trigger:
+                    result.jobs_triggered += 1
 
-                # Create intent if tx required
-                if job_result.trigger.tx_required:
-                    try:
-                        intent, is_new = self._create_intent_for_trigger(
-                            job, block, job_result.trigger
-                        )
-                        if is_new:
-                            result.intents_created += 1
-                            metrics = get_metrics()
-                            metrics.counter(INTENTS_CREATED).inc(
-                                chain_id=self._chain_id,
+                    # Create intent if tx required
+                    if job_result.trigger.tx_required:
+                        if pause_new_intents:
+                            logger.warning(
+                                "runtime.control.pause_new_intents",
                                 job_id=job_id,
+                                block_number=block.block_number,
                             )
-                            metrics.gauge(LAST_INTENT_CREATED_TIMESTAMP).set(
-                                time.time(),
+                            continue
+                        try:
+                            intent, is_new = self._create_intent_for_trigger(
+                                job,
+                                block,
+                                job_result.trigger,
+                                cancellation_token=job_result.check_token,
+                            )
+                            if is_new:
+                                result.intents_created += 1
+                                metrics = get_metrics()
+                                metrics.counter(INTENTS_CREATED).inc(
+                                    chain_id=self._chain_id,
+                                    job_id=job_id,
+                                )
+                                metrics.gauge(LAST_INTENT_CREATED_TIMESTAMP).set(
+                                    time.time(),
+                                    chain_id=self._chain_id,
+                                )
+                                if self._lifecycle:
+                                    self._lifecycle.on_triggered(
+                                        job,
+                                        job_result.trigger,
+                                        block,
+                                        intent.intent_id,
+                                    )
+                        except asyncio.CancelledError:
+                            raise
+                        except DatabaseError as e:
+                            logger.warning(
+                                "intent.creation_failed",
                                 chain_id=self._chain_id,
+                                job_id=job_id,
+                                error=str(e),
                             )
-                            if self._lifecycle:
-                                self._lifecycle.on_triggered(
-                                    job,
-                                    job_result.trigger,
-                                    block,
-                                    intent.intent_id,
+                            if self._config.job_error_backoff_blocks > 0:
+                                self._db.set_job_kv(
+                                    job_id,
+                                    "backoff_until_block",
+                                    block.block_number + self._config.job_error_backoff_blocks,
                                 )
-                    except Exception as e:
-                        logger.error(
-                            "intent.creation_failed",
-                            job_id=job_id,
-                            error=str(e),
-                        )
-                        if self._config.job_error_backoff_blocks > 0:
-                            self._db.set_job_kv(
-                                job_id,
-                                "backoff_until_block",
-                                block.block_number + self._config.job_error_backoff_blocks,
+                            result.errors.append(f"{job_id} intent: {e}")
+                        except Exception as e:
+                            logger.error(
+                                "intent.creation_failed",
+                                chain_id=self._chain_id,
+                                job_id=job_id,
+                                error=str(e),
+                                exc_info=True,
                             )
-                        result.errors.append(f"{job_id} intent: {e}")
-                else:
-                    if self._lifecycle:
-                        self._lifecycle.on_triggered(
-                            job,
-                            job_result.trigger,
-                            block,
-                            None,
-                        )
+                            raise
+                    else:
+                        if self._lifecycle:
+                            self._lifecycle.on_triggered(
+                                job,
+                                job_result.trigger,
+                                block,
+                                None,
+                            )
+            except DatabaseError as e:
+                self._handle_db_busy(e)
+                result.errors.append(f"{job_id}: {e}")
+                break
 
         return result
 
@@ -352,19 +507,16 @@ class JobRunner:
         metrics = get_metrics()
         start_time = time.perf_counter()
 
+        check_token = CancellationToken()
         # Build check context (phase-specific)
-        ctx = self._build_check_context(job, block)
+        ctx = self._build_check_context(job, block, check_token)
 
         from brawny.scripting import set_job_context
 
         try:
             # Use async path if loop is available
-            if self._loop is not None and self._loop_thread_id is not None:
-                # Assert we're on the correct thread (loop owner)
-                assert threading.get_ident() == self._loop_thread_id, \
-                    "check_job called from wrong thread"
-
-                trigger = self._loop.run_until_complete(
+            if self._loop is not None:
+                trigger = run_sync(
                     asyncio.wait_for(
                         self._run_check_async(job, block, ctx),
                         timeout=job.check_timeout_seconds,
@@ -384,11 +536,12 @@ class JobRunner:
                         block_number=ctx.block.number,
                     )
                     try:
-                        # Call with or without ctx based on signature
-                        if _accepts_ctx(type(job), "check"):
-                            return job.check(ctx)
-                        else:
-                            return job.check()
+                        with job_network_guard():
+                            # Call with or without ctx based on signature
+                            if _accepts_ctx(type(job), "check"):
+                                return job.check(ctx)
+                            else:
+                                return job.check()
                     finally:
                         set_job_context(False)
                         reset_rpc_job_context(thread_rpc_ctx_token)
@@ -415,11 +568,13 @@ class JobRunner:
                     job_id=job.job_id,
                     triggered=True,
                     trigger=trigger,
+                    check_token=check_token,
                 )
             else:
-                return JobResult(job_id=job.job_id, triggered=False)
+                return JobResult(job_id=job.job_id, triggered=False, check_token=check_token)
 
         except (asyncio.TimeoutError, FuturesTimeout):
+            check_token.cancel()
             logger.error(
                 LogEvents.JOB_CHECK_TIMEOUT,
                 job_id=job.job_id,
@@ -436,16 +591,30 @@ class JobRunner:
             return JobResult(
                 job_id=job.job_id,
                 error=TimeoutError(f"Job check timed out after {job.check_timeout_seconds}s"),
+                check_token=check_token,
             )
 
+        except asyncio.CancelledError:
+            raise
+        except (DatabaseError, RPCError) as e:
+            logger.warning(
+                "job.check.error",
+                job_id=job.job_id,
+                chain_id=self._chain_id,
+                block_number=block.block_number,
+                error=str(e),
+            )
+            return JobResult(job_id=job.job_id, error=e, check_token=check_token)
         except Exception as e:
             logger.error(
                 "job.check.error",
                 job_id=job.job_id,
+                chain_id=self._chain_id,
                 block_number=block.block_number,
                 error=str(e),
+                exc_info=True,
             )
-            return JobResult(job_id=job.job_id, error=e)
+            raise
         finally:
             duration = time.perf_counter() - start_time
             metrics.histogram(JOB_CHECK_SECONDS).observe(
@@ -470,15 +639,16 @@ class JobRunner:
         )
 
         try:
-            # Call with or without ctx based on signature
-            if _accepts_ctx(type(job), "check"):
-                result = job.check(ctx)
-            else:
-                result = job.check()
+            with job_network_guard():
+                # Call with or without ctx based on signature
+                if _accepts_ctx(type(job), "check"):
+                    result = job.check(ctx)
+                else:
+                    result = job.check()
 
-            if inspect.isawaitable(result):
-                return await result
-            return result
+                if inspect.isawaitable(result):
+                    return await result
+                return result
         finally:
             set_job_context(False)
             reset_rpc_job_context(async_rpc_ctx_token)
@@ -486,7 +656,26 @@ class JobRunner:
             _job_ctx.reset(ctx_token)
             _current_job.reset(job_token)
 
-    def _build_check_context(self, job: Job, block: BlockInfo) -> CheckContext:
+    def _handle_db_busy(self, error: DatabaseError) -> None:
+        if self._controls is None:
+            logger.warning("db.busy_sustained", error=str(error))
+            return
+        self._db.set_runtime_control(
+            control="pause_new_intents",
+            active=True,
+            expires_at=datetime.utcnow() + timedelta(seconds=300),
+            reason="db_busy",
+            actor="runner",
+            mode="auto",
+        )
+        logger.warning("db.busy_sustained", error=str(error))
+
+    def _build_check_context(
+        self,
+        job: Job,
+        block: BlockInfo,
+        cancellation_token: CancellationToken,
+    ) -> CheckContext:
         """Build a CheckContext for job check phase.
 
         Args:
@@ -508,7 +697,7 @@ class JobRunner:
             number=block.block_number,
             timestamp=block.timestamp,
             hash=block.block_hash,
-            base_fee=0,  # TODO: Get from block if available
+            base_fee=block.base_fee,
             chain_id=block.chain_id,
         )
 
@@ -520,8 +709,10 @@ class JobRunner:
             kv=DatabaseJobKVStore(self._db, job.job_id),
             job_id=job.job_id,
             rpc=rpc,
+            http=self._http_client,
             logger=logger.bind(job_id=job.job_id, chain_id=block.chain_id),
             contracts=contracts,
+            cancellation_token=cancellation_token,
             _db=self._db,
         )
 
@@ -550,7 +741,7 @@ class JobRunner:
             number=block.block_number,
             timestamp=block.timestamp,
             hash=block.block_hash,
-            base_fee=0,
+            base_fee=block.base_fee,
             chain_id=block.chain_id,
         )
 
@@ -562,6 +753,7 @@ class JobRunner:
             job_id=job.job_id,
             signer_address=signer_address,
             rpc=rpc,
+            http=self._http_client,
             logger=logger.bind(job_id=job.job_id, chain_id=block.chain_id),
             contracts=contracts,
             kv=DatabaseJobKVStore(self._db, job.job_id),  # KVReader (read-only access)
@@ -572,6 +764,7 @@ class JobRunner:
         job: Job,
         block: BlockInfo,
         trigger: Trigger,
+        cancellation_token: CancellationToken | None = None,
     ) -> tuple[TxIntent | None, bool]:
         """Create a transaction intent for a triggered job.
 
@@ -580,11 +773,80 @@ class JobRunner:
             block: Block information
             trigger: Trigger with intent details
         """
+        from brawny.model.errors import CancelledCheckError
         from brawny.tx.intent import create_intent
 
         # Resolve signer address for build context
         signer_address = job.signer_address
 
+        cooldown_cfg = self._config.intent_cooldown
+        if cooldown_cfg.enabled:
+            job_override = job.cooldown_seconds
+            effective_cooldown = (
+                job_override if job_override is not None else cooldown_cfg.default_seconds
+            )
+            if effective_cooldown < 0:
+                raise ValueError("cooldown_seconds cannot be negative")
+            if effective_cooldown == 0:
+                pass
+            else:
+                if effective_cooldown > cooldown_cfg.max_seconds:
+                    effective_cooldown = cooldown_cfg.max_seconds
+
+                now = int(time.time())
+                target_key = job.cooldown_key(trigger)
+                if target_key is None:
+                    target_key = "*"
+                target_key_str = str(target_key)
+                if len(target_key_str) > 64:
+                    target_key_str = hashlib.sha256(target_key_str.encode("utf-8")).hexdigest()[:16]
+
+                cooldown_key = (
+                    f"{job.job_id}:{self._chain_id}:{signer_address.lower()}:{target_key_str}"
+                )
+                try:
+                    allowed, last_intent_at = self._db.should_create_intent(
+                        cooldown_key,
+                        now,
+                        int(effective_cooldown),
+                    )
+                except asyncio.CancelledError:
+                    raise
+                except DatabaseError as e:
+                    logger.warning(
+                        "intent.cooldown_error",
+                        job_id=job.job_id,
+                        chain_id=self._chain_id,
+                        error=str(e)[:200],
+                    )
+                    metrics = get_metrics()
+                    metrics.counter(INTENT_COOLDOWN_ERRORS).inc(chain_id=self._chain_id)
+                except Exception as e:
+                    logger.error(
+                        "intent.cooldown_error",
+                        job_id=job.job_id,
+                        chain_id=self._chain_id,
+                        error=str(e)[:200],
+                        exc_info=True,
+                    )
+                    raise
+                else:
+                    if not allowed:
+                        last_seen = last_intent_at if last_intent_at is not None else now
+                        next_allowed_at = last_seen + int(effective_cooldown)
+                        logger.debug(
+                            "intent.cooldown_skip",
+                            reason="cooldown_active",
+                            job_id=job.job_id,
+                            chain_id=self._chain_id,
+                            signer=signer_address,
+                            cooldown_seconds=int(effective_cooldown),
+                            next_allowed_at=next_allowed_at,
+                        )
+                        metrics = get_metrics()
+                        metrics.counter(INTENT_COOLDOWN_SKIPPED).inc(chain_id=self._chain_id)
+                        return None, False
+
         # Build context for build_tx (phase-specific)
         ctx = self._build_build_context(job, block, trigger, signer_address)
 
@@ -597,15 +859,16 @@ class JobRunner:
             rpc_ctx_token = set_rpc_job_context(job.job_id)
             set_job_context(True)
             try:
-                # Support legacy build_intent(trigger) API:
-                # If job has build_intent but didn't override build_tx, use legacy API
-                if hasattr(job, "build_intent") and type(job).build_tx is Job.build_tx:
-                    return job.build_intent(ctx.trigger)
-                # Call with or without ctx based on signature
-                if _accepts_ctx(type(job), "build_tx"):
-                    return job.build_tx(ctx)
-                else:
-                    return job.build_tx()
+                with job_network_guard():
+                    # Support legacy build_intent(trigger) API:
+                    # If job has build_intent but didn't override build_tx, use legacy API
+                    if hasattr(job, "build_intent") and type(job).build_tx is Job.build_tx:
+                        return job.build_intent(ctx.trigger)
+                    # Call with or without ctx based on signature
+                    if _accepts_ctx(type(job), "build_tx"):
+                        return job.build_tx(ctx)
+                    else:
+                        return job.build_tx()
             finally:
                 set_job_context(False)
                 reset_rpc_job_context(rpc_ctx_token)
@@ -660,23 +923,30 @@ class JobRunner:
                 existing_status = existing.get("status")
                 existing_claimed_at = existing.get("claimed_at")
                 existing_attempts = 0
+                attempts_read_error = False
                 if existing_id:
-                    try:
-                        existing_attempts = len(
-                            self._db.get_attempts_for_intent(UUID(existing_id))
-                        )
-                    except Exception:
-                        existing_attempts = 0
-                logger.info(
-                    "intent.create.skipped_inflight",
-                    job_id=job.job_id,
-                    signer=signer_address,
-                    to_address=spec.to_address,
-                    existing_intent_id=str(existing_id) if existing_id else None,
-                    existing_status=existing_status,
-                    existing_claimed_at=existing_claimed_at,
-                    existing_attempt_count=existing_attempts,
-                )
+                    existing_attempts = self._safe_get_attempts_for_intent(
+                        intent_id=existing_id,
+                        job_id=job.job_id,
+                        signer=signer_address,
+                        to_address=spec.to_address,
+                        existing_status=existing_status,
+                        existing_claimed_at=existing_claimed_at,
+                    )
+                    attempts_read_error = existing_attempts < 0
+                if not attempts_read_error:
+                    logger.info(
+                        "intent.create.skipped_inflight",
+                        job_id=job.job_id,
+                        signer=signer_address,
+                        to_address=spec.to_address,
+                        intent_id=str(existing_id) if existing_id else None,
+                        existing_intent_id=str(existing_id) if existing_id else None,
+                        existing_status=existing_status,
+                        existing_claimed_at=existing_claimed_at,
+                        existing_attempt_count=existing_attempts,
+                        chain_id=self._chain_id,
+                    )
                 if len(inflight) > 1:
                     logger.warning(
                         "invariant.multiple_inflight_intents",
@@ -687,16 +957,20 @@ class JobRunner:
                     )
                 return None, False
 
-            intent, is_new = create_intent(
-                db=self._db,
-                job_id=job.job_id,
-                chain_id=self._chain_id,
-                spec=spec,
-                idem_parts=idem_parts,
-                broadcast_group=broadcast_group,
-                broadcast_endpoints=broadcast_endpoints,
-                trigger=trigger,
-            )
+            try:
+                intent, is_new = create_intent(
+                    db=self._db,
+                    job_id=job.job_id,
+                    chain_id=self._chain_id,
+                    spec=spec,
+                    idem_parts=idem_parts,
+                    broadcast_group=broadcast_group,
+                    broadcast_endpoints=broadcast_endpoints,
+                    trigger=trigger,
+                    cancellation_token=cancellation_token,
+                )
+            except CancelledCheckError:
+                return None, False
 
         if is_new and self._on_intent_created:
             self._on_intent_created(str(intent.intent_id))