brawny 0.1.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brawny/__init__.py +106 -0
- brawny/_context.py +232 -0
- brawny/_rpc/__init__.py +38 -0
- brawny/_rpc/broadcast.py +172 -0
- brawny/_rpc/clients.py +98 -0
- brawny/_rpc/context.py +49 -0
- brawny/_rpc/errors.py +252 -0
- brawny/_rpc/gas.py +158 -0
- brawny/_rpc/manager.py +982 -0
- brawny/_rpc/selector.py +156 -0
- brawny/accounts.py +534 -0
- brawny/alerts/__init__.py +132 -0
- brawny/alerts/abi_resolver.py +530 -0
- brawny/alerts/base.py +152 -0
- brawny/alerts/context.py +271 -0
- brawny/alerts/contracts.py +635 -0
- brawny/alerts/encoded_call.py +201 -0
- brawny/alerts/errors.py +267 -0
- brawny/alerts/events.py +680 -0
- brawny/alerts/function_caller.py +364 -0
- brawny/alerts/health.py +185 -0
- brawny/alerts/routing.py +118 -0
- brawny/alerts/send.py +364 -0
- brawny/api.py +660 -0
- brawny/chain.py +93 -0
- brawny/cli/__init__.py +16 -0
- brawny/cli/app.py +17 -0
- brawny/cli/bootstrap.py +37 -0
- brawny/cli/commands/__init__.py +41 -0
- brawny/cli/commands/abi.py +93 -0
- brawny/cli/commands/accounts.py +632 -0
- brawny/cli/commands/console.py +495 -0
- brawny/cli/commands/contract.py +139 -0
- brawny/cli/commands/health.py +112 -0
- brawny/cli/commands/init_project.py +86 -0
- brawny/cli/commands/intents.py +130 -0
- brawny/cli/commands/job_dev.py +254 -0
- brawny/cli/commands/jobs.py +308 -0
- brawny/cli/commands/logs.py +87 -0
- brawny/cli/commands/maintenance.py +182 -0
- brawny/cli/commands/migrate.py +51 -0
- brawny/cli/commands/networks.py +253 -0
- brawny/cli/commands/run.py +249 -0
- brawny/cli/commands/script.py +209 -0
- brawny/cli/commands/signer.py +248 -0
- brawny/cli/helpers.py +265 -0
- brawny/cli_templates.py +1445 -0
- brawny/config/__init__.py +74 -0
- brawny/config/models.py +404 -0
- brawny/config/parser.py +633 -0
- brawny/config/routing.py +55 -0
- brawny/config/validation.py +246 -0
- brawny/daemon/__init__.py +14 -0
- brawny/daemon/context.py +69 -0
- brawny/daemon/core.py +702 -0
- brawny/daemon/loops.py +327 -0
- brawny/db/__init__.py +78 -0
- brawny/db/base.py +986 -0
- brawny/db/base_new.py +165 -0
- brawny/db/circuit_breaker.py +97 -0
- brawny/db/global_cache.py +298 -0
- brawny/db/mappers.py +182 -0
- brawny/db/migrate.py +349 -0
- brawny/db/migrations/001_init.sql +186 -0
- brawny/db/migrations/002_add_included_block.sql +7 -0
- brawny/db/migrations/003_add_broadcast_at.sql +10 -0
- brawny/db/migrations/004_broadcast_binding.sql +20 -0
- brawny/db/migrations/005_add_retry_after.sql +9 -0
- brawny/db/migrations/006_add_retry_count_column.sql +11 -0
- brawny/db/migrations/007_add_gap_tracking.sql +18 -0
- brawny/db/migrations/008_add_transactions.sql +72 -0
- brawny/db/migrations/009_add_intent_metadata.sql +5 -0
- brawny/db/migrations/010_add_nonce_gap_index.sql +9 -0
- brawny/db/migrations/011_add_job_logs.sql +24 -0
- brawny/db/migrations/012_add_claimed_by.sql +5 -0
- brawny/db/ops/__init__.py +29 -0
- brawny/db/ops/attempts.py +108 -0
- brawny/db/ops/blocks.py +83 -0
- brawny/db/ops/cache.py +93 -0
- brawny/db/ops/intents.py +296 -0
- brawny/db/ops/jobs.py +110 -0
- brawny/db/ops/logs.py +97 -0
- brawny/db/ops/nonces.py +322 -0
- brawny/db/postgres.py +2535 -0
- brawny/db/postgres_new.py +196 -0
- brawny/db/queries.py +584 -0
- brawny/db/sqlite.py +2733 -0
- brawny/db/sqlite_new.py +191 -0
- brawny/history.py +126 -0
- brawny/interfaces.py +136 -0
- brawny/invariants.py +155 -0
- brawny/jobs/__init__.py +26 -0
- brawny/jobs/base.py +287 -0
- brawny/jobs/discovery.py +233 -0
- brawny/jobs/job_validation.py +111 -0
- brawny/jobs/kv.py +125 -0
- brawny/jobs/registry.py +283 -0
- brawny/keystore.py +484 -0
- brawny/lifecycle.py +551 -0
- brawny/logging.py +290 -0
- brawny/metrics.py +594 -0
- brawny/model/__init__.py +53 -0
- brawny/model/contexts.py +319 -0
- brawny/model/enums.py +70 -0
- brawny/model/errors.py +194 -0
- brawny/model/events.py +93 -0
- brawny/model/startup.py +20 -0
- brawny/model/types.py +483 -0
- brawny/networks/__init__.py +96 -0
- brawny/networks/config.py +269 -0
- brawny/networks/manager.py +423 -0
- brawny/obs/__init__.py +67 -0
- brawny/obs/emit.py +158 -0
- brawny/obs/health.py +175 -0
- brawny/obs/heartbeat.py +133 -0
- brawny/reconciliation.py +108 -0
- brawny/scheduler/__init__.py +19 -0
- brawny/scheduler/poller.py +472 -0
- brawny/scheduler/reorg.py +632 -0
- brawny/scheduler/runner.py +708 -0
- brawny/scheduler/shutdown.py +371 -0
- brawny/script_tx.py +297 -0
- brawny/scripting.py +251 -0
- brawny/startup.py +76 -0
- brawny/telegram.py +393 -0
- brawny/testing.py +108 -0
- brawny/tx/__init__.py +41 -0
- brawny/tx/executor.py +1071 -0
- brawny/tx/fees.py +50 -0
- brawny/tx/intent.py +423 -0
- brawny/tx/monitor.py +628 -0
- brawny/tx/nonce.py +498 -0
- brawny/tx/replacement.py +456 -0
- brawny/tx/utils.py +26 -0
- brawny/utils.py +205 -0
- brawny/validation.py +69 -0
- brawny-0.1.13.dist-info/METADATA +156 -0
- brawny-0.1.13.dist-info/RECORD +141 -0
- brawny-0.1.13.dist-info/WHEEL +5 -0
- brawny-0.1.13.dist-info/entry_points.txt +2 -0
- brawny-0.1.13.dist-info/top_level.txt +1 -0
brawny/tx/executor.py
ADDED
|
@@ -0,0 +1,1071 @@
|
|
|
1
|
+
"""Transaction executor for signing, broadcasting, and monitoring transactions.
|
|
2
|
+
|
|
3
|
+
Implements the tx execution flow from SPEC 9:
|
|
4
|
+
1. Validate deadline
|
|
5
|
+
2. Reserve nonce
|
|
6
|
+
3. Build tx dict with gas estimation
|
|
7
|
+
4. Sign transaction
|
|
8
|
+
5. Broadcast transaction
|
|
9
|
+
6. Monitor for confirmation
|
|
10
|
+
7. Handle replacement for stuck txs
|
|
11
|
+
|
|
12
|
+
Golden Rule: Intents are persisted BEFORE signing - the executor only
|
|
13
|
+
works with already-persisted intents.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import random
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from datetime import datetime, timezone, timedelta
|
|
21
|
+
from enum import Enum
|
|
22
|
+
from typing import TYPE_CHECKING, Callable
|
|
23
|
+
from uuid import UUID, uuid4
|
|
24
|
+
|
|
25
|
+
from web3 import Web3
|
|
26
|
+
|
|
27
|
+
from brawny.logging import LogEvents, get_logger
|
|
28
|
+
from brawny.tx.utils import normalize_tx_dict
|
|
29
|
+
from brawny.metrics import (
|
|
30
|
+
ATTEMPT_WRITE_FAILURES,
|
|
31
|
+
SIMULATION_NETWORK_ERRORS,
|
|
32
|
+
SIMULATION_RETRIES,
|
|
33
|
+
SIMULATION_REVERTED,
|
|
34
|
+
TX_BROADCAST,
|
|
35
|
+
TX_FAILED,
|
|
36
|
+
INTENT_RETRY_ATTEMPTS,
|
|
37
|
+
get_metrics,
|
|
38
|
+
)
|
|
39
|
+
from brawny.model.enums import AttemptStatus, IntentStatus
|
|
40
|
+
from brawny.model.errors import (
|
|
41
|
+
DatabaseError,
|
|
42
|
+
FailureStage,
|
|
43
|
+
FailureType,
|
|
44
|
+
SimulationNetworkError,
|
|
45
|
+
SimulationReverted,
|
|
46
|
+
)
|
|
47
|
+
from brawny.model.types import GasParams, TxAttempt, TxIntent
|
|
48
|
+
from brawny._rpc.context import set_job_context as set_rpc_job_context, reset_job_context as reset_rpc_job_context
|
|
49
|
+
from brawny._rpc.errors import RPCError
|
|
50
|
+
from brawny.tx.nonce import NonceManager
|
|
51
|
+
from brawny.tx.intent import transition_intent
|
|
52
|
+
from brawny.utils import ensure_utc, utc_now
|
|
53
|
+
|
|
54
|
+
if TYPE_CHECKING:
|
|
55
|
+
from brawny.config import Config
|
|
56
|
+
from brawny.db.base import Database
|
|
57
|
+
from brawny.jobs.base import Job
|
|
58
|
+
from brawny.keystore import Keystore
|
|
59
|
+
from brawny.lifecycle import LifecycleDispatcher
|
|
60
|
+
from brawny._rpc.manager import RPCManager
|
|
61
|
+
|
|
62
|
+
logger = get_logger(__name__)
|
|
63
|
+
|
|
64
|
+
# Simulation retry settings
|
|
65
|
+
MAX_SIMULATION_RETRIES = 2 # Total attempts = 3 (1 initial + 2 retries)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class ExecutionResult(str, Enum):
|
|
69
|
+
"""Result of transaction execution."""
|
|
70
|
+
|
|
71
|
+
PENDING = "pending"
|
|
72
|
+
CONFIRMED = "confirmed"
|
|
73
|
+
REVERTED = "reverted"
|
|
74
|
+
DROPPED = "dropped"
|
|
75
|
+
STUCK = "stuck"
|
|
76
|
+
DEADLINE_EXPIRED = "deadline_expired"
|
|
77
|
+
FAILED = "failed"
|
|
78
|
+
BLOCKED = "blocked" # Signer blocked by nonce gap
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class ExecutionOutcome:
|
|
83
|
+
"""Outcome of executing an intent."""
|
|
84
|
+
|
|
85
|
+
result: ExecutionResult
|
|
86
|
+
intent: TxIntent
|
|
87
|
+
attempt: TxAttempt | None
|
|
88
|
+
tx_hash: str | None = None
|
|
89
|
+
error: Exception | None = None
|
|
90
|
+
block_number: int | None = None
|
|
91
|
+
confirmations: int = 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class TxExecutor:
|
|
95
|
+
"""Transaction executor with full lifecycle management.
|
|
96
|
+
|
|
97
|
+
Handles:
|
|
98
|
+
- Gas estimation (EIP-1559)
|
|
99
|
+
- Nonce reservation via NonceManager
|
|
100
|
+
- Transaction signing via Keystore
|
|
101
|
+
- Broadcasting with retry
|
|
102
|
+
- Confirmation monitoring
|
|
103
|
+
- Stuck tx detection and replacement
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
def __init__(
|
|
107
|
+
self,
|
|
108
|
+
db: Database,
|
|
109
|
+
rpc: RPCManager,
|
|
110
|
+
keystore: Keystore,
|
|
111
|
+
config: Config,
|
|
112
|
+
lifecycle: "LifecycleDispatcher | None" = None,
|
|
113
|
+
jobs: dict[str, "Job"] | None = None,
|
|
114
|
+
) -> None:
|
|
115
|
+
"""Initialize transaction executor.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
db: Database connection
|
|
119
|
+
rpc: RPC manager for chain operations
|
|
120
|
+
keystore: Keystore for transaction signing
|
|
121
|
+
config: Application configuration
|
|
122
|
+
lifecycle: Optional lifecycle dispatcher for events
|
|
123
|
+
jobs: Optional jobs registry for simulation hooks
|
|
124
|
+
"""
|
|
125
|
+
self._db = db
|
|
126
|
+
self._rpc = rpc
|
|
127
|
+
self._keystore = keystore
|
|
128
|
+
self._config = config
|
|
129
|
+
self._nonce_manager = NonceManager(db, rpc, config.chain_id)
|
|
130
|
+
self._lifecycle = lifecycle
|
|
131
|
+
self._jobs = jobs
|
|
132
|
+
self._chain_id = config.chain_id
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def nonce_manager(self) -> NonceManager:
|
|
136
|
+
"""Get the nonce manager."""
|
|
137
|
+
return self._nonce_manager
|
|
138
|
+
|
|
139
|
+
# =========================================================================
|
|
140
|
+
# Nonce Gap Detection (Pre-flight check)
|
|
141
|
+
# =========================================================================
|
|
142
|
+
|
|
143
|
+
def _check_nonce_gap(
|
|
144
|
+
self, signer_address: str
|
|
145
|
+
) -> tuple[bool, int | None, float | None]:
|
|
146
|
+
"""Check if signer is blocked by a nonce gap.
|
|
147
|
+
|
|
148
|
+
Returns (is_blocked, oldest_in_flight_nonce, oldest_age_seconds).
|
|
149
|
+
|
|
150
|
+
Checks both RESERVED and IN_FLIGHT records - a gap can exist with either.
|
|
151
|
+
"""
|
|
152
|
+
from brawny.model.enums import NonceStatus
|
|
153
|
+
|
|
154
|
+
chain_pending = self._rpc.get_transaction_count(signer_address, "pending")
|
|
155
|
+
|
|
156
|
+
# Get all active reservations (RESERVED or IN_FLIGHT)
|
|
157
|
+
active = self._nonce_manager.get_active_reservations(signer_address)
|
|
158
|
+
|
|
159
|
+
if not active:
|
|
160
|
+
# No reservations = no gap possible
|
|
161
|
+
self._clear_gap_tracking(signer_address)
|
|
162
|
+
return False, None, None
|
|
163
|
+
|
|
164
|
+
# Find the lowest nonce we're tracking
|
|
165
|
+
expected_next = min(r.nonce for r in active)
|
|
166
|
+
|
|
167
|
+
if chain_pending >= expected_next:
|
|
168
|
+
# No gap - chain has caught up or is ahead
|
|
169
|
+
self._clear_gap_tracking(signer_address)
|
|
170
|
+
return False, None, None
|
|
171
|
+
|
|
172
|
+
# Gap exists: chain_pending < expected_next
|
|
173
|
+
# Find oldest IN_FLIGHT for TxReplacer visibility
|
|
174
|
+
from brawny.model.enums import NonceStatus
|
|
175
|
+
in_flight = [r for r in active if r.status == NonceStatus.IN_FLIGHT]
|
|
176
|
+
oldest_nonce = None
|
|
177
|
+
oldest_age = None
|
|
178
|
+
|
|
179
|
+
if in_flight:
|
|
180
|
+
oldest = min(in_flight, key=lambda r: r.nonce)
|
|
181
|
+
oldest_nonce = oldest.nonce
|
|
182
|
+
oldest_age = (utc_now() - ensure_utc(oldest.created_at)).total_seconds()
|
|
183
|
+
|
|
184
|
+
return True, oldest_nonce, oldest_age
|
|
185
|
+
|
|
186
|
+
def _get_gap_duration(self, signer_address: str) -> float:
|
|
187
|
+
"""Get how long this signer has been blocked by a nonce gap (persisted in DB)."""
|
|
188
|
+
signer_state = self._db.get_signer_state(self._chain_id, signer_address.lower())
|
|
189
|
+
|
|
190
|
+
if signer_state is None:
|
|
191
|
+
return 0.0
|
|
192
|
+
|
|
193
|
+
if signer_state.gap_started_at is None:
|
|
194
|
+
# First time seeing gap - record it
|
|
195
|
+
self._db.set_gap_started_at(self._chain_id, signer_address.lower(), utc_now())
|
|
196
|
+
return 0.0
|
|
197
|
+
|
|
198
|
+
return (utc_now() - ensure_utc(signer_state.gap_started_at)).total_seconds()
|
|
199
|
+
|
|
200
|
+
def _clear_gap_tracking(self, signer_address: str) -> None:
|
|
201
|
+
"""Clear gap tracking when gap is resolved or force_reset runs."""
|
|
202
|
+
self._db.clear_gap_started_at(self._chain_id, signer_address.lower())
|
|
203
|
+
|
|
204
|
+
def _alert_nonce_gap(
|
|
205
|
+
self,
|
|
206
|
+
signer_address: str,
|
|
207
|
+
duration: float,
|
|
208
|
+
oldest_nonce: int | None,
|
|
209
|
+
oldest_age: float | None,
|
|
210
|
+
) -> None:
|
|
211
|
+
"""Alert on prolonged nonce gap (rate-limited per signer)."""
|
|
212
|
+
if not self._lifecycle:
|
|
213
|
+
return
|
|
214
|
+
|
|
215
|
+
context = f"Signer {signer_address} blocked for {duration:.0f}s."
|
|
216
|
+
if oldest_nonce is not None and oldest_age is not None:
|
|
217
|
+
context += f" Oldest IN_FLIGHT: nonce {oldest_nonce} ({oldest_age:.0f}s old)."
|
|
218
|
+
context += f" TxReplacer should recover, or run: brawny signer force-reset {signer_address}"
|
|
219
|
+
|
|
220
|
+
self._lifecycle.alert(
|
|
221
|
+
level="warning",
|
|
222
|
+
title=f"Nonce gap blocking signer {signer_address[:10]}...",
|
|
223
|
+
message=context,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
def estimate_gas(
|
|
227
|
+
self,
|
|
228
|
+
intent: TxIntent,
|
|
229
|
+
signer_address: str | None = None,
|
|
230
|
+
to_address: str | None = None,
|
|
231
|
+
job: "Job | None" = None,
|
|
232
|
+
) -> GasParams:
|
|
233
|
+
"""Estimate gas for a transaction intent.
|
|
234
|
+
|
|
235
|
+
Uses EIP-1559 gas pricing with cached gas quotes.
|
|
236
|
+
|
|
237
|
+
Args:
|
|
238
|
+
intent: Transaction intent
|
|
239
|
+
signer_address: Resolved signer address (optional, uses intent if not provided)
|
|
240
|
+
to_address: Resolved to address (optional, uses intent if not provided)
|
|
241
|
+
job: Job instance for gas overrides (optional)
|
|
242
|
+
|
|
243
|
+
Returns:
|
|
244
|
+
Estimated gas parameters
|
|
245
|
+
|
|
246
|
+
Raises:
|
|
247
|
+
RetriableExecutionError: If no cached gas quote available
|
|
248
|
+
"""
|
|
249
|
+
from brawny.model.errors import RetriableExecutionError
|
|
250
|
+
|
|
251
|
+
# Use resolved addresses if provided, otherwise fall back to intent
|
|
252
|
+
from_addr = signer_address or intent.signer_address
|
|
253
|
+
to_addr = to_address or intent.to_address
|
|
254
|
+
|
|
255
|
+
# Gas limit
|
|
256
|
+
if intent.gas_limit:
|
|
257
|
+
gas_limit = intent.gas_limit
|
|
258
|
+
else:
|
|
259
|
+
try:
|
|
260
|
+
tx_params = {
|
|
261
|
+
"from": from_addr,
|
|
262
|
+
"to": to_addr,
|
|
263
|
+
"value": int(intent.value_wei),
|
|
264
|
+
}
|
|
265
|
+
if intent.data:
|
|
266
|
+
tx_params["data"] = intent.data
|
|
267
|
+
|
|
268
|
+
estimated = self._rpc.estimate_gas(tx_params)
|
|
269
|
+
gas_limit = int(estimated * self._config.gas_limit_multiplier)
|
|
270
|
+
except Exception as e:
|
|
271
|
+
logger.warning(
|
|
272
|
+
"gas.estimate_failed",
|
|
273
|
+
intent_id=str(intent.intent_id),
|
|
274
|
+
error=str(e),
|
|
275
|
+
)
|
|
276
|
+
gas_limit = self._config.fallback_gas_limit
|
|
277
|
+
|
|
278
|
+
# Resolve effective priority_fee (priority: intent > job > config)
|
|
279
|
+
if intent.max_priority_fee_per_gas:
|
|
280
|
+
priority_fee = int(intent.max_priority_fee_per_gas)
|
|
281
|
+
elif job is not None and job.priority_fee is not None:
|
|
282
|
+
priority_fee = int(job.priority_fee)
|
|
283
|
+
else:
|
|
284
|
+
priority_fee = int(self._config.priority_fee)
|
|
285
|
+
|
|
286
|
+
# Gas price (EIP-1559)
|
|
287
|
+
if intent.max_fee_per_gas:
|
|
288
|
+
# Explicit in intent - use directly
|
|
289
|
+
max_fee = int(intent.max_fee_per_gas)
|
|
290
|
+
else:
|
|
291
|
+
# Compute from quote (sync cache only)
|
|
292
|
+
quote = self._rpc.gas_quote_sync()
|
|
293
|
+
|
|
294
|
+
if quote is None:
|
|
295
|
+
# No cached quote - raise retriable error (don't guess)
|
|
296
|
+
# This should rarely happen (gas_ok warms cache)
|
|
297
|
+
# NOTE: Executor must handle RetriableExecutionError with backoff,
|
|
298
|
+
# not tight-loop retry. See intent_retry_backoff_seconds config.
|
|
299
|
+
raise RetriableExecutionError("No gas quote available, will retry")
|
|
300
|
+
|
|
301
|
+
computed_max_fee = int((2 * quote.base_fee) + priority_fee)
|
|
302
|
+
|
|
303
|
+
# Apply cap if configured
|
|
304
|
+
effective_max_fee = job.max_fee if job and job.max_fee is not None else self._config.max_fee
|
|
305
|
+
|
|
306
|
+
if effective_max_fee is not None:
|
|
307
|
+
max_fee = min(int(effective_max_fee), computed_max_fee)
|
|
308
|
+
else:
|
|
309
|
+
max_fee = computed_max_fee
|
|
310
|
+
|
|
311
|
+
return GasParams(
|
|
312
|
+
gas_limit=gas_limit,
|
|
313
|
+
max_fee_per_gas=max_fee,
|
|
314
|
+
max_priority_fee_per_gas=priority_fee,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
def calculate_replacement_fees(self, old_params: GasParams) -> GasParams:
|
|
318
|
+
"""Calculate bumped fees for replacement transaction.
|
|
319
|
+
|
|
320
|
+
Per Ethereum protocol, replacement must have at least 10% higher fees.
|
|
321
|
+
Uses configured fee_bump_percent (default 15%).
|
|
322
|
+
|
|
323
|
+
Args:
|
|
324
|
+
old_params: Previous gas parameters
|
|
325
|
+
|
|
326
|
+
Returns:
|
|
327
|
+
New gas parameters with bumped fees
|
|
328
|
+
"""
|
|
329
|
+
from brawny.tx.fees import bump_fees
|
|
330
|
+
|
|
331
|
+
return bump_fees(
|
|
332
|
+
old_params,
|
|
333
|
+
bump_percent=self._config.fee_bump_percent,
|
|
334
|
+
max_fee_cap=self._config.max_fee,
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
def execute(self, intent: TxIntent) -> ExecutionOutcome:
|
|
338
|
+
"""Execute a transaction intent.
|
|
339
|
+
|
|
340
|
+
Full execution flow:
|
|
341
|
+
1. Validate deadline
|
|
342
|
+
2. Reserve nonce
|
|
343
|
+
3. Estimate gas
|
|
344
|
+
4. Build tx dict
|
|
345
|
+
5. Simulate (unless job opts out)
|
|
346
|
+
6. Sign transaction
|
|
347
|
+
7. Broadcast
|
|
348
|
+
|
|
349
|
+
Args:
|
|
350
|
+
intent: Transaction intent to execute
|
|
351
|
+
|
|
352
|
+
Returns:
|
|
353
|
+
Execution outcome with result and details
|
|
354
|
+
"""
|
|
355
|
+
def _retry_intent(reason: str) -> None:
|
|
356
|
+
"""Reset intent to created with exponential backoff, or abandon if max retries exceeded."""
|
|
357
|
+
metrics = get_metrics()
|
|
358
|
+
metrics.counter(INTENT_RETRY_ATTEMPTS).inc(
|
|
359
|
+
chain_id=self._chain_id,
|
|
360
|
+
reason=reason,
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
# Atomically increment retry count on intent row
|
|
364
|
+
retry_count = self._db.increment_intent_retry_count(intent.intent_id)
|
|
365
|
+
|
|
366
|
+
# Check if max retries exceeded
|
|
367
|
+
if retry_count > self._config.max_executor_retries:
|
|
368
|
+
logger.warning(
|
|
369
|
+
"intent.max_retries_exceeded",
|
|
370
|
+
intent_id=str(intent.intent_id),
|
|
371
|
+
retry_count=retry_count,
|
|
372
|
+
max_retries=self._config.max_executor_retries,
|
|
373
|
+
reason=reason,
|
|
374
|
+
)
|
|
375
|
+
transition_intent(
|
|
376
|
+
self._db,
|
|
377
|
+
intent.intent_id,
|
|
378
|
+
IntentStatus.ABANDONED,
|
|
379
|
+
"max_retries_exceeded",
|
|
380
|
+
chain_id=self._chain_id,
|
|
381
|
+
)
|
|
382
|
+
if self._lifecycle:
|
|
383
|
+
self._lifecycle.on_failed(
|
|
384
|
+
intent, None,
|
|
385
|
+
RuntimeError(f"Max executor retries ({self._config.max_executor_retries}) exceeded"),
|
|
386
|
+
failure_type=FailureType.UNKNOWN,
|
|
387
|
+
failure_stage=FailureStage.PRE_BROADCAST,
|
|
388
|
+
)
|
|
389
|
+
return
|
|
390
|
+
|
|
391
|
+
# Calculate exponential backoff with jitter
|
|
392
|
+
if self._config.intent_retry_backoff_seconds > 0:
|
|
393
|
+
base_backoff = self._config.intent_retry_backoff_seconds * (2 ** (retry_count - 1))
|
|
394
|
+
jitter = random.uniform(0, min(base_backoff * 0.1, 10)) # 10% jitter, max 10s
|
|
395
|
+
backoff_seconds = min(base_backoff + jitter, 300) # Cap at 5 minutes
|
|
396
|
+
retry_after = datetime.now(timezone.utc) + timedelta(seconds=backoff_seconds)
|
|
397
|
+
self._db.set_intent_retry_after(intent.intent_id, retry_after)
|
|
398
|
+
logger.info(
|
|
399
|
+
"intent.retry_scheduled",
|
|
400
|
+
intent_id=str(intent.intent_id),
|
|
401
|
+
job_id=intent.job_id,
|
|
402
|
+
retry_count=retry_count,
|
|
403
|
+
backoff_seconds=round(backoff_seconds, 1),
|
|
404
|
+
retry_after=retry_after.isoformat(),
|
|
405
|
+
reason=reason,
|
|
406
|
+
)
|
|
407
|
+
else:
|
|
408
|
+
logger.info(
|
|
409
|
+
"intent.retry_scheduled",
|
|
410
|
+
intent_id=str(intent.intent_id),
|
|
411
|
+
job_id=intent.job_id,
|
|
412
|
+
retry_count=retry_count,
|
|
413
|
+
retry_after=None,
|
|
414
|
+
reason=reason,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
# Transition to CREATED (auto-clears claim via transition_intent)
|
|
418
|
+
if not transition_intent(
|
|
419
|
+
self._db,
|
|
420
|
+
intent.intent_id,
|
|
421
|
+
IntentStatus.CREATED,
|
|
422
|
+
reason,
|
|
423
|
+
chain_id=self._chain_id,
|
|
424
|
+
):
|
|
425
|
+
logger.warning(
|
|
426
|
+
"intent.retry_reset_failed",
|
|
427
|
+
intent_id=str(intent.intent_id),
|
|
428
|
+
reason=reason,
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
# Set RPC job context for metrics attribution
|
|
432
|
+
rpc_ctx_token = set_rpc_job_context(intent.job_id)
|
|
433
|
+
try:
|
|
434
|
+
return self._execute_with_context(intent, _retry_intent)
|
|
435
|
+
finally:
|
|
436
|
+
reset_rpc_job_context(rpc_ctx_token)
|
|
437
|
+
|
|
438
|
+
def _execute_with_context(
|
|
439
|
+
self,
|
|
440
|
+
intent: TxIntent,
|
|
441
|
+
_retry_intent: Callable[[str], None],
|
|
442
|
+
) -> ExecutionOutcome:
|
|
443
|
+
"""Execute intent with RPC context already set (internal)."""
|
|
444
|
+
# 0. Resolve signer alias to actual checksum address
|
|
445
|
+
try:
|
|
446
|
+
signer_address = self._keystore.get_address(intent.signer_address)
|
|
447
|
+
except Exception as e:
|
|
448
|
+
logger.error(
|
|
449
|
+
"signer.resolution_failed",
|
|
450
|
+
intent_id=str(intent.intent_id),
|
|
451
|
+
signer=intent.signer_address,
|
|
452
|
+
error=str(e),
|
|
453
|
+
)
|
|
454
|
+
if self._lifecycle:
|
|
455
|
+
self._lifecycle.on_failed(
|
|
456
|
+
intent, None, e,
|
|
457
|
+
failure_type=FailureType.SIGNER_FAILED,
|
|
458
|
+
failure_stage=FailureStage.PRE_BROADCAST,
|
|
459
|
+
cleanup_trigger=False,
|
|
460
|
+
)
|
|
461
|
+
_retry_intent("signer_resolution_failed")
|
|
462
|
+
return ExecutionOutcome(
|
|
463
|
+
result=ExecutionResult.FAILED,
|
|
464
|
+
intent=intent,
|
|
465
|
+
attempt=None,
|
|
466
|
+
error=e,
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
# Update intent with resolved signer address (so monitor can use it)
|
|
470
|
+
if signer_address.lower() != intent.signer_address.lower():
|
|
471
|
+
self._db.update_intent_signer(intent.intent_id, signer_address)
|
|
472
|
+
logger.debug(
|
|
473
|
+
"signer.resolved",
|
|
474
|
+
intent_id=str(intent.intent_id),
|
|
475
|
+
alias=intent.signer_address,
|
|
476
|
+
address=signer_address,
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
# Ensure to_address is checksummed
|
|
480
|
+
to_address = Web3.to_checksum_address(intent.to_address)
|
|
481
|
+
|
|
482
|
+
# 1. Validate deadline
|
|
483
|
+
if intent.deadline_ts:
|
|
484
|
+
if datetime.now(timezone.utc) > intent.deadline_ts:
|
|
485
|
+
transition_intent(
|
|
486
|
+
self._db,
|
|
487
|
+
intent.intent_id,
|
|
488
|
+
IntentStatus.ABANDONED,
|
|
489
|
+
"deadline_expired",
|
|
490
|
+
chain_id=self._chain_id,
|
|
491
|
+
)
|
|
492
|
+
if self._lifecycle:
|
|
493
|
+
self._lifecycle.on_failed(
|
|
494
|
+
intent,
|
|
495
|
+
None,
|
|
496
|
+
TimeoutError("Intent deadline expired"),
|
|
497
|
+
failure_type=FailureType.DEADLINE_EXPIRED,
|
|
498
|
+
failure_stage=FailureStage.PRE_BROADCAST,
|
|
499
|
+
)
|
|
500
|
+
return ExecutionOutcome(
|
|
501
|
+
result=ExecutionResult.DEADLINE_EXPIRED,
|
|
502
|
+
intent=intent,
|
|
503
|
+
attempt=None,
|
|
504
|
+
error=TimeoutError("Intent deadline expired"),
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
# 1.5 Pre-flight gap check - don't reserve if signer is blocked
|
|
508
|
+
try:
|
|
509
|
+
is_blocked, oldest_nonce, oldest_age = self._check_nonce_gap(signer_address)
|
|
510
|
+
except Exception as e:
|
|
511
|
+
# Fail-safe: if we cannot validate nonce-gap safety, do NOT proceed
|
|
512
|
+
logger.warning(
|
|
513
|
+
"nonce.gap_check_failed",
|
|
514
|
+
intent_id=str(intent.intent_id),
|
|
515
|
+
signer=signer_address,
|
|
516
|
+
error=str(e)[:100],
|
|
517
|
+
)
|
|
518
|
+
_retry_intent("nonce_gap_check_failed")
|
|
519
|
+
return ExecutionOutcome(
|
|
520
|
+
result=ExecutionResult.FAILED,
|
|
521
|
+
intent=intent,
|
|
522
|
+
attempt=None,
|
|
523
|
+
error=e,
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
if is_blocked:
|
|
527
|
+
gap_duration = self._get_gap_duration(signer_address)
|
|
528
|
+
|
|
529
|
+
logger.warning(
|
|
530
|
+
"nonce.gap_blocked",
|
|
531
|
+
intent_id=str(intent.intent_id),
|
|
532
|
+
job_id=intent.job_id,
|
|
533
|
+
signer=signer_address,
|
|
534
|
+
blocked_duration_seconds=gap_duration,
|
|
535
|
+
oldest_in_flight_nonce=oldest_nonce,
|
|
536
|
+
oldest_in_flight_age_seconds=oldest_age,
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
# Check config for unsafe reset mode
|
|
540
|
+
if self._config.allow_unsafe_nonce_reset:
|
|
541
|
+
logger.warning("nonce.unsafe_reset_triggered", signer=signer_address)
|
|
542
|
+
self._nonce_manager.reconcile(signer_address)
|
|
543
|
+
self._clear_gap_tracking(signer_address)
|
|
544
|
+
# Fall through to normal execution
|
|
545
|
+
else:
|
|
546
|
+
# Alert if blocked too long
|
|
547
|
+
if gap_duration > self._config.nonce_gap_alert_seconds:
|
|
548
|
+
self._alert_nonce_gap(signer_address, gap_duration, oldest_nonce, oldest_age)
|
|
549
|
+
|
|
550
|
+
# Return BLOCKED - don't reserve, don't retry immediately
|
|
551
|
+
# Let TxReplacer handle recovery via fee bumping
|
|
552
|
+
return ExecutionOutcome(
|
|
553
|
+
result=ExecutionResult.BLOCKED,
|
|
554
|
+
intent=intent,
|
|
555
|
+
attempt=None,
|
|
556
|
+
error=RuntimeError(
|
|
557
|
+
f"Nonce gap detected for {signer_address}, waiting for TxReplacer"
|
|
558
|
+
),
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
# 2. Reserve nonce
|
|
562
|
+
try:
|
|
563
|
+
nonce = self._nonce_manager.reserve_nonce(
|
|
564
|
+
signer_address,
|
|
565
|
+
intent_id=intent.intent_id,
|
|
566
|
+
)
|
|
567
|
+
except Exception as e:
|
|
568
|
+
logger.error(
|
|
569
|
+
"nonce.reservation_failed",
|
|
570
|
+
intent_id=str(intent.intent_id),
|
|
571
|
+
signer=signer_address,
|
|
572
|
+
error=str(e),
|
|
573
|
+
)
|
|
574
|
+
if self._lifecycle:
|
|
575
|
+
self._lifecycle.on_failed(
|
|
576
|
+
intent, None, e,
|
|
577
|
+
failure_type=FailureType.NONCE_FAILED,
|
|
578
|
+
failure_stage=FailureStage.PRE_BROADCAST,
|
|
579
|
+
cleanup_trigger=False,
|
|
580
|
+
)
|
|
581
|
+
_retry_intent("nonce_reservation_failed")
|
|
582
|
+
return ExecutionOutcome(
|
|
583
|
+
result=ExecutionResult.FAILED,
|
|
584
|
+
intent=intent,
|
|
585
|
+
attempt=None,
|
|
586
|
+
error=e,
|
|
587
|
+
)
|
|
588
|
+
|
|
589
|
+
# NOTE: Gap detection moved to pre-flight check (step 1.5)
|
|
590
|
+
# The pre-flight check returns BLOCKED if there's a nonce gap,
|
|
591
|
+
# allowing TxReplacer to handle recovery instead of auto-abandoning.
|
|
592
|
+
|
|
593
|
+
# 3. Estimate gas
|
|
594
|
+
job = self._jobs.get(intent.job_id) if self._jobs else None
|
|
595
|
+
try:
|
|
596
|
+
gas_params = self.estimate_gas(intent, signer_address, to_address, job=job)
|
|
597
|
+
except Exception as e:
|
|
598
|
+
if "RetriableExecutionError" in type(e).__name__ or "No gas quote" in str(e):
|
|
599
|
+
logger.warning(
|
|
600
|
+
"gas.no_quote_available",
|
|
601
|
+
intent_id=str(intent.intent_id),
|
|
602
|
+
job_id=intent.job_id,
|
|
603
|
+
error=str(e),
|
|
604
|
+
)
|
|
605
|
+
# Release nonce before retry
|
|
606
|
+
self._nonce_manager.release(signer_address, nonce)
|
|
607
|
+
_retry_intent("no_gas_quote")
|
|
608
|
+
return ExecutionOutcome(
|
|
609
|
+
result=ExecutionResult.FAILED,
|
|
610
|
+
intent=intent,
|
|
611
|
+
attempt=None,
|
|
612
|
+
error=e,
|
|
613
|
+
)
|
|
614
|
+
raise
|
|
615
|
+
|
|
616
|
+
# 4. Build tx dict for simulation
|
|
617
|
+
tx_dict = self._build_tx_dict(intent, nonce, gas_params, to_address)
|
|
618
|
+
tx_dict["from"] = signer_address # Required for simulation
|
|
619
|
+
|
|
620
|
+
# 5. Simulation step (runs unless job opts out)
|
|
621
|
+
if job and not getattr(job, "disable_simulation", False):
|
|
622
|
+
try:
|
|
623
|
+
self._simulate_with_retry(job, intent, tx_dict)
|
|
624
|
+
except (SimulationReverted, SimulationNetworkError) as e:
|
|
625
|
+
# Release nonce on simulation failure
|
|
626
|
+
self._nonce_manager.release(signer_address, nonce)
|
|
627
|
+
return self._handle_simulation_failure(job, intent, e)
|
|
628
|
+
|
|
629
|
+
# 6. Sign transaction (only if simulation passed)
|
|
630
|
+
try:
|
|
631
|
+
signed_tx = self._keystore.sign_transaction(
|
|
632
|
+
tx_dict,
|
|
633
|
+
signer_address,
|
|
634
|
+
)
|
|
635
|
+
except Exception as e:
|
|
636
|
+
logger.error(
|
|
637
|
+
"tx.sign_failed",
|
|
638
|
+
intent_id=str(intent.intent_id),
|
|
639
|
+
job_id=intent.job_id,
|
|
640
|
+
error=str(e),
|
|
641
|
+
)
|
|
642
|
+
# Release nonce on sign failure
|
|
643
|
+
self._nonce_manager.release(signer_address, nonce)
|
|
644
|
+
if self._lifecycle:
|
|
645
|
+
self._lifecycle.on_failed(
|
|
646
|
+
intent, None, e,
|
|
647
|
+
failure_type=FailureType.SIGN_FAILED,
|
|
648
|
+
failure_stage=FailureStage.PRE_BROADCAST,
|
|
649
|
+
cleanup_trigger=False,
|
|
650
|
+
)
|
|
651
|
+
_retry_intent("sign_failed")
|
|
652
|
+
return ExecutionOutcome(
|
|
653
|
+
result=ExecutionResult.FAILED,
|
|
654
|
+
intent=intent,
|
|
655
|
+
attempt=None,
|
|
656
|
+
error=e,
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
# Warn if priority fee is suspiciously low (< 0.1 gwei)
|
|
660
|
+
if gas_params.max_priority_fee_per_gas < 100_000_000:
|
|
661
|
+
logger.warning(
|
|
662
|
+
"gas.priority_fee_very_low",
|
|
663
|
+
intent_id=str(intent.intent_id),
|
|
664
|
+
job_id=intent.job_id,
|
|
665
|
+
priority_fee_wei=gas_params.max_priority_fee_per_gas,
|
|
666
|
+
priority_fee_gwei=gas_params.max_priority_fee_per_gas / 1e9,
|
|
667
|
+
hint="Transaction may not be included - validators receive almost no tip",
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
logger.info(
|
|
671
|
+
LogEvents.TX_SIGN,
|
|
672
|
+
intent_id=str(intent.intent_id),
|
|
673
|
+
job_id=intent.job_id,
|
|
674
|
+
signer=signer_address,
|
|
675
|
+
nonce=nonce,
|
|
676
|
+
gas_limit=gas_params.gas_limit,
|
|
677
|
+
max_fee=gas_params.max_fee_per_gas,
|
|
678
|
+
priority_fee=gas_params.max_priority_fee_per_gas,
|
|
679
|
+
)
|
|
680
|
+
|
|
681
|
+
# 7. Broadcast with RPC group routing
|
|
682
|
+
attempt: TxAttempt | None = None
|
|
683
|
+
attempt_id = uuid4()
|
|
684
|
+
tx_hash: str | None = None
|
|
685
|
+
endpoint_url: str | None = None
|
|
686
|
+
|
|
687
|
+
try:
|
|
688
|
+
# Update intent status to sending
|
|
689
|
+
if not transition_intent(
|
|
690
|
+
self._db,
|
|
691
|
+
intent.intent_id,
|
|
692
|
+
IntentStatus.SENDING,
|
|
693
|
+
"broadcast_start",
|
|
694
|
+
chain_id=self._chain_id,
|
|
695
|
+
):
|
|
696
|
+
raise RuntimeError("Intent status not claimable for sending")
|
|
697
|
+
|
|
698
|
+
# Check for existing binding (for retry isolation)
|
|
699
|
+
binding = self._db.get_broadcast_binding(intent.intent_id)
|
|
700
|
+
job_id = job.job_id if job else None
|
|
701
|
+
|
|
702
|
+
if binding is not None:
|
|
703
|
+
# RETRY: Use persisted endpoints (NEVER current config)
|
|
704
|
+
group_name, endpoints = binding
|
|
705
|
+
is_first_broadcast = False
|
|
706
|
+
|
|
707
|
+
# Advisory log if job's config changed
|
|
708
|
+
if job:
|
|
709
|
+
from brawny.config.routing import resolve_job_groups
|
|
710
|
+
|
|
711
|
+
_, job_broadcast_group = resolve_job_groups(self._config, job)
|
|
712
|
+
if job_broadcast_group != group_name:
|
|
713
|
+
logger.warning(
|
|
714
|
+
"broadcast_group_mismatch",
|
|
715
|
+
intent_id=str(intent.intent_id),
|
|
716
|
+
job_id=job_id,
|
|
717
|
+
persisted_group=group_name,
|
|
718
|
+
current_job_group=job_broadcast_group,
|
|
719
|
+
)
|
|
720
|
+
else:
|
|
721
|
+
# FIRST BROADCAST: Resolve group + endpoints from config (no silent fallback)
|
|
722
|
+
if job is None:
|
|
723
|
+
from brawny.config.routing import resolve_default_group
|
|
724
|
+
|
|
725
|
+
group_name = resolve_default_group(self._config)
|
|
726
|
+
else:
|
|
727
|
+
from brawny.config.routing import resolve_job_groups
|
|
728
|
+
|
|
729
|
+
_, group_name = resolve_job_groups(self._config, job)
|
|
730
|
+
endpoints = self._config.rpc_groups[group_name].endpoints
|
|
731
|
+
|
|
732
|
+
is_first_broadcast = True
|
|
733
|
+
|
|
734
|
+
# Broadcast transaction using RPC groups
|
|
735
|
+
from brawny._rpc.broadcast import broadcast_transaction
|
|
736
|
+
from brawny._rpc.errors import RPCGroupUnavailableError
|
|
737
|
+
|
|
738
|
+
try:
|
|
739
|
+
tx_hash, endpoint_url = broadcast_transaction(
|
|
740
|
+
raw_tx=signed_tx.raw_transaction,
|
|
741
|
+
endpoints=endpoints,
|
|
742
|
+
group_name=group_name,
|
|
743
|
+
config=self._config,
|
|
744
|
+
job_id=job_id,
|
|
745
|
+
)
|
|
746
|
+
except RPCGroupUnavailableError as e:
|
|
747
|
+
logger.error(
|
|
748
|
+
"broadcast_unavailable",
|
|
749
|
+
intent_id=str(intent.intent_id),
|
|
750
|
+
job_id=job_id,
|
|
751
|
+
broadcast_group=group_name,
|
|
752
|
+
endpoints=endpoints,
|
|
753
|
+
error=str(e.last_error) if e.last_error else None,
|
|
754
|
+
)
|
|
755
|
+
raise
|
|
756
|
+
|
|
757
|
+
# Create attempt record (+ binding if first broadcast)
|
|
758
|
+
current_block = self._rpc.get_block_number()
|
|
759
|
+
attempt = self._db.create_attempt(
|
|
760
|
+
attempt_id=attempt_id,
|
|
761
|
+
intent_id=intent.intent_id,
|
|
762
|
+
nonce=nonce,
|
|
763
|
+
gas_params_json=gas_params.to_json(),
|
|
764
|
+
status=AttemptStatus.BROADCAST.value,
|
|
765
|
+
tx_hash=tx_hash,
|
|
766
|
+
broadcast_group=group_name,
|
|
767
|
+
endpoint_url=endpoint_url,
|
|
768
|
+
binding=(group_name, endpoints) if is_first_broadcast else None,
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
# Update attempt with broadcast block and time
|
|
772
|
+
self._db.update_attempt_status(
|
|
773
|
+
attempt_id,
|
|
774
|
+
AttemptStatus.BROADCAST.value,
|
|
775
|
+
broadcast_block=current_block,
|
|
776
|
+
broadcast_at=datetime.now(timezone.utc),
|
|
777
|
+
)
|
|
778
|
+
|
|
779
|
+
# Mark nonce as in-flight
|
|
780
|
+
self._nonce_manager.mark_in_flight(signer_address, nonce, intent.intent_id)
|
|
781
|
+
|
|
782
|
+
# Update intent to pending
|
|
783
|
+
if not transition_intent(
|
|
784
|
+
self._db,
|
|
785
|
+
intent.intent_id,
|
|
786
|
+
IntentStatus.PENDING,
|
|
787
|
+
"broadcast_complete",
|
|
788
|
+
chain_id=self._chain_id,
|
|
789
|
+
):
|
|
790
|
+
raise RuntimeError("Intent status not in sending state")
|
|
791
|
+
|
|
792
|
+
logger.info(
|
|
793
|
+
LogEvents.TX_BROADCAST,
|
|
794
|
+
intent_id=str(intent.intent_id),
|
|
795
|
+
job_id=intent.job_id,
|
|
796
|
+
attempt_id=str(attempt_id),
|
|
797
|
+
tx_hash=tx_hash,
|
|
798
|
+
signer=signer_address,
|
|
799
|
+
nonce=nonce,
|
|
800
|
+
broadcast_group=group_name,
|
|
801
|
+
endpoint_url=endpoint_url[:50] if endpoint_url else None,
|
|
802
|
+
)
|
|
803
|
+
metrics = get_metrics()
|
|
804
|
+
metrics.counter(TX_BROADCAST).inc(
|
|
805
|
+
chain_id=self._chain_id,
|
|
806
|
+
job_id=intent.job_id,
|
|
807
|
+
)
|
|
808
|
+
|
|
809
|
+
# Refresh attempt
|
|
810
|
+
attempt = self._db.get_attempt(attempt_id)
|
|
811
|
+
if self._lifecycle and attempt is not None:
|
|
812
|
+
self._lifecycle.on_submitted(intent, attempt)
|
|
813
|
+
|
|
814
|
+
except (RPCError, DatabaseError, OSError, ValueError, RuntimeError) as e:
|
|
815
|
+
# Expected broadcast-related errors - handle gracefully
|
|
816
|
+
logger.error(
|
|
817
|
+
"tx.broadcast_failed",
|
|
818
|
+
intent_id=str(intent.intent_id),
|
|
819
|
+
job_id=intent.job_id,
|
|
820
|
+
attempt_id=str(attempt_id),
|
|
821
|
+
error=str(e),
|
|
822
|
+
)
|
|
823
|
+
metrics = get_metrics()
|
|
824
|
+
metrics.counter(TX_FAILED).inc(
|
|
825
|
+
chain_id=self._chain_id,
|
|
826
|
+
job_id=intent.job_id,
|
|
827
|
+
reason="broadcast_failed",
|
|
828
|
+
)
|
|
829
|
+
|
|
830
|
+
# Create failed attempt record if we haven't yet
|
|
831
|
+
if attempt is None:
|
|
832
|
+
try:
|
|
833
|
+
attempt = self._db.create_attempt(
|
|
834
|
+
attempt_id=attempt_id,
|
|
835
|
+
intent_id=intent.intent_id,
|
|
836
|
+
nonce=nonce,
|
|
837
|
+
gas_params_json=gas_params.to_json(),
|
|
838
|
+
status=AttemptStatus.FAILED.value,
|
|
839
|
+
)
|
|
840
|
+
except Exception as attempt_error:
|
|
841
|
+
# Never silently swallow - log with full context for reconstruction
|
|
842
|
+
# exc_info=True captures attempt_error traceback (current exception)
|
|
843
|
+
logger.error(
|
|
844
|
+
"attempt.write_failed",
|
|
845
|
+
intent_id=str(intent.intent_id),
|
|
846
|
+
nonce=nonce,
|
|
847
|
+
tx_hash=tx_hash if "tx_hash" in dir() else None,
|
|
848
|
+
original_error=str(e),
|
|
849
|
+
attempt_error=str(attempt_error),
|
|
850
|
+
attempt_error_type=type(attempt_error).__name__,
|
|
851
|
+
exc_info=True,
|
|
852
|
+
)
|
|
853
|
+
metrics.counter(ATTEMPT_WRITE_FAILURES).inc(stage="broadcast_failure")
|
|
854
|
+
# Continue with cleanup - attempt is None but we have logs
|
|
855
|
+
|
|
856
|
+
if attempt is not None:
|
|
857
|
+
self._db.update_attempt_status(
|
|
858
|
+
attempt_id,
|
|
859
|
+
AttemptStatus.FAILED.value,
|
|
860
|
+
error_code="broadcast_failed",
|
|
861
|
+
error_detail=str(e)[:500],
|
|
862
|
+
)
|
|
863
|
+
|
|
864
|
+
# Release nonce on broadcast failure
|
|
865
|
+
self._nonce_manager.release(signer_address, nonce)
|
|
866
|
+
|
|
867
|
+
if self._lifecycle:
|
|
868
|
+
self._lifecycle.on_failed(
|
|
869
|
+
intent, attempt, e,
|
|
870
|
+
failure_type=FailureType.BROADCAST_FAILED,
|
|
871
|
+
failure_stage=FailureStage.BROADCAST,
|
|
872
|
+
cleanup_trigger=False,
|
|
873
|
+
)
|
|
874
|
+
_retry_intent("broadcast_failed")
|
|
875
|
+
|
|
876
|
+
return ExecutionOutcome(
|
|
877
|
+
result=ExecutionResult.FAILED,
|
|
878
|
+
intent=intent,
|
|
879
|
+
attempt=attempt,
|
|
880
|
+
error=e,
|
|
881
|
+
)
|
|
882
|
+
|
|
883
|
+
return ExecutionOutcome(
|
|
884
|
+
result=ExecutionResult.PENDING,
|
|
885
|
+
intent=intent,
|
|
886
|
+
attempt=attempt,
|
|
887
|
+
tx_hash=tx_hash,
|
|
888
|
+
)
|
|
889
|
+
|
|
890
|
+
def _build_tx_dict(
|
|
891
|
+
self,
|
|
892
|
+
intent: TxIntent,
|
|
893
|
+
nonce: int,
|
|
894
|
+
gas_params: GasParams,
|
|
895
|
+
to_address: str | None = None,
|
|
896
|
+
) -> dict:
|
|
897
|
+
"""Build transaction dictionary for signing.
|
|
898
|
+
|
|
899
|
+
Args:
|
|
900
|
+
intent: Transaction intent
|
|
901
|
+
nonce: Nonce to use
|
|
902
|
+
gas_params: Gas parameters
|
|
903
|
+
to_address: Resolved to address (optional, uses intent if not provided)
|
|
904
|
+
|
|
905
|
+
Returns:
|
|
906
|
+
Transaction dictionary ready for signing
|
|
907
|
+
"""
|
|
908
|
+
tx = {
|
|
909
|
+
"nonce": nonce,
|
|
910
|
+
"to": to_address or intent.to_address,
|
|
911
|
+
"value": intent.value_wei,
|
|
912
|
+
"gas": gas_params.gas_limit,
|
|
913
|
+
"maxFeePerGas": gas_params.max_fee_per_gas,
|
|
914
|
+
"maxPriorityFeePerGas": gas_params.max_priority_fee_per_gas,
|
|
915
|
+
"chainId": intent.chain_id,
|
|
916
|
+
"type": 2, # EIP-1559
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
if intent.data:
|
|
920
|
+
tx["data"] = intent.data
|
|
921
|
+
|
|
922
|
+
return normalize_tx_dict(tx)
|
|
923
|
+
|
|
924
|
+
# =========================================================================
|
|
925
|
+
# Simulation
|
|
926
|
+
# =========================================================================
|
|
927
|
+
|
|
928
|
+
def _simulate_with_retry(
|
|
929
|
+
self,
|
|
930
|
+
job: "Job",
|
|
931
|
+
intent: TxIntent,
|
|
932
|
+
tx: dict,
|
|
933
|
+
) -> str:
|
|
934
|
+
"""Simulate transaction with retry on network errors.
|
|
935
|
+
|
|
936
|
+
Args:
|
|
937
|
+
job: Job instance for validation hook
|
|
938
|
+
intent: Transaction intent
|
|
939
|
+
tx: Transaction dict for simulation
|
|
940
|
+
|
|
941
|
+
Returns:
|
|
942
|
+
Hex-encoded output on success
|
|
943
|
+
|
|
944
|
+
Raises:
|
|
945
|
+
SimulationReverted: Permanent failure (no retry)
|
|
946
|
+
SimulationNetworkError: After all retries exhausted
|
|
947
|
+
"""
|
|
948
|
+
last_error: SimulationNetworkError | None = None
|
|
949
|
+
|
|
950
|
+
# Resolve per-job RPC override (job.rpc overrides global)
|
|
951
|
+
rpc_url = getattr(job, "rpc", None)
|
|
952
|
+
|
|
953
|
+
for attempt in range(MAX_SIMULATION_RETRIES + 1):
|
|
954
|
+
try:
|
|
955
|
+
# Run simulation (uses job RPC if specified)
|
|
956
|
+
output = self._rpc.simulate_transaction(tx, rpc_url=rpc_url)
|
|
957
|
+
|
|
958
|
+
# Run job's custom validation (if defined)
|
|
959
|
+
if hasattr(job, "validate_simulation"):
|
|
960
|
+
if not job.validate_simulation(output):
|
|
961
|
+
raise SimulationReverted("Job validation rejected")
|
|
962
|
+
|
|
963
|
+
# Success
|
|
964
|
+
if attempt > 0:
|
|
965
|
+
logger.info(
|
|
966
|
+
"simulation.retry_succeeded",
|
|
967
|
+
intent_id=str(intent.intent_id),
|
|
968
|
+
job_id=job.job_id,
|
|
969
|
+
attempt=attempt + 1,
|
|
970
|
+
)
|
|
971
|
+
return output
|
|
972
|
+
|
|
973
|
+
except SimulationReverted:
|
|
974
|
+
# Permanent failure - don't retry
|
|
975
|
+
raise
|
|
976
|
+
|
|
977
|
+
except SimulationNetworkError as e:
|
|
978
|
+
last_error = e
|
|
979
|
+
metrics = get_metrics()
|
|
980
|
+
|
|
981
|
+
# Log retry attempt
|
|
982
|
+
if attempt < MAX_SIMULATION_RETRIES:
|
|
983
|
+
metrics.counter(SIMULATION_RETRIES).inc(
|
|
984
|
+
chain_id=intent.chain_id,
|
|
985
|
+
job_id=job.job_id,
|
|
986
|
+
)
|
|
987
|
+
logger.warning(
|
|
988
|
+
"simulation.network_error_retrying",
|
|
989
|
+
intent_id=str(intent.intent_id),
|
|
990
|
+
job_id=job.job_id,
|
|
991
|
+
attempt=attempt + 1,
|
|
992
|
+
max_attempts=MAX_SIMULATION_RETRIES + 1,
|
|
993
|
+
error=str(e),
|
|
994
|
+
)
|
|
995
|
+
else:
|
|
996
|
+
metrics.counter(SIMULATION_NETWORK_ERRORS).inc(
|
|
997
|
+
chain_id=intent.chain_id,
|
|
998
|
+
job_id=job.job_id,
|
|
999
|
+
)
|
|
1000
|
+
logger.error(
|
|
1001
|
+
"simulation.network_error_exhausted",
|
|
1002
|
+
intent_id=str(intent.intent_id),
|
|
1003
|
+
job_id=job.job_id,
|
|
1004
|
+
attempts=MAX_SIMULATION_RETRIES + 1,
|
|
1005
|
+
error=str(e),
|
|
1006
|
+
)
|
|
1007
|
+
|
|
1008
|
+
# All retries exhausted
|
|
1009
|
+
if last_error is None:
|
|
1010
|
+
last_error = SimulationNetworkError("Unknown simulation error")
|
|
1011
|
+
raise last_error
|
|
1012
|
+
|
|
1013
|
+
def _handle_simulation_failure(
|
|
1014
|
+
self,
|
|
1015
|
+
job: "Job",
|
|
1016
|
+
intent: TxIntent,
|
|
1017
|
+
error: SimulationReverted | SimulationNetworkError,
|
|
1018
|
+
) -> ExecutionOutcome:
|
|
1019
|
+
"""Handle simulation failure - mark intent failed and alert.
|
|
1020
|
+
|
|
1021
|
+
Args:
|
|
1022
|
+
job: Job instance
|
|
1023
|
+
intent: Transaction intent
|
|
1024
|
+
error: Simulation error
|
|
1025
|
+
|
|
1026
|
+
Returns:
|
|
1027
|
+
ExecutionOutcome with failure details
|
|
1028
|
+
"""
|
|
1029
|
+
metrics = get_metrics()
|
|
1030
|
+
|
|
1031
|
+
if isinstance(error, SimulationReverted):
|
|
1032
|
+
failure_message = f"Simulation reverted: {error.reason}"
|
|
1033
|
+
metrics.counter(SIMULATION_REVERTED).inc(
|
|
1034
|
+
chain_id=intent.chain_id,
|
|
1035
|
+
job_id=job.job_id,
|
|
1036
|
+
)
|
|
1037
|
+
else:
|
|
1038
|
+
failure_message = f"Simulation error: {error}"
|
|
1039
|
+
# Note: SIMULATION_NETWORK_ERRORS is already recorded in _simulate_with_retry
|
|
1040
|
+
|
|
1041
|
+
logger.warning(
|
|
1042
|
+
"simulation.failed",
|
|
1043
|
+
intent_id=str(intent.intent_id),
|
|
1044
|
+
job_id=job.job_id,
|
|
1045
|
+
error=failure_message,
|
|
1046
|
+
)
|
|
1047
|
+
|
|
1048
|
+
# Mark intent as failed in database (transition_intent auto-clears claim)
|
|
1049
|
+
transition_intent(
|
|
1050
|
+
self._db,
|
|
1051
|
+
intent.intent_id,
|
|
1052
|
+
IntentStatus.FAILED,
|
|
1053
|
+
"simulation_failed",
|
|
1054
|
+
chain_id=intent.chain_id,
|
|
1055
|
+
)
|
|
1056
|
+
|
|
1057
|
+
# Fire alert
|
|
1058
|
+
if self._lifecycle:
|
|
1059
|
+
self._lifecycle.on_simulation_failed(job, intent, error)
|
|
1060
|
+
|
|
1061
|
+
return ExecutionOutcome(
|
|
1062
|
+
result=ExecutionResult.FAILED,
|
|
1063
|
+
intent=intent,
|
|
1064
|
+
attempt=None,
|
|
1065
|
+
error=error,
|
|
1066
|
+
)
|
|
1067
|
+
|
|
1068
|
+
# NOTE: _abandon_stranded_intents() has been removed as part of the
|
|
1069
|
+
# nonce policy simplification. Stranded intents are now recovered by
|
|
1070
|
+
# TxReplacer via fee bumping, rather than being auto-abandoned.
|
|
1071
|
+
# See NONCE.md for the new policy.
|