brawny 0.1.13__py3-none-any.whl → 0.1.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brawny/__init__.py +2 -0
- brawny/_context.py +5 -5
- brawny/_rpc/__init__.py +36 -12
- brawny/_rpc/broadcast.py +14 -13
- brawny/_rpc/caller.py +243 -0
- brawny/_rpc/client.py +539 -0
- brawny/_rpc/clients.py +11 -11
- brawny/_rpc/context.py +23 -0
- brawny/_rpc/errors.py +465 -31
- brawny/_rpc/gas.py +7 -6
- brawny/_rpc/pool.py +18 -0
- brawny/_rpc/retry.py +266 -0
- brawny/_rpc/retry_policy.py +81 -0
- brawny/accounts.py +28 -9
- brawny/alerts/__init__.py +15 -18
- brawny/alerts/abi_resolver.py +212 -36
- brawny/alerts/base.py +2 -2
- brawny/alerts/contracts.py +77 -10
- brawny/alerts/errors.py +30 -3
- brawny/alerts/events.py +38 -5
- brawny/alerts/health.py +19 -13
- brawny/alerts/send.py +513 -55
- brawny/api.py +39 -11
- brawny/assets/AGENTS.md +325 -0
- brawny/async_runtime.py +48 -0
- brawny/chain.py +3 -3
- brawny/cli/commands/__init__.py +2 -0
- brawny/cli/commands/console.py +69 -19
- brawny/cli/commands/contract.py +2 -2
- brawny/cli/commands/controls.py +121 -0
- brawny/cli/commands/health.py +2 -2
- brawny/cli/commands/job_dev.py +6 -5
- brawny/cli/commands/jobs.py +99 -2
- brawny/cli/commands/maintenance.py +13 -29
- brawny/cli/commands/migrate.py +1 -0
- brawny/cli/commands/run.py +10 -3
- brawny/cli/commands/script.py +8 -3
- brawny/cli/commands/signer.py +143 -26
- brawny/cli/helpers.py +0 -3
- brawny/cli_templates.py +25 -349
- brawny/config/__init__.py +4 -1
- brawny/config/models.py +43 -57
- brawny/config/parser.py +268 -57
- brawny/config/validation.py +52 -15
- brawny/daemon/context.py +4 -2
- brawny/daemon/core.py +185 -63
- brawny/daemon/loops.py +166 -98
- brawny/daemon/supervisor.py +261 -0
- brawny/db/__init__.py +14 -26
- brawny/db/base.py +248 -151
- brawny/db/global_cache.py +11 -1
- brawny/db/migrate.py +175 -28
- brawny/db/migrations/001_init.sql +4 -3
- brawny/db/migrations/010_add_nonce_gap_index.sql +1 -1
- brawny/db/migrations/011_add_job_logs.sql +1 -2
- brawny/db/migrations/012_add_claimed_by.sql +2 -2
- brawny/db/migrations/013_attempt_unique.sql +10 -0
- brawny/db/migrations/014_add_lease_expires_at.sql +5 -0
- brawny/db/migrations/015_add_signer_alias.sql +14 -0
- brawny/db/migrations/016_runtime_controls_and_quarantine.sql +32 -0
- brawny/db/migrations/017_add_job_drain.sql +6 -0
- brawny/db/migrations/018_add_nonce_reset_audit.sql +20 -0
- brawny/db/migrations/019_add_job_cooldowns.sql +8 -0
- brawny/db/migrations/020_attempt_unique_initial.sql +7 -0
- brawny/db/ops/__init__.py +3 -25
- brawny/db/ops/logs.py +1 -2
- brawny/db/queries.py +47 -91
- brawny/db/serialized.py +65 -0
- brawny/db/sqlite/__init__.py +1001 -0
- brawny/db/sqlite/connection.py +231 -0
- brawny/db/sqlite/execute.py +116 -0
- brawny/db/sqlite/mappers.py +190 -0
- brawny/db/sqlite/repos/attempts.py +372 -0
- brawny/db/sqlite/repos/block_state.py +102 -0
- brawny/db/sqlite/repos/cache.py +104 -0
- brawny/db/sqlite/repos/intents.py +1021 -0
- brawny/db/sqlite/repos/jobs.py +200 -0
- brawny/db/sqlite/repos/maintenance.py +182 -0
- brawny/db/sqlite/repos/signers_nonces.py +566 -0
- brawny/db/sqlite/tx.py +119 -0
- brawny/http.py +194 -0
- brawny/invariants.py +11 -24
- brawny/jobs/base.py +8 -0
- brawny/jobs/job_validation.py +2 -1
- brawny/keystore.py +83 -7
- brawny/lifecycle.py +64 -12
- brawny/logging.py +0 -2
- brawny/metrics.py +84 -12
- brawny/model/contexts.py +111 -9
- brawny/model/enums.py +1 -0
- brawny/model/errors.py +18 -0
- brawny/model/types.py +47 -131
- brawny/network_guard.py +133 -0
- brawny/networks/__init__.py +5 -5
- brawny/networks/config.py +1 -7
- brawny/networks/manager.py +14 -11
- brawny/runtime_controls.py +74 -0
- brawny/scheduler/poller.py +11 -7
- brawny/scheduler/reorg.py +95 -39
- brawny/scheduler/runner.py +442 -168
- brawny/scheduler/shutdown.py +3 -3
- brawny/script_tx.py +3 -3
- brawny/telegram.py +53 -7
- brawny/testing.py +1 -0
- brawny/timeout.py +38 -0
- brawny/tx/executor.py +922 -308
- brawny/tx/intent.py +54 -16
- brawny/tx/monitor.py +31 -12
- brawny/tx/nonce.py +212 -90
- brawny/tx/replacement.py +69 -18
- brawny/tx/retry_policy.py +24 -0
- brawny/tx/stages/types.py +75 -0
- brawny/types.py +18 -0
- brawny/utils.py +41 -0
- {brawny-0.1.13.dist-info → brawny-0.1.22.dist-info}/METADATA +3 -3
- brawny-0.1.22.dist-info/RECORD +163 -0
- brawny/_rpc/manager.py +0 -982
- brawny/_rpc/selector.py +0 -156
- brawny/db/base_new.py +0 -165
- brawny/db/mappers.py +0 -182
- brawny/db/migrations/008_add_transactions.sql +0 -72
- brawny/db/ops/attempts.py +0 -108
- brawny/db/ops/blocks.py +0 -83
- brawny/db/ops/cache.py +0 -93
- brawny/db/ops/intents.py +0 -296
- brawny/db/ops/jobs.py +0 -110
- brawny/db/ops/nonces.py +0 -322
- brawny/db/postgres.py +0 -2535
- brawny/db/postgres_new.py +0 -196
- brawny/db/sqlite.py +0 -2733
- brawny/db/sqlite_new.py +0 -191
- brawny-0.1.13.dist-info/RECORD +0 -141
- {brawny-0.1.13.dist-info → brawny-0.1.22.dist-info}/WHEEL +0 -0
- {brawny-0.1.13.dist-info → brawny-0.1.22.dist-info}/entry_points.txt +0 -0
- {brawny-0.1.13.dist-info → brawny-0.1.22.dist-info}/top_level.txt +0 -0
brawny/metrics.py
CHANGED
|
@@ -424,20 +424,38 @@ JOBS_TRIGGERED = "brawny_jobs_triggered_total"
|
|
|
424
424
|
INTENTS_CREATED = "brawny_intents_created_total"
|
|
425
425
|
INTENT_TRANSITIONS = "brawny_intent_transitions_total"
|
|
426
426
|
INTENT_RETRY_ATTEMPTS = "brawny_intent_retry_attempts_total"
|
|
427
|
+
EXECUTOR_STAGE_STARTED = "brawny_executor_stage_started_total"
|
|
428
|
+
EXECUTOR_STAGE_OUTCOME = "brawny_executor_stage_outcome_total"
|
|
429
|
+
EXECUTOR_STAGE_TIMEOUTS = "brawny_executor_stage_timeouts_total"
|
|
427
430
|
INTENT_CLAIMED = "brawny_intent_claimed_total"
|
|
428
431
|
INTENT_RELEASED = "brawny_intent_released_total"
|
|
432
|
+
CLAIM_RELEASED_PRE_ATTEMPT = "brawny_claim_released_pre_attempt_total"
|
|
433
|
+
CLAIM_RELEASE_SKIPPED = "brawny_claim_release_skipped_total"
|
|
434
|
+
CLAIM_RECLAIM_SKIPPED = "brawny_claim_reclaim_skipped_total"
|
|
435
|
+
INTENT_COOLDOWN_SKIPPED = "brawny_intent_cooldown_skipped_total"
|
|
436
|
+
INTENT_COOLDOWN_ERRORS = "brawny_intent_cooldown_errors_total"
|
|
429
437
|
INTENT_STATE_INCONSISTENT = "brawny_intent_state_inconsistent_total"
|
|
430
438
|
INTENT_SENDING_STUCK = "brawny_intent_sending_stuck_total"
|
|
439
|
+
BACKGROUND_TASK_ERRORS = "brawny_background_task_errors_total"
|
|
440
|
+
ALERTS_ENQUEUED = "brawny_alerts_enqueued_total"
|
|
441
|
+
ALERTS_DROPPED = "brawny_alerts_dropped_total"
|
|
442
|
+
ALERTS_SENT = "brawny_alerts_sent_total"
|
|
443
|
+
ALERTS_RETRIED = "brawny_alerts_retried_total"
|
|
444
|
+
ALERTS_LAST_SUCCESS_TIMESTAMP = "brawny_alerts_last_success_timestamp"
|
|
445
|
+
ALERTS_LAST_ERROR_TIMESTAMP = "brawny_alerts_last_error_timestamp"
|
|
446
|
+
ALERTS_WORKER_ALIVE = "brawny_alerts_worker_alive"
|
|
447
|
+
ALERTS_OLDEST_QUEUED_AGE_SECONDS = "brawny_alerts_oldest_queued_age_seconds"
|
|
431
448
|
TX_BROADCAST = "brawny_tx_broadcast_total"
|
|
432
449
|
TX_CONFIRMED = "brawny_tx_confirmed_total"
|
|
433
450
|
TX_FAILED = "brawny_tx_failed_total"
|
|
434
451
|
TX_REPLACED = "brawny_tx_replaced_total"
|
|
435
452
|
RPC_REQUESTS = "brawny_rpc_requests_total"
|
|
436
453
|
RPC_ERRORS = "brawny_rpc_errors_total"
|
|
454
|
+
RPC_CALL_TIMEOUTS = "brawny_rpc_call_timeouts_total"
|
|
437
455
|
RPC_REQUESTS_BY_JOB = "brawny_rpc_requests_by_job_total"
|
|
438
|
-
RPC_RATE_LIMITED = "brawny_rpc_rate_limited_total"
|
|
439
456
|
RPC_FAILOVERS = "brawny_rpc_failovers_total"
|
|
440
|
-
|
|
457
|
+
RPC_ERROR_CLASSIFIED = "brawny_rpc_error_classified_total"
|
|
458
|
+
RPC_ERROR_UNKNOWN = "brawny_rpc_error_unknown_total"
|
|
441
459
|
JOB_CHECK_TIMEOUTS = "brawny_job_check_timeouts_total"
|
|
442
460
|
JOB_BUILD_TIMEOUTS = "brawny_job_build_timeouts_total"
|
|
443
461
|
REORGS_DETECTED = "brawny_reorg_detected_total"
|
|
@@ -446,14 +464,17 @@ SIMULATION_REVERTED = "brawny_simulation_reverted_total"
|
|
|
446
464
|
SIMULATION_NETWORK_ERRORS = "brawny_simulation_network_errors_total"
|
|
447
465
|
SIMULATION_RETRIES = "brawny_simulation_retries_total"
|
|
448
466
|
BROADCAST_ATTEMPTS = "brawny_broadcast_attempts_total"
|
|
449
|
-
|
|
450
|
-
|
|
467
|
+
NETWORK_GUARD_ALLOW = "brawny_network_guard_allow_total"
|
|
468
|
+
NETWORK_GUARD_VIOLATION = "brawny_network_guard_violation_total"
|
|
469
|
+
NONCE_GAP_DETECTED = "brawny_nonce_gap_detected_total"
|
|
470
|
+
NONCE_FORCE_RESET = "brawny_nonce_force_reset_total"
|
|
451
471
|
|
|
452
472
|
# Gauges
|
|
453
473
|
LAST_PROCESSED_BLOCK = "brawny_last_processed_block"
|
|
454
474
|
PENDING_INTENTS = "brawny_pending_intents"
|
|
455
475
|
INTENTS_BACKING_OFF = "brawny_intents_backing_off"
|
|
456
476
|
ACTIVE_WORKERS = "brawny_active_workers"
|
|
477
|
+
ALERTS_QUEUE_DEPTH = "brawny_alerts_queue_depth"
|
|
457
478
|
RPC_ENDPOINT_HEALTH = "brawny_rpc_endpoint_health"
|
|
458
479
|
DB_CIRCUIT_BREAKER_STATE = "brawny_db_circuit_breaker_open"
|
|
459
480
|
|
|
@@ -479,8 +500,11 @@ INVARIANT_ORPHANED_NONCES = "brawny_invariant_orphaned_nonces"
|
|
|
479
500
|
BLOCK_PROCESSING_SECONDS = "brawny_block_processing_seconds"
|
|
480
501
|
TX_CONFIRMATION_SECONDS = "brawny_tx_confirmation_seconds"
|
|
481
502
|
RPC_REQUEST_SECONDS = "brawny_rpc_request_seconds"
|
|
503
|
+
EXECUTOR_ATTEMPT_DURATION_SECONDS = "brawny_executor_attempt_duration_seconds"
|
|
482
504
|
JOB_CHECK_SECONDS = "brawny_job_check_seconds"
|
|
483
505
|
BROADCAST_LATENCY_SECONDS = "brawny_broadcast_latency_seconds"
|
|
506
|
+
RUNTIME_CONTROL_ACTIVE = "brawny_runtime_control_active"
|
|
507
|
+
RUNTIME_CONTROL_TTL_SECONDS = "brawny_runtime_control_ttl_seconds"
|
|
484
508
|
|
|
485
509
|
# Metric label schema (fixed, low-cardinality)
|
|
486
510
|
METRIC_LABELS = {
|
|
@@ -489,19 +513,31 @@ METRIC_LABELS = {
|
|
|
489
513
|
INTENTS_CREATED: ["chain_id", "job_id"],
|
|
490
514
|
INTENT_TRANSITIONS: ["chain_id", "from_status", "to_status", "reason"],
|
|
491
515
|
INTENT_RETRY_ATTEMPTS: ["chain_id", "reason"],
|
|
516
|
+
EXECUTOR_STAGE_STARTED: ["stage"],
|
|
517
|
+
EXECUTOR_STAGE_OUTCOME: ["stage", "outcome"],
|
|
518
|
+
EXECUTOR_STAGE_TIMEOUTS: ["stage"],
|
|
492
519
|
INTENT_CLAIMED: ["chain_id"],
|
|
493
520
|
INTENT_RELEASED: ["chain_id", "reason"],
|
|
521
|
+
CLAIM_RELEASED_PRE_ATTEMPT: ["stage"],
|
|
522
|
+
CLAIM_RELEASE_SKIPPED: ["stage"],
|
|
523
|
+
CLAIM_RECLAIM_SKIPPED: ["chain_id"],
|
|
524
|
+
INTENT_COOLDOWN_SKIPPED: ["chain_id"],
|
|
525
|
+
INTENT_COOLDOWN_ERRORS: ["chain_id"],
|
|
494
526
|
INTENT_STATE_INCONSISTENT: ["chain_id", "reason"],
|
|
495
527
|
INTENT_SENDING_STUCK: ["chain_id", "age_bucket"],
|
|
528
|
+
BACKGROUND_TASK_ERRORS: ["task"],
|
|
529
|
+
ALERTS_ENQUEUED: [],
|
|
530
|
+
ALERTS_DROPPED: ["reason", "channel"],
|
|
531
|
+
ALERTS_SENT: [],
|
|
532
|
+
ALERTS_RETRIED: [],
|
|
496
533
|
TX_BROADCAST: ["chain_id", "job_id"],
|
|
497
534
|
TX_CONFIRMED: ["chain_id", "job_id"],
|
|
498
535
|
TX_FAILED: ["chain_id", "job_id", "reason"],
|
|
499
536
|
TX_REPLACED: ["chain_id", "job_id"],
|
|
500
537
|
RPC_REQUESTS: ["chain_id", "method", "rpc_category", "rpc_host"],
|
|
501
538
|
RPC_ERRORS: ["chain_id", "method", "rpc_category", "rpc_host"],
|
|
539
|
+
RPC_CALL_TIMEOUTS: ["chain_id", "method", "rpc_category", "rpc_host"],
|
|
502
540
|
RPC_REQUESTS_BY_JOB: ["chain_id", "job_id", "rpc_category"],
|
|
503
|
-
RPC_RATE_LIMITED: ["endpoint"],
|
|
504
|
-
ALERTS_SENT: ["chain_id", "channel"],
|
|
505
541
|
JOB_CHECK_TIMEOUTS: ["chain_id", "job_id"],
|
|
506
542
|
JOB_BUILD_TIMEOUTS: ["chain_id", "job_id"],
|
|
507
543
|
REORGS_DETECTED: ["chain_id"],
|
|
@@ -510,12 +546,21 @@ METRIC_LABELS = {
|
|
|
510
546
|
SIMULATION_NETWORK_ERRORS: ["chain_id", "job_id"],
|
|
511
547
|
SIMULATION_RETRIES: ["chain_id", "job_id"],
|
|
512
548
|
BROADCAST_ATTEMPTS: ["chain_id", "job_id", "broadcast_group", "result"],
|
|
513
|
-
|
|
514
|
-
|
|
549
|
+
RPC_ERROR_CLASSIFIED: ["kind", "method", "source"],
|
|
550
|
+
RPC_ERROR_UNKNOWN: ["method", "exception_type", "provider", "http_status", "jsonrpc_code"],
|
|
551
|
+
NETWORK_GUARD_ALLOW: ["reason"],
|
|
552
|
+
NETWORK_GUARD_VIOLATION: ["context", "caller_module"],
|
|
553
|
+
NONCE_GAP_DETECTED: ["chain_id", "signer"],
|
|
554
|
+
NONCE_FORCE_RESET: ["chain_id", "signer", "source"],
|
|
515
555
|
LAST_PROCESSED_BLOCK: ["chain_id"],
|
|
516
556
|
PENDING_INTENTS: ["chain_id"],
|
|
517
557
|
INTENTS_BACKING_OFF: ["chain_id"],
|
|
518
558
|
ACTIVE_WORKERS: ["chain_id"],
|
|
559
|
+
ALERTS_QUEUE_DEPTH: [],
|
|
560
|
+
ALERTS_LAST_SUCCESS_TIMESTAMP: [],
|
|
561
|
+
ALERTS_LAST_ERROR_TIMESTAMP: [],
|
|
562
|
+
ALERTS_WORKER_ALIVE: [],
|
|
563
|
+
ALERTS_OLDEST_QUEUED_AGE_SECONDS: [],
|
|
519
564
|
RPC_ENDPOINT_HEALTH: ["endpoint"],
|
|
520
565
|
DB_CIRCUIT_BREAKER_STATE: ["db_backend"],
|
|
521
566
|
OLDEST_PENDING_INTENT_AGE_SECONDS: ["chain_id"],
|
|
@@ -528,8 +573,11 @@ METRIC_LABELS = {
|
|
|
528
573
|
BLOCK_PROCESSING_SECONDS: ["chain_id"],
|
|
529
574
|
TX_CONFIRMATION_SECONDS: ["chain_id"],
|
|
530
575
|
RPC_REQUEST_SECONDS: ["chain_id", "method", "rpc_category", "rpc_host"],
|
|
576
|
+
EXECUTOR_ATTEMPT_DURATION_SECONDS: ["stage"],
|
|
531
577
|
JOB_CHECK_SECONDS: ["chain_id", "job_id"],
|
|
532
578
|
BROADCAST_LATENCY_SECONDS: ["chain_id", "job_id", "broadcast_group"],
|
|
579
|
+
RUNTIME_CONTROL_ACTIVE: ["control"],
|
|
580
|
+
RUNTIME_CONTROL_TTL_SECONDS: ["control"],
|
|
533
581
|
# Invariants (Phase 2)
|
|
534
582
|
INVARIANT_STUCK_CLAIMED: ["chain_id"],
|
|
535
583
|
INVARIANT_NONCE_GAP_AGE: ["chain_id"],
|
|
@@ -544,19 +592,35 @@ METRIC_DESCRIPTIONS = {
|
|
|
544
592
|
INTENTS_CREATED: "Total intents created",
|
|
545
593
|
INTENT_TRANSITIONS: "Total intent status transitions",
|
|
546
594
|
INTENT_RETRY_ATTEMPTS: "Total intent retry attempts",
|
|
595
|
+
EXECUTOR_STAGE_STARTED: "Total executor stages started",
|
|
596
|
+
EXECUTOR_STAGE_OUTCOME: "Total executor stage outcomes",
|
|
597
|
+
EXECUTOR_STAGE_TIMEOUTS: "Total executor stage timeouts",
|
|
547
598
|
INTENT_CLAIMED: "Total intents claimed",
|
|
548
599
|
INTENT_RELEASED: "Total intents released",
|
|
600
|
+
CLAIM_RELEASED_PRE_ATTEMPT: "Total claims released before attempts exist",
|
|
601
|
+
CLAIM_RELEASE_SKIPPED: "Total claim releases skipped (token mismatch or attempts)",
|
|
602
|
+
CLAIM_RECLAIM_SKIPPED: "Expired claims skipped due to attempts",
|
|
603
|
+
INTENT_COOLDOWN_SKIPPED: "Intents skipped due to cooldown",
|
|
604
|
+
INTENT_COOLDOWN_ERRORS: "Cooldown check errors",
|
|
549
605
|
INTENT_STATE_INCONSISTENT: "Total inconsistent intent state detections",
|
|
550
606
|
INTENT_SENDING_STUCK: "Total intents detected stuck in sending",
|
|
607
|
+
BACKGROUND_TASK_ERRORS: "Total background loop errors",
|
|
608
|
+
ALERTS_ENQUEUED: "Total alerts enqueued for delivery",
|
|
609
|
+
ALERTS_DROPPED: "Total alerts dropped before sending",
|
|
610
|
+
ALERTS_SENT: "Total alerts delivered successfully",
|
|
611
|
+
ALERTS_RETRIED: "Total alert send retries",
|
|
612
|
+
ALERTS_LAST_SUCCESS_TIMESTAMP: "Unix timestamp of last alert success",
|
|
613
|
+
ALERTS_LAST_ERROR_TIMESTAMP: "Unix timestamp of last alert error",
|
|
614
|
+
ALERTS_WORKER_ALIVE: "Alert worker alive state (1=alive, 0=dead)",
|
|
615
|
+
ALERTS_OLDEST_QUEUED_AGE_SECONDS: "Age in seconds of oldest queued alert",
|
|
551
616
|
TX_BROADCAST: "Total transactions broadcast",
|
|
552
617
|
TX_CONFIRMED: "Total transactions confirmed",
|
|
553
618
|
TX_FAILED: "Total transactions failed",
|
|
554
619
|
TX_REPLACED: "Total transactions replaced",
|
|
555
620
|
RPC_REQUESTS: "Total RPC requests",
|
|
556
621
|
RPC_ERRORS: "Total RPC errors (failed attempts)",
|
|
622
|
+
RPC_CALL_TIMEOUTS: "Total RPC call timeouts",
|
|
557
623
|
RPC_REQUESTS_BY_JOB: "RPC requests attributed to jobs",
|
|
558
|
-
RPC_RATE_LIMITED: "RPC requests delayed by rate limiting",
|
|
559
|
-
ALERTS_SENT: "Total alerts sent",
|
|
560
624
|
JOB_CHECK_TIMEOUTS: "Total job check timeouts",
|
|
561
625
|
JOB_BUILD_TIMEOUTS: "Total job build_intent timeouts",
|
|
562
626
|
REORGS_DETECTED: "Total reorgs detected",
|
|
@@ -565,12 +629,17 @@ METRIC_DESCRIPTIONS = {
|
|
|
565
629
|
SIMULATION_NETWORK_ERRORS: "Total simulation network errors (after all retries)",
|
|
566
630
|
SIMULATION_RETRIES: "Total simulation retry attempts",
|
|
567
631
|
BROADCAST_ATTEMPTS: "Total broadcast attempts by result (success, unavailable, fatal, recoverable)",
|
|
568
|
-
|
|
569
|
-
|
|
632
|
+
RPC_ERROR_CLASSIFIED: "Total RPC errors classified by the new classifier",
|
|
633
|
+
RPC_ERROR_UNKNOWN: "Total RPC errors that are unknown to the new classifier",
|
|
634
|
+
NETWORK_GUARD_ALLOW: "Network guard allowlist escapes (approved wrappers)",
|
|
635
|
+
NETWORK_GUARD_VIOLATION: "Network guard blocked direct network calls",
|
|
636
|
+
NONCE_GAP_DETECTED: "Nonce gap detected (chain_pending < db_next_nonce) - observability only, no auto-reset",
|
|
637
|
+
NONCE_FORCE_RESET: "Explicit nonce force reset (CLI or allow_unsafe_nonce_reset config)",
|
|
570
638
|
LAST_PROCESSED_BLOCK: "Last processed block",
|
|
571
639
|
PENDING_INTENTS: "Pending intents",
|
|
572
640
|
INTENTS_BACKING_OFF: "Intents in backoff window (retry_after in future)",
|
|
573
641
|
ACTIVE_WORKERS: "Active worker threads",
|
|
642
|
+
ALERTS_QUEUE_DEPTH: "Alert queue depth",
|
|
574
643
|
RPC_ENDPOINT_HEALTH: "RPC endpoint health (1=healthy, 0=unhealthy)",
|
|
575
644
|
DB_CIRCUIT_BREAKER_STATE: "Database circuit breaker open state (1=open, 0=closed)",
|
|
576
645
|
OLDEST_PENDING_INTENT_AGE_SECONDS: "Age in seconds of oldest pending intent (CREATED, PENDING, CLAIMED, SENDING)",
|
|
@@ -583,8 +652,11 @@ METRIC_DESCRIPTIONS = {
|
|
|
583
652
|
BLOCK_PROCESSING_SECONDS: "Block processing duration in seconds",
|
|
584
653
|
TX_CONFIRMATION_SECONDS: "Transaction confirmation duration in seconds",
|
|
585
654
|
RPC_REQUEST_SECONDS: "RPC request duration in seconds",
|
|
655
|
+
EXECUTOR_ATTEMPT_DURATION_SECONDS: "Executor stage duration in seconds",
|
|
586
656
|
JOB_CHECK_SECONDS: "Job check duration in seconds",
|
|
587
657
|
BROADCAST_LATENCY_SECONDS: "Broadcast transaction latency in seconds",
|
|
658
|
+
RUNTIME_CONTROL_ACTIVE: "Runtime control active state (1=active, 0=inactive)",
|
|
659
|
+
RUNTIME_CONTROL_TTL_SECONDS: "Runtime control TTL remaining in seconds",
|
|
588
660
|
# Invariants (Phase 2)
|
|
589
661
|
INVARIANT_STUCK_CLAIMED: "Intents stuck in claimed status > threshold minutes",
|
|
590
662
|
INVARIANT_NONCE_GAP_AGE: "Age in seconds of oldest nonce gap (reserved below chain nonce)",
|
brawny/model/contexts.py
CHANGED
|
@@ -8,15 +8,18 @@ Each phase gets only what it needs:
|
|
|
8
8
|
Contract access is explicit and block-aware:
|
|
9
9
|
- at_block(name, addr, block): Pinned reads for check()
|
|
10
10
|
- at(name, addr): Latest reads for build/alerts
|
|
11
|
+
|
|
12
|
+
Lifecycle hooks (on_trigger, on_success, on_failure) have ctx.alert() for
|
|
13
|
+
sending alerts to job destinations. See AlertMixin for details.
|
|
11
14
|
"""
|
|
12
15
|
|
|
13
16
|
from __future__ import annotations
|
|
14
17
|
|
|
15
|
-
from dataclasses import dataclass
|
|
18
|
+
from dataclasses import dataclass, field
|
|
16
19
|
from typing import TYPE_CHECKING, Any, Protocol
|
|
17
20
|
|
|
18
21
|
from brawny.model.events import DecodedEvent
|
|
19
|
-
from brawny.model.errors import FailureType
|
|
22
|
+
from brawny.model.errors import FailureType, FailureStage
|
|
20
23
|
|
|
21
24
|
if TYPE_CHECKING:
|
|
22
25
|
from brawny.jobs.kv import KVReader, KVStore
|
|
@@ -28,6 +31,70 @@ if TYPE_CHECKING:
|
|
|
28
31
|
import structlog
|
|
29
32
|
|
|
30
33
|
|
|
34
|
+
# =============================================================================
|
|
35
|
+
# Alert Sender Protocol + Mixin
|
|
36
|
+
# =============================================================================
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class AlertSender(Protocol):
|
|
40
|
+
"""Protocol for alert sending. Injected into lifecycle contexts."""
|
|
41
|
+
|
|
42
|
+
def send(
|
|
43
|
+
self,
|
|
44
|
+
message: str,
|
|
45
|
+
*,
|
|
46
|
+
to: str | list[str] | None = None,
|
|
47
|
+
parse_mode: str | None = None,
|
|
48
|
+
) -> None:
|
|
49
|
+
"""Send alert to configured destinations.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
message: Alert text
|
|
53
|
+
to: Override routing (name, ID, or list). None = job's default.
|
|
54
|
+
parse_mode: "Markdown", "MarkdownV2", "HTML", or None
|
|
55
|
+
"""
|
|
56
|
+
...
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class AlertMixin:
|
|
60
|
+
"""Mixin providing ctx.alert() for lifecycle hooks.
|
|
61
|
+
|
|
62
|
+
Targets job alert destinations (job._alert_to or telegram.default).
|
|
63
|
+
For health/operator alerts, use health_alert() directly.
|
|
64
|
+
|
|
65
|
+
This mixin expects the class to have an `_alert_sender` attribute
|
|
66
|
+
that implements the AlertSender protocol.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
_alert_sender: AlertSender | None
|
|
70
|
+
|
|
71
|
+
def alert(
|
|
72
|
+
self,
|
|
73
|
+
message: str,
|
|
74
|
+
*,
|
|
75
|
+
to: str | list[str] | None = None,
|
|
76
|
+
parse_mode: str | None = None,
|
|
77
|
+
) -> None:
|
|
78
|
+
"""Send alert to job destinations.
|
|
79
|
+
|
|
80
|
+
Usage:
|
|
81
|
+
def on_success(self, ctx):
|
|
82
|
+
ctx.alert(f"Confirmed: {ctx.receipt.transactionHash.hex()}")
|
|
83
|
+
|
|
84
|
+
Routing:
|
|
85
|
+
- Uses job's configured alert destinations (job._alert_to)
|
|
86
|
+
- Falls back to telegram.default if not set
|
|
87
|
+
- Respects config parse_mode and rate limiting
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
message: Alert text (up to 4096 characters)
|
|
91
|
+
to: Override routing target (name, ID, or list). None = job's default.
|
|
92
|
+
parse_mode: "Markdown", "MarkdownV2", "HTML", or None for config default
|
|
93
|
+
"""
|
|
94
|
+
if self._alert_sender is not None:
|
|
95
|
+
self._alert_sender.send(message, to=to, parse_mode=parse_mode)
|
|
96
|
+
|
|
97
|
+
|
|
31
98
|
# =============================================================================
|
|
32
99
|
# Contract Factory Protocol
|
|
33
100
|
# =============================================================================
|
|
@@ -94,6 +161,20 @@ class BlockContext:
|
|
|
94
161
|
chain_id: int
|
|
95
162
|
|
|
96
163
|
|
|
164
|
+
class CancellationToken:
|
|
165
|
+
"""Cooperative cancellation marker for runner-owned deadlines."""
|
|
166
|
+
|
|
167
|
+
def __init__(self) -> None:
|
|
168
|
+
self._cancelled = False
|
|
169
|
+
|
|
170
|
+
def cancel(self) -> None:
|
|
171
|
+
self._cancelled = True
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def cancelled(self) -> bool:
|
|
175
|
+
return self._cancelled
|
|
176
|
+
|
|
177
|
+
|
|
97
178
|
# =============================================================================
|
|
98
179
|
# Check Context
|
|
99
180
|
# =============================================================================
|
|
@@ -112,9 +193,11 @@ class CheckContext:
|
|
|
112
193
|
block: BlockContext
|
|
113
194
|
kv: KVStore # Read+write allowed
|
|
114
195
|
job_id: str
|
|
115
|
-
rpc: Any #
|
|
196
|
+
rpc: Any # ReadClient/BroadcastClient or similar
|
|
197
|
+
http: Any # ApprovedHttpClient
|
|
116
198
|
logger: "structlog.stdlib.BoundLogger"
|
|
117
199
|
contracts: ContractFactory
|
|
200
|
+
cancellation_token: CancellationToken | None = None
|
|
118
201
|
_db: Any = None # Internal: for log()
|
|
119
202
|
|
|
120
203
|
def log(self, level: str = "info", **fields: Any) -> None:
|
|
@@ -152,7 +235,8 @@ class BuildContext:
|
|
|
152
235
|
trigger: "Trigger"
|
|
153
236
|
job_id: str
|
|
154
237
|
signer_address: str # Signer belongs here, not on RPC
|
|
155
|
-
rpc: Any #
|
|
238
|
+
rpc: Any # ReadClient/BroadcastClient or similar
|
|
239
|
+
http: Any # ApprovedHttpClient
|
|
156
240
|
logger: "structlog.stdlib.BoundLogger"
|
|
157
241
|
contracts: ContractFactory
|
|
158
242
|
kv: KVReader # Read-only
|
|
@@ -184,6 +268,7 @@ class AlertContext:
|
|
|
184
268
|
logger: "structlog.stdlib.BoundLogger"
|
|
185
269
|
contracts: ContractFactory
|
|
186
270
|
kv: KVReader # Read-only
|
|
271
|
+
http: Any | None = None # ApprovedHttpClient (optional)
|
|
187
272
|
|
|
188
273
|
@property
|
|
189
274
|
def has_receipt(self) -> bool:
|
|
@@ -214,7 +299,7 @@ class AlertContext:
|
|
|
214
299
|
|
|
215
300
|
|
|
216
301
|
@dataclass(frozen=True)
|
|
217
|
-
class TriggerContext:
|
|
302
|
+
class TriggerContext(AlertMixin):
|
|
218
303
|
"""Passed to on_trigger. Trigger still exists here.
|
|
219
304
|
|
|
220
305
|
Use for:
|
|
@@ -224,12 +309,15 @@ class TriggerContext:
|
|
|
224
309
|
|
|
225
310
|
Note: No intent exists yet. After this hook, trigger is gone -
|
|
226
311
|
only intent.metadata persists.
|
|
312
|
+
|
|
313
|
+
Alert routing: ctx.alert() → job._alert_to → telegram.default
|
|
227
314
|
"""
|
|
228
315
|
|
|
229
316
|
trigger: "Trigger"
|
|
230
317
|
block: BlockContext
|
|
231
318
|
kv: "KVStore" # Read+write (pre-intent)
|
|
232
319
|
logger: "structlog.stdlib.BoundLogger"
|
|
320
|
+
http: Any # ApprovedHttpClient
|
|
233
321
|
# Alert routing
|
|
234
322
|
job_id: str
|
|
235
323
|
job_name: str
|
|
@@ -239,14 +327,18 @@ class TriggerContext:
|
|
|
239
327
|
# Optional telegram fields
|
|
240
328
|
telegram_bot: "TelegramBot | None" = None
|
|
241
329
|
job_alert_to: list[str] | None = None
|
|
330
|
+
# Alert sender (injected by lifecycle dispatcher)
|
|
331
|
+
_alert_sender: AlertSender | None = field(default=None, repr=False)
|
|
242
332
|
|
|
243
333
|
|
|
244
334
|
@dataclass(frozen=True)
|
|
245
|
-
class SuccessContext:
|
|
335
|
+
class SuccessContext(AlertMixin):
|
|
246
336
|
"""Passed to on_success. No trigger - use intent.metadata.
|
|
247
337
|
|
|
248
338
|
ctx.intent.metadata["reason"] = original trigger.reason
|
|
249
339
|
ctx.intent.metadata[...] = your custom data from build_tx()
|
|
340
|
+
|
|
341
|
+
Alert routing: ctx.alert() → job._alert_to → telegram.default
|
|
250
342
|
"""
|
|
251
343
|
|
|
252
344
|
intent: "TxIntent"
|
|
@@ -255,6 +347,7 @@ class SuccessContext:
|
|
|
255
347
|
block: BlockContext
|
|
256
348
|
kv: "KVReader" # Read-only
|
|
257
349
|
logger: "structlog.stdlib.BoundLogger"
|
|
350
|
+
http: Any # ApprovedHttpClient
|
|
258
351
|
# Alert routing
|
|
259
352
|
job_id: str
|
|
260
353
|
job_name: str
|
|
@@ -264,16 +357,20 @@ class SuccessContext:
|
|
|
264
357
|
# Optional telegram fields
|
|
265
358
|
telegram_bot: "TelegramBot | None" = None
|
|
266
359
|
job_alert_to: list[str] | None = None
|
|
360
|
+
# Alert sender (injected by lifecycle dispatcher)
|
|
361
|
+
_alert_sender: AlertSender | None = field(default=None, repr=False)
|
|
267
362
|
|
|
268
363
|
|
|
269
364
|
@dataclass(frozen=True)
|
|
270
|
-
class FailureContext:
|
|
365
|
+
class FailureContext(AlertMixin):
|
|
271
366
|
"""Passed to on_failure. Intent may be None for pre-intent failures.
|
|
272
367
|
|
|
273
368
|
Pre-intent failures include:
|
|
274
369
|
- check() exception
|
|
275
370
|
- build_tx() exception
|
|
276
371
|
- intent creation failure
|
|
372
|
+
|
|
373
|
+
Alert routing: ctx.alert() → job._alert_to → telegram.default
|
|
277
374
|
"""
|
|
278
375
|
|
|
279
376
|
intent: "TxIntent | None" # None for pre-intent failures
|
|
@@ -284,6 +381,7 @@ class FailureContext:
|
|
|
284
381
|
block: BlockContext
|
|
285
382
|
kv: "KVReader" # Read-only (no side effects during failure handling)
|
|
286
383
|
logger: "structlog.stdlib.BoundLogger"
|
|
384
|
+
http: Any # ApprovedHttpClient
|
|
287
385
|
# Alert routing
|
|
288
386
|
job_id: str
|
|
289
387
|
job_name: str
|
|
@@ -293,6 +391,8 @@ class FailureContext:
|
|
|
293
391
|
# Optional telegram fields
|
|
294
392
|
telegram_bot: "TelegramBot | None" = None
|
|
295
393
|
job_alert_to: list[str] | None = None
|
|
394
|
+
# Alert sender (injected by lifecycle dispatcher)
|
|
395
|
+
_alert_sender: AlertSender | None = field(default=None, repr=False)
|
|
296
396
|
|
|
297
397
|
|
|
298
398
|
# =============================================================================
|
|
@@ -301,9 +401,9 @@ class FailureContext:
|
|
|
301
401
|
|
|
302
402
|
# These are validated in tests to prevent gradual re-growth into god object:
|
|
303
403
|
# - BlockContext: <= 5 fields
|
|
304
|
-
# - CheckContext: <=
|
|
404
|
+
# - CheckContext: <= 8 fields
|
|
305
405
|
# - BuildContext: <= 9 fields
|
|
306
|
-
# - AlertContext: <=
|
|
406
|
+
# - AlertContext: <= 11 fields
|
|
307
407
|
|
|
308
408
|
|
|
309
409
|
__all__ = [
|
|
@@ -316,4 +416,6 @@ __all__ = [
|
|
|
316
416
|
"FailureContext",
|
|
317
417
|
"ContractFactory",
|
|
318
418
|
"ContractHandle",
|
|
419
|
+
"AlertSender",
|
|
420
|
+
"AlertMixin",
|
|
319
421
|
]
|
brawny/model/enums.py
CHANGED
brawny/model/errors.py
CHANGED
|
@@ -98,6 +98,18 @@ class DatabaseError(BrawnyError):
|
|
|
98
98
|
pass
|
|
99
99
|
|
|
100
100
|
|
|
101
|
+
class TransactionFailed(DatabaseError):
|
|
102
|
+
"""Transaction marked rollback-only due to an inner failure."""
|
|
103
|
+
|
|
104
|
+
pass
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class InvariantViolation(DatabaseError):
|
|
108
|
+
"""Internal invariant violated."""
|
|
109
|
+
|
|
110
|
+
pass
|
|
111
|
+
|
|
112
|
+
|
|
101
113
|
class NonceError(BrawnyError):
|
|
102
114
|
"""Nonce management error."""
|
|
103
115
|
|
|
@@ -180,6 +192,12 @@ class DatabaseCircuitBreakerOpenError(BrawnyError):
|
|
|
180
192
|
pass
|
|
181
193
|
|
|
182
194
|
|
|
195
|
+
class CancelledCheckError(BrawnyError):
|
|
196
|
+
"""Check was cancelled by the runner; intent creation should not proceed."""
|
|
197
|
+
|
|
198
|
+
pass
|
|
199
|
+
|
|
200
|
+
|
|
183
201
|
class SimulationReverted(BrawnyError):
|
|
184
202
|
"""Transaction would revert on-chain. Permanent failure - do not retry or broadcast."""
|
|
185
203
|
|