olas-operate-middleware 0.10.19__py3-none-any.whl → 0.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {olas_operate_middleware-0.10.19.dist-info → olas_operate_middleware-0.11.0.dist-info}/METADATA +3 -1
  2. {olas_operate_middleware-0.10.19.dist-info → olas_operate_middleware-0.11.0.dist-info}/RECORD +30 -27
  3. operate/bridge/bridge_manager.py +10 -12
  4. operate/bridge/providers/lifi_provider.py +5 -4
  5. operate/bridge/providers/native_bridge_provider.py +6 -5
  6. operate/bridge/providers/provider.py +22 -87
  7. operate/bridge/providers/relay_provider.py +5 -4
  8. operate/cli.py +446 -168
  9. operate/constants.py +22 -2
  10. operate/keys.py +13 -0
  11. operate/ledger/__init__.py +107 -2
  12. operate/ledger/profiles.py +79 -11
  13. operate/operate_types.py +205 -2
  14. operate/quickstart/run_service.py +6 -10
  15. operate/services/agent_runner.py +5 -3
  16. operate/services/deployment_runner.py +3 -0
  17. operate/services/funding_manager.py +904 -0
  18. operate/services/health_checker.py +4 -4
  19. operate/services/manage.py +183 -310
  20. operate/services/protocol.py +392 -140
  21. operate/services/service.py +81 -5
  22. operate/settings.py +70 -0
  23. operate/utils/__init__.py +0 -29
  24. operate/utils/gnosis.py +79 -24
  25. operate/utils/single_instance.py +226 -0
  26. operate/wallet/master.py +221 -181
  27. operate/wallet/wallet_recovery_manager.py +5 -5
  28. {olas_operate_middleware-0.10.19.dist-info → olas_operate_middleware-0.11.0.dist-info}/WHEEL +0 -0
  29. {olas_operate_middleware-0.10.19.dist-info → olas_operate_middleware-0.11.0.dist-info}/entry_points.txt +0 -0
  30. {olas_operate_middleware-0.10.19.dist-info → olas_operate_middleware-0.11.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,904 @@
1
+ # -*- coding: utf-8 -*-
2
+ # ------------------------------------------------------------------------------
3
+ #
4
+ # Copyright 2025 Valory AG
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ # ------------------------------------------------------------------------------
19
+
20
+ """Funding manager"""
21
+
22
+
23
+ # pylint: disable=too-many-locals,too-many-statements
24
+
25
+ import asyncio
26
+ import threading
27
+ import traceback
28
+ import typing as t
29
+ from concurrent.futures import ThreadPoolExecutor
30
+ from logging import Logger
31
+ from time import time
32
+
33
+ from aea_ledger_ethereum import defaultdict
34
+ from autonomy.chain.base import registry_contracts
35
+ from autonomy.chain.config import CHAIN_PROFILES, ChainType
36
+ from web3 import Web3
37
+
38
+ from operate.constants import (
39
+ DEFAULT_FUNDING_REQUESTS_COOLDOWN_SECONDS,
40
+ MASTER_EOA_PLACEHOLDER,
41
+ MASTER_SAFE_PLACEHOLDER,
42
+ MIN_AGENT_BOND,
43
+ MIN_SECURITY_DEPOSIT,
44
+ NO_STAKING_PROGRAM_ID,
45
+ SERVICE_SAFE_PLACEHOLDER,
46
+ ZERO_ADDRESS,
47
+ )
48
+ from operate.keys import KeysManager
49
+ from operate.ledger import get_currency_denom, get_default_ledger_api
50
+ from operate.ledger.profiles import (
51
+ CONTRACTS,
52
+ DEFAULT_EOA_THRESHOLD,
53
+ DEFAULT_EOA_TOPUPS,
54
+ DEFAULT_EOA_TOPUPS_WITHOUT_SAFE,
55
+ OLAS,
56
+ USDC,
57
+ WRAPPED_NATIVE_ASSET,
58
+ get_asset_name,
59
+ )
60
+ from operate.operate_types import Chain, ChainAmounts, LedgerType, OnChainState
61
+ from operate.services.protocol import EthSafeTxBuilder, StakingManager, StakingState
62
+ from operate.services.service import NON_EXISTENT_TOKEN, Service
63
+ from operate.utils.gnosis import drain_eoa, get_asset_balance, get_owners
64
+ from operate.utils.gnosis import transfer as transfer_from_safe
65
+ from operate.utils.gnosis import transfer_erc20_from_safe
66
+ from operate.wallet.master import InsufficientFundsException, MasterWalletManager
67
+
68
+
69
+ if t.TYPE_CHECKING:
70
+ from operate.services.manage import ServiceManager # pylint: disable=unused-import
71
+
72
+
73
+ class FundingInProgressError(RuntimeError):
74
+ """Raised when an attempt is made to fund a service that is already being funded."""
75
+
76
+
77
+ class FundingManager:
78
+ """FundingManager"""
79
+
80
+ def __init__(
81
+ self,
82
+ wallet_manager: MasterWalletManager,
83
+ logger: Logger,
84
+ funding_requests_cooldown_seconds: int = DEFAULT_FUNDING_REQUESTS_COOLDOWN_SECONDS,
85
+ ) -> None:
86
+ """Initialize funding manager."""
87
+ self.wallet_manager = wallet_manager
88
+ self.logger = logger
89
+ self.funding_requests_cooldown_seconds = funding_requests_cooldown_seconds
90
+ self._lock = threading.Lock()
91
+ self._funding_in_progress: t.Dict[str, bool] = {}
92
+ self._funding_requests_cooldown_until: t.Dict[str, float] = {}
93
+
94
+ def drain_agents_eoas(
95
+ self, service: Service, withdrawal_address: str, chain: Chain
96
+ ) -> None:
97
+ """Drain the funds out of the service agents EOAs."""
98
+ service_config_id = service.service_config_id
99
+ ledger_api = get_default_ledger_api(chain)
100
+ self.logger.info(
101
+ f"Draining service agents {service.name} ({service_config_id=})"
102
+ )
103
+ for agent_address in service.agent_addresses:
104
+ ethereum_crypto = KeysManager().get_crypto_instance(agent_address)
105
+ balance = ledger_api.get_balance(agent_address)
106
+ self.logger.info(
107
+ f"Draining {balance} (approx) {get_currency_denom(chain)} from {agent_address} (agent) to {withdrawal_address}"
108
+ )
109
+ drain_eoa(
110
+ ledger_api=ledger_api,
111
+ crypto=ethereum_crypto,
112
+ withdrawal_address=withdrawal_address,
113
+ chain_id=chain.id,
114
+ )
115
+ self.logger.info(f"{service.name} signer drained")
116
+
117
+ def drain_service_safe( # pylint: disable=too-many-locals
118
+ self, service: Service, withdrawal_address: str, chain: Chain
119
+ ) -> None:
120
+ """Drain the funds out of the service safe."""
121
+ service_config_id = service.service_config_id
122
+ self.logger.info(f"Draining service safe {service.name} ({service_config_id=})")
123
+ chain_config = service.chain_configs[chain.value]
124
+ chain_data = chain_config.chain_data
125
+ ledger_api = get_default_ledger_api(chain)
126
+ withdrawal_address = Web3.to_checksum_address(withdrawal_address)
127
+ service_safe = chain_data.multisig
128
+ wallet = self.wallet_manager.load(chain.ledger_type)
129
+ master_safe = wallet.safes[chain]
130
+ ledger_config = chain_config.ledger_config
131
+ sftxb = EthSafeTxBuilder(
132
+ rpc=ledger_config.rpc,
133
+ wallet=wallet,
134
+ contracts=CONTRACTS[ledger_config.chain],
135
+ chain_type=ChainType(ledger_config.chain.value),
136
+ )
137
+
138
+ owners = get_owners(ledger_api=ledger_api, safe=service_safe)
139
+
140
+ # Drain ERC20 tokens from service Safe
141
+ tokens = {
142
+ WRAPPED_NATIVE_ASSET[chain],
143
+ OLAS[chain],
144
+ USDC[chain],
145
+ } | service.chain_configs[
146
+ chain.value
147
+ ].chain_data.user_params.fund_requirements.keys()
148
+ tokens.discard(ZERO_ADDRESS)
149
+
150
+ for token_address in tokens:
151
+ token_instance = registry_contracts.erc20.get_instance(
152
+ ledger_api=ledger_api,
153
+ contract_address=token_address,
154
+ )
155
+ balance = token_instance.functions.balanceOf(service_safe).call()
156
+ token_name = get_asset_name(chain, token_address)
157
+ if balance == 0:
158
+ self.logger.info(
159
+ f"No {token_name} to drain from service safe: {service_safe}"
160
+ )
161
+ continue
162
+
163
+ self.logger.info(
164
+ f"Draining {balance} {token_name} from {service_safe} (service safe) to {withdrawal_address}"
165
+ )
166
+
167
+ # Safe not swapped
168
+ if set(owners) == set(service.agent_addresses):
169
+ ethereum_crypto = KeysManager().get_crypto_instance(
170
+ service.agent_addresses[0]
171
+ )
172
+ transfer_erc20_from_safe(
173
+ ledger_api=ledger_api,
174
+ crypto=ethereum_crypto,
175
+ safe=chain_data.multisig,
176
+ token=token_address,
177
+ to=withdrawal_address,
178
+ amount=balance,
179
+ )
180
+ elif set(owners) == {master_safe}:
181
+ messages = sftxb.get_safe_b_erc20_transfer_messages(
182
+ safe_b_address=service_safe,
183
+ token=token_address,
184
+ to=withdrawal_address,
185
+ amount=balance,
186
+ )
187
+ tx = sftxb.new_tx()
188
+ for message in messages:
189
+ tx.add(message)
190
+ tx.settle()
191
+
192
+ else:
193
+ raise RuntimeError(
194
+ f"Cannot drain service safe: unrecognized owner set {owners=}"
195
+ )
196
+
197
+ # Drain native asset from service Safe
198
+ balance = ledger_api.get_balance(service_safe)
199
+ if balance == 0:
200
+ self.logger.info(
201
+ f"No {get_currency_denom(chain)} to drain from service safe: {service_safe}"
202
+ )
203
+ else:
204
+ self.logger.info(
205
+ f"Draining {balance} {get_currency_denom(chain)} from {service_safe} (service safe) to {withdrawal_address}"
206
+ )
207
+
208
+ if set(owners) == set(service.agent_addresses):
209
+ ethereum_crypto = KeysManager().get_crypto_instance(
210
+ service.agent_addresses[0]
211
+ )
212
+ transfer_from_safe(
213
+ ledger_api=ledger_api,
214
+ crypto=ethereum_crypto,
215
+ safe=chain_data.multisig,
216
+ to=withdrawal_address,
217
+ amount=balance,
218
+ )
219
+ elif set(owners) == {master_safe}:
220
+ messages = sftxb.get_safe_b_native_transfer_messages(
221
+ safe_b_address=service_safe,
222
+ to=withdrawal_address,
223
+ amount=balance,
224
+ )
225
+ tx = sftxb.new_tx()
226
+ for message in messages:
227
+ tx.add(message)
228
+ tx.settle()
229
+
230
+ else:
231
+ raise RuntimeError(
232
+ f"Cannot drain service safe: unrecognized owner set {owners=}"
233
+ )
234
+
235
+ self.logger.info(f"Service safe {service.name} drained ({service_config_id=})")
236
+
237
+ # -------------------------------------------------------------------------------------
238
+ # -------------------------------------------------------------------------------------
239
+
240
+ def _compute_protocol_asset_requirements(self, service: Service) -> ChainAmounts:
241
+ """Computes the protocol asset requirements to deploy on-chain and stake (if necessary)"""
242
+
243
+ self.logger.info(
244
+ f"[FUNDING MANAGER] Computing protocol asset requirements for service {service.service_config_id}"
245
+ )
246
+ protocol_asset_requirements = ChainAmounts()
247
+
248
+ for chain, chain_config in service.chain_configs.items():
249
+ user_params = chain_config.chain_data.user_params
250
+ number_of_agents = len(service.agent_addresses)
251
+
252
+ requirements: defaultdict = defaultdict(int)
253
+
254
+ if (
255
+ not user_params.use_staking
256
+ or not user_params.staking_program_id
257
+ or user_params.staking_program_id == NO_STAKING_PROGRAM_ID
258
+ ):
259
+ protocol_agent_bonds = (
260
+ max(MIN_AGENT_BOND, user_params.cost_of_bond) * number_of_agents
261
+ )
262
+ protocol_security_deposit = max(
263
+ MIN_SECURITY_DEPOSIT, user_params.cost_of_bond
264
+ )
265
+ staking_agent_bonds = 0
266
+ staking_security_deposit = 0
267
+ else:
268
+ protocol_agent_bonds = MIN_AGENT_BOND * number_of_agents
269
+ protocol_security_deposit = MIN_SECURITY_DEPOSIT
270
+
271
+ staking_manager = StakingManager(chain=Chain(chain))
272
+ staking_params = staking_manager.get_staking_params(
273
+ staking_contract=staking_manager.get_staking_contract(
274
+ staking_program_id=user_params.staking_program_id,
275
+ ),
276
+ )
277
+
278
+ staking_agent_bonds = (
279
+ staking_params["min_staking_deposit"] * number_of_agents
280
+ )
281
+ staking_security_deposit = staking_params["min_staking_deposit"]
282
+ staking_token = staking_params["staking_token"]
283
+ requirements[staking_token] += staking_agent_bonds
284
+ requirements[staking_token] += staking_security_deposit
285
+
286
+ for token, amount in staking_params[
287
+ "additional_staking_tokens"
288
+ ].items():
289
+ requirements[token] = amount
290
+
291
+ requirements[ZERO_ADDRESS] += protocol_agent_bonds
292
+ requirements[ZERO_ADDRESS] += protocol_security_deposit
293
+
294
+ master_safe = self._resolve_master_safe(Chain(chain))
295
+
296
+ protocol_asset_requirements[chain] = {master_safe: dict(requirements)}
297
+
298
+ return protocol_asset_requirements
299
+
300
+ # TODO address this comment
301
+ # This computation assumes the service will be/has been minted with these
302
+ # parameters. Otherwise, these values should be retrieved on-chain as follows:
303
+ # - agent_bonds: by combining the output of ServiceRegistry .getAgentParams .getService
304
+ # and ServiceRegistryTokenUtility .getAgentBond
305
+ # - security_deposit: as the maximum agent bond.
306
+
307
+ def _compute_protocol_bonded_assets( # pylint: disable=too-many-locals,too-many-statements
308
+ self, service: Service
309
+ ) -> ChainAmounts:
310
+ """Computes the bonded assets: current agent bonds and current security deposit"""
311
+
312
+ protocol_bonded_assets = ChainAmounts()
313
+
314
+ for chain, chain_config in service.chain_configs.items():
315
+ bonded_assets: defaultdict = defaultdict(int)
316
+ ledger_config = chain_config.ledger_config
317
+ user_params = chain_config.chain_data.user_params
318
+
319
+ if not self.wallet_manager.exists(ledger_config.chain.ledger_type):
320
+ protocol_bonded_assets[chain] = {
321
+ MASTER_SAFE_PLACEHOLDER: dict(bonded_assets)
322
+ }
323
+ continue
324
+
325
+ wallet = self.wallet_manager.load(ledger_config.chain.ledger_type)
326
+ ledger_api = get_default_ledger_api(Chain(chain))
327
+ staking_manager = StakingManager(Chain(chain))
328
+
329
+ if Chain(chain) not in wallet.safes:
330
+ protocol_bonded_assets[chain] = {
331
+ MASTER_SAFE_PLACEHOLDER: dict(bonded_assets)
332
+ }
333
+ continue
334
+
335
+ master_safe = wallet.safes[Chain(chain)]
336
+
337
+ service_id = chain_config.chain_data.token
338
+ if service_id == NON_EXISTENT_TOKEN:
339
+ protocol_bonded_assets[chain] = {master_safe: dict(bonded_assets)}
340
+ continue
341
+
342
+ # os.environ["CUSTOM_CHAIN_RPC"] = ledger_config.rpc # TODO do we need this?
343
+
344
+ # Determine bonded native amount
345
+ service_registry_address = CHAIN_PROFILES[chain]["service_registry"]
346
+ service_registry = registry_contracts.service_registry.get_instance(
347
+ ledger_api=ledger_api,
348
+ contract_address=service_registry_address,
349
+ )
350
+ service_info = service_registry.functions.getService(service_id).call()
351
+ security_deposit = service_info[0]
352
+ service_state = service_info[6]
353
+ agent_ids = service_info[7]
354
+
355
+ if (
356
+ OnChainState.ACTIVE_REGISTRATION
357
+ <= service_state
358
+ < OnChainState.TERMINATED_BONDED
359
+ ):
360
+ bonded_assets[ZERO_ADDRESS] += security_deposit
361
+
362
+ operator_balance = service_registry.functions.getOperatorBalance(
363
+ master_safe, service_id
364
+ ).call()
365
+ bonded_assets[ZERO_ADDRESS] += operator_balance
366
+
367
+ # Determine bonded token amount for staking programs
368
+ current_staking_program = staking_manager.get_current_staking_program(
369
+ service_id=service_id,
370
+ )
371
+ target_staking_program = user_params.staking_program_id
372
+ staking_contract = staking_manager.get_staking_contract(
373
+ staking_program_id=current_staking_program or target_staking_program,
374
+ )
375
+
376
+ if not staking_contract:
377
+ return dict(bonded_assets)
378
+
379
+ staking_manager = StakingManager(Chain(chain))
380
+ staking_params = staking_manager.get_staking_params(
381
+ staking_contract=staking_manager.get_staking_contract(
382
+ staking_program_id=user_params.staking_program_id,
383
+ ),
384
+ )
385
+
386
+ service_registry_token_utility_address = staking_params[
387
+ "service_registry_token_utility"
388
+ ]
389
+ service_registry_token_utility = (
390
+ registry_contracts.service_registry_token_utility.get_instance(
391
+ ledger_api=ledger_api,
392
+ contract_address=service_registry_token_utility_address,
393
+ )
394
+ )
395
+
396
+ agent_bonds = 0
397
+ for agent_id in agent_ids:
398
+ num_agent_instances = service_registry.functions.getInstancesForAgentId(
399
+ service_id, agent_id
400
+ ).call()[0]
401
+ agent_bond = service_registry_token_utility.functions.getAgentBond(
402
+ service_id, agent_id
403
+ ).call()
404
+ agent_bonds += num_agent_instances * agent_bond
405
+
406
+ if service_state == OnChainState.TERMINATED_BONDED:
407
+ num_agent_instances = service_info[5]
408
+ token_bond = (
409
+ service_registry_token_utility.functions.getOperatorBalance(
410
+ master_safe,
411
+ service_id,
412
+ ).call()
413
+ )
414
+ agent_bonds += num_agent_instances * token_bond
415
+
416
+ security_deposit = 0
417
+ if (
418
+ OnChainState.ACTIVE_REGISTRATION
419
+ <= service_state
420
+ < OnChainState.TERMINATED_BONDED
421
+ ):
422
+ security_deposit = (
423
+ service_registry_token_utility.functions.mapServiceIdTokenDeposit(
424
+ service_id
425
+ ).call()[1]
426
+ )
427
+
428
+ bonded_assets[staking_params["staking_token"]] += agent_bonds
429
+ bonded_assets[staking_params["staking_token"]] += security_deposit
430
+
431
+ staking_state = staking_manager.staking_state(
432
+ service_id=service_id,
433
+ staking_contract=staking_params["staking_contract"],
434
+ )
435
+
436
+ if staking_state in (StakingState.STAKED, StakingState.EVICTED):
437
+ for token, amount in staking_params[
438
+ "additional_staking_tokens"
439
+ ].items():
440
+ bonded_assets[token] += amount
441
+
442
+ protocol_bonded_assets[chain] = {master_safe: dict(bonded_assets)}
443
+
444
+ return protocol_bonded_assets
445
+
446
+ @staticmethod
447
+ def _compute_shortfalls(
448
+ balances: ChainAmounts,
449
+ thresholds: ChainAmounts,
450
+ topups: ChainAmounts,
451
+ ) -> ChainAmounts:
452
+ """Compute shortfall per chain/address/asset: if balance < threshold, shortfall = topup - balance, else 0"""
453
+ shortfalls = ChainAmounts()
454
+
455
+ for chain, addresses in thresholds.items():
456
+ shortfalls.setdefault(chain, {})
457
+ for address, assets in addresses.items():
458
+ shortfalls[chain].setdefault(address, {})
459
+ for asset, threshold in assets.items():
460
+ balance = balances.get(chain, {}).get(address, {}).get(asset, 0)
461
+ topup = topups.get(chain, {}).get(address, {}).get(asset, 0)
462
+ if balance < threshold:
463
+ shortfalls[chain][address][asset] = max(topup - balance, 0)
464
+ else:
465
+ shortfalls[chain][address][asset] = 0
466
+
467
+ return shortfalls
468
+
469
+ def _resolve_master_eoa(self, chain: Chain) -> str:
470
+ if self.wallet_manager.exists(chain.ledger_type):
471
+ wallet = self.wallet_manager.load(chain.ledger_type)
472
+ return wallet.address
473
+ return MASTER_EOA_PLACEHOLDER
474
+
475
+ def _resolve_master_safe(self, chain: Chain) -> str:
476
+ if self.wallet_manager.exists(chain.ledger_type):
477
+ wallet = self.wallet_manager.load(chain.ledger_type)
478
+ if chain in wallet.safes:
479
+ return wallet.safes[chain]
480
+ return MASTER_SAFE_PLACEHOLDER
481
+
482
+ def _aggregate_as_master_safe_amounts(self, *amounts: ChainAmounts) -> ChainAmounts:
483
+ output = ChainAmounts()
484
+ for amts in amounts:
485
+ for chain_str, addresses in amts.items():
486
+ chain = Chain(chain_str)
487
+ master_safe = self._resolve_master_safe(chain)
488
+ master_safe_dict = output.setdefault(chain_str, {}).setdefault(
489
+ master_safe, {}
490
+ )
491
+ for _, assets in addresses.items():
492
+ for asset, amount in assets.items():
493
+ master_safe_dict[asset] = (
494
+ master_safe_dict.get(asset, 0) + amount
495
+ )
496
+
497
+ return output
498
+
499
+ def _split_excess_assets_master_eoa_balances(
500
+ self, balances: ChainAmounts
501
+ ) -> t.Tuple[ChainAmounts, ChainAmounts]:
502
+ """Splits excess balances from master EOA only on chains without a Master Safe."""
503
+ excess_balance = ChainAmounts()
504
+ remaining_balance = ChainAmounts()
505
+
506
+ for chain_str, addresses in balances.items():
507
+ chain = Chain(chain_str)
508
+ master_safe = self._resolve_master_safe(chain)
509
+ for address, assets in addresses.items():
510
+ for asset, amount in assets.items():
511
+ if master_safe == MASTER_SAFE_PLACEHOLDER:
512
+ remaining = min(
513
+ amount, DEFAULT_EOA_TOPUPS[chain].get(asset, 0)
514
+ ) # When transferring, the Master Safe will be already created, that is why we are only retaining DEFAULT_EOA_TOPUPS
515
+ excess = amount - remaining
516
+ else:
517
+ remaining = amount
518
+ excess = 0
519
+
520
+ excess_balance.setdefault(chain_str, {}).setdefault(
521
+ master_safe, {}
522
+ )[asset] = excess
523
+ remaining_balance.setdefault(chain_str, {}).setdefault(address, {})[
524
+ asset
525
+ ] = remaining
526
+
527
+ return excess_balance, remaining_balance
528
+
529
+ @staticmethod
530
+ def _split_critical_eoa_shortfalls(
531
+ balances: ChainAmounts, shortfalls: ChainAmounts
532
+ ) -> t.Tuple[ChainAmounts, ChainAmounts]:
533
+ """Splits critical EOA shortfalls in two: the first split containins the native shortfalls whose balance is < threshold / 2. The second one, contains the remaining shortfalls. This is to ensure EOA operational balance."""
534
+ critical_shortfalls = ChainAmounts()
535
+ remaining_shortfalls = ChainAmounts()
536
+
537
+ for chain_str, addresses in shortfalls.items():
538
+ chain = Chain(chain_str)
539
+ for address, assets in addresses.items():
540
+ for asset, amount in assets.items():
541
+ if asset == ZERO_ADDRESS and balances[chain_str][address][
542
+ asset
543
+ ] < int(
544
+ DEFAULT_EOA_TOPUPS[chain][asset] / 4
545
+ ): # TODO Ensure that this is enough to pay a transfer tx at least.
546
+ critical_shortfalls.setdefault(chain_str, {}).setdefault(
547
+ address, {}
548
+ )[asset] = amount
549
+ remaining_shortfalls.setdefault(chain_str, {}).setdefault(
550
+ address, {}
551
+ )[asset] = 0
552
+ else:
553
+ critical_shortfalls.setdefault(chain_str, {}).setdefault(
554
+ address, {}
555
+ )[asset] = 0
556
+ remaining_shortfalls.setdefault(chain_str, {}).setdefault(
557
+ address, {}
558
+ )[asset] = amount
559
+
560
+ return critical_shortfalls, remaining_shortfalls
561
+
562
+ def _get_master_safe_balances(self, thresholds: ChainAmounts) -> ChainAmounts:
563
+ output = ChainAmounts()
564
+ for chain_str, addresses in thresholds.items():
565
+ chain = Chain(chain_str)
566
+ master_safe = self._resolve_master_safe(chain)
567
+ master_safe_dict = output.setdefault(chain_str, {}).setdefault(
568
+ master_safe, {}
569
+ )
570
+ for _, assets in addresses.items():
571
+ for asset, _ in assets.items():
572
+ master_safe_dict[asset] = get_asset_balance(
573
+ ledger_api=get_default_ledger_api(chain),
574
+ asset_address=asset,
575
+ address=master_safe,
576
+ raise_on_invalid_address=False,
577
+ )
578
+
579
+ return output
580
+
581
+ def _get_master_eoa_balances(self, thresholds: ChainAmounts) -> ChainAmounts:
582
+ output = ChainAmounts()
583
+ for chain_str, addresses in thresholds.items():
584
+ chain = Chain(chain_str)
585
+ master_eoa = self._resolve_master_eoa(chain)
586
+ master_eoa_dict = output.setdefault(chain_str, {}).setdefault(
587
+ master_eoa, {}
588
+ )
589
+ for _, assets in addresses.items():
590
+ for asset, _ in assets.items():
591
+ master_eoa_dict[asset] = get_asset_balance(
592
+ ledger_api=get_default_ledger_api(chain),
593
+ asset_address=asset,
594
+ address=master_eoa,
595
+ raise_on_invalid_address=False,
596
+ )
597
+
598
+ return output
599
+
600
+ def fund_master_eoa(self) -> None:
601
+ """Fund Master EOA"""
602
+ if not self.wallet_manager.exists(LedgerType.ETHEREUM):
603
+ self.logger.warning(
604
+ "[FUNDING MANAGER] Cannot fund Master EOA: No Ethereum wallet available."
605
+ )
606
+ return
607
+
608
+ master_wallet = self.wallet_manager.load(
609
+ ledger_type=LedgerType.ETHEREUM
610
+ ) # Only for ethereum for now
611
+ self.logger.info(
612
+ f"[FUNDING MANAGER] Funding Master EOA {master_wallet.address}"
613
+ )
614
+ master_eoa_topups = ChainAmounts(
615
+ {
616
+ chain.value: {
617
+ self._resolve_master_eoa(chain): dict(DEFAULT_EOA_TOPUPS[chain])
618
+ }
619
+ for chain in master_wallet.safes
620
+ }
621
+ )
622
+ master_eoa_balances = self._get_master_eoa_balances(master_eoa_topups)
623
+ master_eoa_shortfalls = self._compute_shortfalls(
624
+ balances=master_eoa_balances,
625
+ thresholds=master_eoa_topups * DEFAULT_EOA_THRESHOLD,
626
+ topups=master_eoa_topups,
627
+ )
628
+ self.fund_chain_amounts(master_eoa_shortfalls)
629
+
630
+ def funding_requirements(self, service: Service) -> t.Dict:
631
+ """Funding requirements"""
632
+ balances: ChainAmounts
633
+ protocol_bonded_assets: ChainAmounts
634
+ protocol_asset_requirements: ChainAmounts
635
+ refill_requirements: ChainAmounts
636
+ total_requirements: ChainAmounts
637
+ chains = [Chain(chain_str) for chain_str in service.chain_configs.keys()]
638
+
639
+ # Protocol shortfall
640
+ protocol_thresholds = self._compute_protocol_asset_requirements(service)
641
+ protocol_balances = self._compute_protocol_bonded_assets(service)
642
+ protocol_topups = protocol_thresholds
643
+ protocol_shortfalls = self._compute_shortfalls(
644
+ balances=protocol_balances,
645
+ thresholds=protocol_thresholds,
646
+ topups=protocol_topups,
647
+ )
648
+
649
+ # Initial service shortfall
650
+ # We assume that if the service safe is created in any chain,
651
+ # we have requested the funding already.
652
+ service_initial_topup = service.get_initial_funding_amounts()
653
+ if not all(
654
+ SERVICE_SAFE_PLACEHOLDER in addresses
655
+ for addresses in service_initial_topup.values()
656
+ ):
657
+ service_initial_shortfalls = ChainAmounts()
658
+ else:
659
+ service_initial_shortfalls = service_initial_topup
660
+
661
+ # Service funding requests
662
+ service_config_id = service.service_config_id
663
+ funding_in_progress = self._funding_in_progress.get(service_config_id, False)
664
+ now = time()
665
+ if funding_in_progress:
666
+ funding_requests = ChainAmounts()
667
+ funding_requests_cooldown = False
668
+ elif now < self._funding_requests_cooldown_until.get(service_config_id, 0):
669
+ funding_requests = ChainAmounts()
670
+ funding_requests_cooldown = True
671
+ else:
672
+ funding_requests = service.get_funding_requests()
673
+ funding_requests_cooldown = False
674
+
675
+ # Master EOA shortfall
676
+ master_eoa_topups = ChainAmounts()
677
+ for chain in chains:
678
+ chain_str = chain.value
679
+ master_eoa = self._resolve_master_eoa(chain)
680
+ master_safe = self._resolve_master_safe(chain)
681
+
682
+ if master_safe != MASTER_SAFE_PLACEHOLDER:
683
+ master_eoa_topups[chain_str] = {
684
+ master_eoa: dict(DEFAULT_EOA_TOPUPS[chain])
685
+ }
686
+ else:
687
+ master_eoa_topups[chain_str] = {
688
+ master_eoa: dict(DEFAULT_EOA_TOPUPS_WITHOUT_SAFE[chain])
689
+ }
690
+
691
+ # Set the topup for MasterEOA for remaining tokens to 0 if they don't exist
692
+ # This ensures that the balances of MasterEOA are collected for relevant tokens
693
+ all_assets = {ZERO_ADDRESS} | {
694
+ asset
695
+ for addresses in (
696
+ protocol_topups[chain_str],
697
+ service_initial_topup[chain_str],
698
+ )
699
+ for assets in addresses.values()
700
+ for asset in assets
701
+ }
702
+ for asset in all_assets:
703
+ master_eoa_topups[chain_str][master_eoa].setdefault(asset, 0)
704
+
705
+ master_eoa_thresholds = master_eoa_topups // 2
706
+ master_eoa_balances = self._get_master_eoa_balances(master_eoa_thresholds)
707
+
708
+ # BEGIN Bridging patch: remove excess balances for chains without a Safe:
709
+ (
710
+ excess_master_eoa_balances,
711
+ master_eoa_balances,
712
+ ) = self._split_excess_assets_master_eoa_balances(master_eoa_balances)
713
+ # END Bridging patch
714
+
715
+ master_eoa_shortfalls = self._compute_shortfalls(
716
+ balances=master_eoa_balances,
717
+ thresholds=master_eoa_thresholds,
718
+ topups=master_eoa_topups,
719
+ )
720
+
721
+ (
722
+ master_eoa_critical_shortfalls,
723
+ master_eoa_shortfalls,
724
+ ) = self._split_critical_eoa_shortfalls(
725
+ balances=master_eoa_balances, shortfalls=master_eoa_shortfalls
726
+ )
727
+
728
+ # Master Safe shortfall
729
+ master_safe_thresholds = self._aggregate_as_master_safe_amounts(
730
+ master_eoa_shortfalls,
731
+ protocol_shortfalls,
732
+ service_initial_shortfalls,
733
+ )
734
+ master_safe_topup = master_safe_thresholds
735
+ master_safe_balances = ChainAmounts.add(
736
+ self._get_master_safe_balances(master_safe_thresholds),
737
+ self._aggregate_as_master_safe_amounts(excess_master_eoa_balances),
738
+ )
739
+ master_safe_shortfalls = self._compute_shortfalls(
740
+ balances=master_safe_balances,
741
+ thresholds=master_safe_thresholds,
742
+ topups=master_safe_topup,
743
+ )
744
+
745
+ # Prepare output values
746
+ protocol_bonded_assets = protocol_balances
747
+ protocol_asset_requirements = protocol_thresholds
748
+ refill_requirements = ChainAmounts.add(
749
+ master_eoa_critical_shortfalls,
750
+ master_safe_shortfalls,
751
+ )
752
+ total_requirements = ChainAmounts.add( # TODO Review if this is correct
753
+ master_eoa_critical_shortfalls,
754
+ master_safe_thresholds,
755
+ )
756
+ balances = ChainAmounts.add(
757
+ master_eoa_balances,
758
+ master_safe_balances,
759
+ )
760
+
761
+ # Compute boolean flags
762
+ is_refill_required = any(
763
+ amount > 0
764
+ for address in refill_requirements.values()
765
+ for assets in address.values()
766
+ for amount in assets.values()
767
+ )
768
+
769
+ allow_start_agent = True
770
+ if any(
771
+ MASTER_SAFE_PLACEHOLDER in addresses
772
+ for addresses in refill_requirements.values()
773
+ ) or any(
774
+ amount > 0
775
+ for address in master_eoa_critical_shortfalls.values()
776
+ for assets in address.values()
777
+ for amount in assets.values()
778
+ ):
779
+ allow_start_agent = False
780
+
781
+ return {
782
+ "balances": balances,
783
+ "bonded_assets": protocol_bonded_assets,
784
+ "total_requirements": total_requirements,
785
+ "refill_requirements": refill_requirements,
786
+ "protocol_asset_requirements": protocol_asset_requirements,
787
+ "is_refill_required": is_refill_required,
788
+ "allow_start_agent": allow_start_agent,
789
+ "agent_funding_requests": funding_requests,
790
+ "agent_funding_requests_cooldown": funding_requests_cooldown,
791
+ "agent_funding_in_progress": funding_in_progress,
792
+ }
793
+
794
+ def fund_service_initial(self, service: Service) -> None:
795
+ """Fund service initially"""
796
+ self.fund_chain_amounts(service.get_initial_funding_amounts())
797
+
798
+ def fund_chain_amounts(self, amounts: ChainAmounts) -> None:
799
+ """Fund chain amounts"""
800
+ required = self._aggregate_as_master_safe_amounts(amounts)
801
+ balances = self._get_master_safe_balances(required)
802
+
803
+ if balances < required:
804
+ raise InsufficientFundsException(
805
+ f"Insufficient funds in Master Safe to perform funding. Required: {amounts}, Available: {balances}"
806
+ )
807
+
808
+ for chain_str, addresses in amounts.items():
809
+ chain = Chain(chain_str)
810
+ wallet = self.wallet_manager.load(chain.ledger_type)
811
+ for address, assets in addresses.items():
812
+ for asset, amount in assets.items():
813
+ if amount <= 0:
814
+ continue
815
+
816
+ self.logger.info(
817
+ f"[FUNDING MANAGER] Funding {amount} of {asset} to {address} on chain {chain.value} from Master Safe {wallet.safes.get(chain, 'N/A')}"
818
+ )
819
+ wallet.transfer(
820
+ chain=chain,
821
+ to=address,
822
+ asset=asset,
823
+ amount=amount,
824
+ from_safe=True,
825
+ )
826
+
827
+ def fund_service(self, service: Service, amounts: ChainAmounts) -> None:
828
+ """Fund service-related wallets."""
829
+ service_config_id = service.service_config_id
830
+
831
+ # Atomic, thread-safe get-and-set of the _funding_in_progress boolean.
832
+ # This ensures only one funding operation per service at a time, and
833
+ # any call to fund_service while a funding operation is in progress will
834
+ # raise a FundingInProgressError (instead of blocking and piling up calls).
835
+ with self._lock:
836
+ if self._funding_in_progress.get(service_config_id, False):
837
+ raise FundingInProgressError(
838
+ f"Funding already in progress for service {service_config_id}."
839
+ )
840
+ self._funding_in_progress[service_config_id] = True
841
+
842
+ try:
843
+ for chain_str, addresses in amounts.items():
844
+ for address in addresses:
845
+ if (
846
+ address not in service.agent_addresses
847
+ and address
848
+ != service.chain_configs[chain_str].chain_data.multisig
849
+ ):
850
+ raise ValueError(
851
+ f"Failed to fund from Master Safe: Address {address} is not an agent EOA or service Safe for service {service.service_config_id}."
852
+ )
853
+
854
+ self.fund_chain_amounts(amounts)
855
+ self._funding_requests_cooldown_until[service_config_id] = (
856
+ time() + self.funding_requests_cooldown_seconds
857
+ )
858
+ finally:
859
+ with self._lock:
860
+ self._funding_in_progress[service_config_id] = False
861
+
862
+ async def funding_job(
863
+ self,
864
+ service_manager: "ServiceManager",
865
+ loop: t.Optional[asyncio.AbstractEventLoop] = None,
866
+ ) -> None:
867
+ """Start a background funding job."""
868
+ loop = loop or asyncio.get_event_loop()
869
+ with ThreadPoolExecutor() as executor:
870
+ last_claim = 0.0
871
+ last_master_eoa_funding = 0.0
872
+ while True:
873
+ # try claiming rewards every hour
874
+ if last_claim + 3600 < time():
875
+ try:
876
+ await loop.run_in_executor(
877
+ executor,
878
+ service_manager.claim_all_on_chain_from_safe,
879
+ )
880
+ except Exception: # pylint: disable=broad-except
881
+ self.logger.info(
882
+ f"Error occured while claiming rewards\n{traceback.format_exc()}"
883
+ )
884
+ last_claim = time()
885
+
886
+ # fund Master EOA every hour
887
+ if last_master_eoa_funding + 3600 < time():
888
+ try:
889
+ await loop.run_in_executor(
890
+ executor,
891
+ self.fund_master_eoa,
892
+ )
893
+ except Exception: # pylint: disable=broad-except
894
+ self.logger.info(
895
+ f"Error occured while funding Master EOA\n{traceback.format_exc()}"
896
+ )
897
+ last_master_eoa_funding = time()
898
+
899
+ await asyncio.sleep(60)
900
+
901
+ # TODO Below this line - pending finish funding Job for Master EOA
902
+ # TODO cache _resolve methods to avoid loading multiple times file.
903
+ # TODO refactor: move protocol_ methods to protocol class, refactor to accomodate arbitrary owner and operators,
904
+ # refactor to manage multiple chains with different master safes, etc.