olas-operate-middleware 0.10.20__py3-none-any.whl → 0.11.0__py3-none-any.whl

operate/constants.py

@@ -33,20 +33,37 @@ DEPLOYMENT_JSON = "deployment.json"
 CONFIG_JSON = "config.json"
 USER_JSON = "user.json"
 HEALTHCHECK_JSON = "healthcheck.json"
+VERSION_FILE = "operate.version"
+SETTINGS_JSON = "settings.json"
+FUNDING_REQUIREMENTS_JSON = "funding_requirements.json"
+DEFAULT_TOPUP_THRESHOLD = 0.5
+
+MASTER_EOA_PLACEHOLDER = "master_eoa"
+MASTER_SAFE_PLACEHOLDER = "master_safe"
+AGENT_EOA_PLACEHOLDER = "agent_eoa"
+SERVICE_SAFE_PLACEHOLDER = "service_safe"
+
+FERNET_KEY_LENGTH = 32
 
 AGENT_PERSISTENT_STORAGE_DIR = "persistent_data"
 AGENT_PERSISTENT_STORAGE_ENV_VAR = "STORE_PATH"
 AGENT_LOG_DIR = "benchmarks"
 AGENT_LOG_ENV_VAR = "LOG_DIR"
+AGENT_RUNNER_PREFIX = "agent_runner"
 
 ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"
 
+MIN_AGENT_BOND = 1
+MIN_SECURITY_DEPOSIT = 1
+
 ON_CHAIN_INTERACT_TIMEOUT = 120.0
 ON_CHAIN_INTERACT_RETRIES = 12
 ON_CHAIN_INTERACT_SLEEP = 5.0
 MIN_PASSWORD_LENGTH = 8
+DEFAULT_FUNDING_REQUESTS_COOLDOWN_SECONDS = 300  # Seconds to wait after an agent has been funded during which it will not be asked for fund requirements again
 
 HEALTH_CHECK_URL = "http://127.0.0.1:8716/healthcheck"  # possible DNS issues on windows so use IP address
+AGENT_FUNDS_STATUS_URL = "http://127.0.0.1:8716/funds-status"
 SAFE_WEBAPP_URL = "https://app.safe.global/home?safe=gno:"
 TM_CONTROL_URL = "http://localhost:8080"
 IPFS_ADDRESS = "https://gateway.autonolas.tech/ipfs/f01701220{hash}"
@@ -63,3 +80,6 @@ NO_STAKING_PROGRAM_ID = "no_staking"
 
 DEPLOYMENT_START_TRIES_NUM = 3
 IPFS_CHECK_URL = "https://gateway.autonolas.tech/ipfs/bafybeigcllaxn4ycjjvika3zd6eicksuriez2wtg67gx7pamhcazl3tv54/echo/README.md"
+MSG_NEW_PASSWORD_MISSING = "'new_password' is required."  # nosec
+MSG_INVALID_PASSWORD = "Password is not valid."  # nosec
+MSG_INVALID_MNEMONIC = "Seed phrase is not valid."  # nosec

operate/ledger/__init__.py

@@ -21,9 +21,13 @@
 
 import os
 import typing as t
+from copy import deepcopy
 from math import ceil
 
 from aea.crypto.base import LedgerApi
+from aea.crypto.registries import make_ledger_api
+from aea_ledger_ethereum import DEFAULT_GAS_PRICE_STRATEGIES, EIP1559, GWEI, to_wei
+from web3.middleware import geth_poa_middleware
 
 from operate.operate_types import Chain
 
@@ -89,15 +93,17 @@ CURRENCY_SMALLEST_UNITS = {
     Chain.SOLANA: "Lamport",
 }
 
+NATIVE_CURRENCY_DECIMALS = 18
+
 
 def get_default_rpc(chain: Chain) -> str:
     """Get default RPC chain type."""
-    return DEFAULT_RPCS.get(chain, ETHEREUM_RPC)
+    return DEFAULT_RPCS[chain]
 
 
 def get_currency_denom(chain: Chain) -> str:
     """Get currency denom by chain type."""
-    return CURRENCY_DENOMS.get(chain, "ETH")
+    return CURRENCY_DENOMS.get(chain, "NATIVE")
 
 
 def get_currency_smallest_unit(chain: Chain) -> str:
@@ -105,6 +111,50 @@ def get_currency_smallest_unit(chain: Chain) -> str:
     return CURRENCY_SMALLEST_UNITS.get(chain, "Wei")
 
 
+DEFAULT_LEDGER_APIS: t.Dict[Chain, LedgerApi] = {}
+
+
+def make_chain_ledger_api(
+    chain: Chain,
+    rpc: t.Optional[str] = None,
+) -> LedgerApi:
+    """Get default RPC chain type."""
+
+    if chain not in DEFAULT_LEDGER_APIS:
+        if chain == Chain.SOLANA:  # TODO: Complete when Solana is supported
+            raise NotImplementedError("Solana not yet supported.")
+
+        gas_price_strategies = deepcopy(DEFAULT_GAS_PRICE_STRATEGIES)
+        if chain in (Chain.BASE, Chain.MODE, Chain.OPTIMISM):
+            gas_price_strategies[EIP1559]["fallback_estimate"]["maxFeePerGas"] = to_wei(
+                5, GWEI
+            )
+
+        ledger_api = make_ledger_api(
+            chain.ledger_type.name.lower(),
+            address=rpc or get_default_rpc(chain=chain),
+            chain_id=chain.id,
+            gas_price_strategies=gas_price_strategies,
+            poa_chain=chain == Chain.POLYGON,
+        )
+
+        if chain == Chain.OPTIMISM:
+            ledger_api.api.middleware_onion.inject(geth_poa_middleware, layer=0)
+
+        DEFAULT_LEDGER_APIS[chain] = ledger_api
+
+    return DEFAULT_LEDGER_APIS[chain]
+
+
+def get_default_ledger_api(chain: Chain) -> LedgerApi:
+    """Get default RPC chain type."""
+    if chain not in DEFAULT_LEDGER_APIS:
+        DEFAULT_LEDGER_APIS[chain] = make_chain_ledger_api(
+            chain=chain, rpc=get_default_rpc(chain=chain)
+        )
+    return DEFAULT_LEDGER_APIS[chain]
+
+
 GAS_ESTIMATE_FALLBACK_ADDRESSES = [
     "0x000000000000000000000000000000000000dEaD",
     "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE",  # nosec
@@ -137,7 +187,7 @@ def update_tx_with_gas_pricing(tx: t.Dict, ledger_api: LedgerApi) -> None:
 # TODO This gas management should be done at a lower level in the library
 def update_tx_with_gas_estimate(tx: t.Dict, ledger_api: LedgerApi) -> None:
     """Update transaction with gas estimate."""
-    print(f"[PROVIDER] Trying to update transaction gas {tx['from']=} {tx['gas']=}.")
+    print(f"[LEDGER] Trying to update transaction gas {tx['from']=} {tx['gas']=}.")
     original_from = tx["from"]
     original_gas = tx.get("gas", 1)
 
@@ -146,11 +196,11 @@ def update_tx_with_gas_estimate(tx: t.Dict, ledger_api: LedgerApi) -> None:
         tx["gas"] = 1
         ledger_api.update_with_gas_estimate(tx)
         if tx["gas"] > 1:
-            print(f"[PROVIDER] Gas estimated successfully {tx['from']=} {tx['gas']=}.")
+            print(f"[LEDGER] Gas estimated successfully {tx['from']=} {tx['gas']=}.")
             break
 
     tx["from"] = original_from
     if tx["gas"] == 1:
         tx["gas"] = original_gas
-        print(f"[PROVIDER] Unable to estimate gas. Restored {tx['gas']=}.")
+        print(f"[LEDGER] Unable to estimate gas. Restored {tx['gas']=}.")
     tx["gas"] = ceil(tx["gas"] * GAS_ESTIMATE_BUFFER)

operate/ledger/profiles.py

@@ -20,11 +20,18 @@
 """Chain profiles."""
 
 import typing as t
+from functools import cache
 
+from autonomy.chain.base import registry_contracts
 from autonomy.chain.constants import CHAIN_PROFILES, DEFAULT_MULTISEND
 
 from operate.constants import NO_STAKING_PROGRAM_ID, ZERO_ADDRESS
-from operate.ledger import CHAINS
+from operate.ledger import (
+    CHAINS,
+    NATIVE_CURRENCY_DECIMALS,
+    get_currency_denom,
+    get_default_ledger_api,
+)
 from operate.operate_types import Chain, ContractAddresses
 
 
@@ -177,7 +184,22 @@ WRAPPED_NATIVE_ASSET = {
     Chain.POLYGON: "0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270",
 }
 
-ERC20_TOKENS = [OLAS, USDC, WRAPPED_NATIVE_ASSET]
+ERC20_TOKENS = {
+    "OLAS": OLAS,
+    "USDC": USDC,
+    "WRAPPED_NATIVE": WRAPPED_NATIVE_ASSET,
+}
+
+DUST = {
+    Chain.ARBITRUM_ONE: int(1e14),
+    Chain.BASE: int(1e14),
+    Chain.CELO: int(1e15),
+    Chain.ETHEREUM: int(1e14),
+    Chain.GNOSIS: int(1e15),
+    Chain.MODE: int(1e14),
+    Chain.OPTIMISM: int(1e14),
+    Chain.POLYGON: int(1e14),
+}
 
 DEFAULT_NEW_SAFE_FUNDS: t.Dict[Chain, t.Dict[str, int]] = {
     Chain.ARBITRUM_ONE: {
@@ -206,17 +228,24 @@ DEFAULT_NEW_SAFE_FUNDS: t.Dict[Chain, t.Dict[str, int]] = {
     },
 }
 
-DEFAULT_MASTER_EOA_FUNDS = {
-    Chain.ARBITRUM_ONE: {ZERO_ADDRESS: 5_000_000_000_000_000},
-    Chain.BASE: {ZERO_ADDRESS: 5_000_000_000_000_000},
-    Chain.CELO: {ZERO_ADDRESS: 1_500_000_000_000_000_000},
-    Chain.ETHEREUM: {ZERO_ADDRESS: 20_000_000_000_000_000},
-    Chain.GNOSIS: {ZERO_ADDRESS: 1_500_000_000_000_000_000},
-    Chain.MODE: {ZERO_ADDRESS: 500_000_000_000_000},
-    Chain.OPTIMISM: {ZERO_ADDRESS: 5_000_000_000_000_000},
-    Chain.POLYGON: {ZERO_ADDRESS: 1_500_000_000_000_000_000},
+DEFAULT_EOA_TOPUPS = {
+    Chain.ARBITRUM_ONE: {ZERO_ADDRESS: 2_500_000_000_000_000},
+    Chain.BASE: {ZERO_ADDRESS: 2_500_000_000_000_000},
+    Chain.CELO: {ZERO_ADDRESS: 750_000_000_000_000_000},
+    Chain.ETHEREUM: {ZERO_ADDRESS: 10_000_000_000_000_000},
+    Chain.GNOSIS: {ZERO_ADDRESS: 750_000_000_000_000_000},
+    Chain.MODE: {ZERO_ADDRESS: 250_000_000_000_000},
+    Chain.OPTIMISM: {ZERO_ADDRESS: 2_500_000_000_000_000},
+    Chain.POLYGON: {ZERO_ADDRESS: 750_000_000_000_000_000},
+}
+
+DEFAULT_EOA_TOPUPS_WITHOUT_SAFE = {
+    chain: {asset: amount * 2 for asset, amount in amounts.items()}
+    for chain, amounts in DEFAULT_EOA_TOPUPS.items()
 }
 
+DEFAULT_EOA_THRESHOLD = 0.5
+
 EXPLORER_URL = {
     Chain.ARBITRUM_ONE: {
         "tx": "https://arbiscan.io/tx/{tx_hash}",
@@ -253,6 +282,45 @@ EXPLORER_URL = {
 }
 
 
+@cache
+def get_asset_name(chain: Chain, asset_address: str) -> str:
+    """Get token name."""
+    if asset_address == ZERO_ADDRESS:
+        return get_currency_denom(chain)
+
+    if WRAPPED_NATIVE_ASSET.get(chain) == asset_address:
+        return f"W{get_currency_denom(chain)}"
+
+    for symbol, tokens in ERC20_TOKENS.items():
+        if tokens.get(chain) == asset_address:
+            return symbol
+
+    return asset_address
+
+
+@cache
+def get_asset_decimals(chain: Chain, asset_address: str) -> int:
+    """Get token decimals."""
+    if asset_address == ZERO_ADDRESS:
+        return NATIVE_CURRENCY_DECIMALS
+    erc20_token = registry_contracts.erc20.get_instance(
+        ledger_api=get_default_ledger_api(chain),
+        contract_address=asset_address,
+    )
+    return erc20_token.functions.decimals().call()
+
+
+def format_asset_amount(
+    chain: Chain, asset_address: str, amount: int, precision: int = 4
+) -> str:
+    """Convert smallest unit to human-readable string with token symbol."""
+    decimals = get_asset_decimals(chain, asset_address)
+    symbol = get_asset_name(chain, asset_address)
+    value = amount / (10**decimals)
+    return f"{value:.{precision}f} {symbol}"
+
+
+# TODO: Deprecate in favour of StakingManager method
 def get_staking_contract(
     chain: str, staking_program_id: t.Optional[str]
 ) -> t.Optional[str]:

operate/operate_types.py

@@ -19,16 +19,22 @@
 
 """Types module."""
 
+import base64
+import copy
 import enum
 import os
 import typing as t
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from pathlib import Path
 
+import argon2
+from aea_ledger_ethereum import cast
 from autonomy.chain.config import ChainType
 from autonomy.chain.constants import CHAIN_NAME_TO_CHAIN_ID
+from cryptography.fernet import Fernet
 from typing_extensions import TypedDict
 
-from operate.constants import NO_STAKING_PROGRAM_ID
+from operate.constants import FERNET_KEY_LENGTH, NO_STAKING_PROGRAM_ID
 from operate.resource import LocalResource
 
 
@@ -55,6 +61,11 @@ class LedgerType(str, enum.Enum):
         """Key filename."""
         return f"{self.name.lower()}.txt"
 
+    @property
+    def mnemonic_file(self) -> str:
+        """Mnemonic filename."""
+        return f"{self.name.lower()}.mnemonic.json"
+
     @classmethod
     def from_id(cls, chain_id: int) -> "LedgerType":
         """Load from chain ID."""
@@ -323,3 +334,195 @@ class MechMarketplaceConfig:
     mech_marketplace_address: str
     priority_mech_address: str
     priority_mech_service_id: int
+
+
+class Version:
+    """Version class."""
+
+    def __init__(self, version: str) -> None:
+        """Initialize the version."""
+        version = version.strip()
+        version_parts = version.split(".") if version else []
+        self.major = int(version_parts[0]) if len(version_parts) > 0 else 0
+        self.minor = int(version_parts[1]) if len(version_parts) > 1 else 0
+        self.patch = int(version_parts[2]) if len(version_parts) > 2 else 0
+
+    def __str__(self) -> str:
+        """String representation of the version."""
+        return f"{self.major}.{self.minor}.{self.patch}"
+
+    def __eq__(self, other: object) -> bool:
+        """Equality comparison."""
+        if not isinstance(other, Version):
+            return NotImplemented
+        return (
+            self.major == other.major
+            and self.minor == other.minor
+            and self.patch == other.patch
+        )
+
+    def __lt__(self, other: "Version") -> bool:
+        """Less than comparison."""
+        if self.major != other.major:
+            return self.major < other.major
+        if self.minor != other.minor:
+            return self.minor < other.minor
+        return self.patch < other.patch
+
+
+class ChainAmounts(dict[str, dict[str, dict[str, int]]]):
+    """
+    Class that represents chain amounts as a dictionary
+
+    The standard format follows the convention {chain: {address: {token: amount}}}
+    """
+
+    @classmethod
+    def shortfalls(
+        cls, requirements: "ChainAmounts", balances: "ChainAmounts"
+    ) -> "ChainAmounts":
+        """Return the shortfalls between requirements and balances."""
+        result: dict[str, dict[str, dict[str, int]]] = {}
+
+        for chain, addresses in requirements.items():
+            for address, assets in addresses.items():
+                for asset, required_amount in assets.items():
+                    available = balances.get(chain, {}).get(address, {}).get(asset, 0)
+                    shortfall = max(required_amount - available, 0)
+                    result.setdefault(chain, {}).setdefault(address, {})[
+                        asset
+                    ] = shortfall
+
+        return cls(result)
+
+    @classmethod
+    def add(cls, *chainamounts: "ChainAmounts") -> "ChainAmounts":
+        """Add multiple ChainAmounts"""
+        result: dict[str, dict[str, dict[str, int]]] = {}
+
+        for ca in chainamounts:
+            for chain, addresses in ca.items():
+                result_addresses = result.setdefault(chain, {})
+                for address, assets in addresses.items():
+                    result_assets = result_addresses.setdefault(address, {})
+                    for asset, amount in assets.items():
+                        result_assets[asset] = result_assets.get(asset, 0) + amount
+
+        return cls(result)
+
+    def __add__(self, other: "ChainAmounts") -> "ChainAmounts":
+        """Add two ChainAmounts"""
+        return ChainAmounts.add(self, other)
+
+    def __mul__(self, multiplier: float) -> "ChainAmounts":
+        """Multiply all amounts by the specified multiplier"""
+        output = copy.deepcopy(self)
+        for _, addresses in output.items():
+            for _, balances in addresses.items():
+                for asset, amount in balances.items():
+                    balances[asset] = int(amount * multiplier)
+        return output
+
+    def __sub__(self, other: "ChainAmounts") -> "ChainAmounts":
+        """Subtract two ChainAmounts"""
+        return self + (other * -1)
+
+    def __floordiv__(self, divisor: float) -> "ChainAmounts":
+        """Divide all amounts by the specified divisor"""
+        if divisor == 0:
+            raise ValueError("Cannot divide by zero")
+
+        output = copy.deepcopy(self)
+        for _, addresses in output.items():
+            for _, balances in addresses.items():
+                for asset, amount in balances.items():
+                    balances[asset] = int(amount // divisor)
+        return output
+
+    def __lt__(self, other: "ChainAmounts") -> bool:
+        """Return True if all amounts in self are strictly less than the corresponding amounts in other."""
+        for chain, addresses in self.items():
+            for address, assets in addresses.items():
+                for asset, amount in assets.items():
+                    if amount >= other.get(chain, {}).get(address, {}).get(asset, 0):
+                        return False
+        return True
+
+
+@dataclass
+class EncryptedData(LocalResource):
+    """EncryptedData type."""
+
+    path: Path
+    version: int
+    cipher: str
+    cipherparams: t.Dict[str, t.Union[int, str]] = field(repr=False)
+    ciphertext: str = field(repr=False)
+    kdf: str
+    kdfparams: t.Dict[str, t.Union[int, str]] = field(repr=False)
+
+    @classmethod
+    def new(cls, path: Path, password: str, plaintext_bytes: bytes) -> "EncryptedData":
+        """Creates a new EncryptedData"""
+        ph = argon2.PasswordHasher()
+        salt = os.urandom(ph.salt_len)
+        time_cost = ph.time_cost
+        memory_cost = ph.memory_cost
+        parallelism = ph.parallelism
+        hash_len = FERNET_KEY_LENGTH
+        argon2_type = argon2.Type.ID
+        key = argon2.low_level.hash_secret_raw(
+            secret=password.encode(),
+            salt=salt,
+            time_cost=time_cost,
+            memory_cost=memory_cost,
+            parallelism=parallelism,
+            hash_len=hash_len,
+            type=argon2_type,
+        )
+
+        fernet_key = base64.urlsafe_b64encode(key)
+        fernet = Fernet(fernet_key)
+        ciphertext_bytes = fernet.encrypt(plaintext_bytes)
+
+        return cls(
+            path=path,
+            version=1,
+            cipher=f"{fernet.__class__.__module__}.{fernet.__class__.__qualname__}",
+            cipherparams={  # Fernet token (ciphertext variable) already stores them
+                "version": ciphertext_bytes[0]
+            },
+            ciphertext=ciphertext_bytes.hex(),
+            kdf=f"{ph.__class__.__module__}.{ph.__class__.__qualname__}",
+            kdfparams={
+                "salt": salt.hex(),
+                "time_cost": time_cost,
+                "memory_cost": memory_cost,
+                "parallelism": parallelism,
+                "hash_len": hash_len,
+                "type": argon2_type.name,
+            },
+        )
+
+    def decrypt(self, password: str) -> bytes:
+        """Decrypts the EncryptedData"""
+        kdfparams = self.kdfparams
+        key = argon2.low_level.hash_secret_raw(
+            secret=password.encode(),
+            salt=bytes.fromhex(kdfparams["salt"]),
+            time_cost=kdfparams["time_cost"],
+            memory_cost=kdfparams["memory_cost"],
+            parallelism=kdfparams["parallelism"],
+            hash_len=kdfparams["hash_len"],
+            type=argon2.Type[kdfparams["type"]],
+        )
+        fernet_key = base64.urlsafe_b64encode(key)
+        fernet = Fernet(fernet_key)
+        ciphertext_bytes = bytes.fromhex(self.ciphertext)
+        plaintext_bytes = fernet.decrypt(ciphertext_bytes)
+        return plaintext_bytes
+
+    @classmethod
+    def load(cls, path: Path) -> "EncryptedData":
+        """Load EncryptedData."""
+        return cast(EncryptedData, super().load(path))

operate/quickstart/run_service.py

@@ -690,7 +690,7 @@ def run_service(
     )
 
     print_section("Funding the service")
-    manager.fund_service(service_config_id=service.service_config_id)
+    manager.funding_manager.fund_service_initial(service=service)
 
     print_section("Deploying the service")
     manager.deploy_service_locally(

operate/services/agent_runner.py

@@ -32,6 +32,8 @@ import requests
 from aea.configurations.data_types import PublicId
 from aea.helpers.logging import setup_logger
 
+from operate.constants import AGENT_RUNNER_PREFIX
+
 
 @dataclass
 class AgentRelease:
@@ -65,10 +67,10 @@ class AgentRelease:
 # list of agents releases supported
 AGENTS_SUPPORTED = {
     "valory/trader": AgentRelease(
-        owner="valory-xyz", repo="trader", release="v0.27.2-rc.1"
+        owner="valory-xyz", repo="trader", release="v0.27.2-1-rc.2"
     ),
     "valory/optimus": AgentRelease(
-        owner="valory-xyz", repo="optimus", release="v0.0.1051"
+        owner="valory-xyz", repo="optimus", release="v0.6.0-rc.1"
     ),
     "dvilela/memeooorr": AgentRelease(
         owner="valory-xyz", repo="meme-ooorr", release="v0.0.101"
@@ -101,7 +103,7 @@ class AgentRunnerManager:
         else:
             raise ValueError(f"unsupported arch: {platform.machine()}")
 
-        exec_name = f"agent_runner_{os_name}_{arch}"
+        exec_name = f"{AGENT_RUNNER_PREFIX}_{os_name}_{arch}"
         if platform.system() == "Windows":
             exec_name += ".exe"
         return exec_name

operate/services/deployment_runner.py

@@ -224,6 +224,9 @@ class BaseDeploymentRunner(AbstractDeploymentRunner, metaclass=ABCMeta):
         )
 
         self._run_aea_command("-s", "add-key", "ethereum", cwd=working_dir / "agent")
+        self._run_aea_command(
+            "-s", "add-key", "ethereum", "--connection", cwd=working_dir / "agent"
+        )
 
         self._run_aea_command("-s", "issue-certificates", cwd=working_dir / "agent")