olas-operate-middleware 0.9.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

operate/services/manage.py

@@ -23,7 +23,6 @@ import asyncio
 import json
 import logging
 import os
-import time
 import traceback
 import typing as t
 from collections import Counter, defaultdict
@@ -34,8 +33,7 @@ from pathlib import Path
 
 import requests
 from aea.helpers.base import IPFSHash
-from aea.helpers.logging import setup_logger
-from aea_ledger_ethereum import EthereumCrypto, LedgerApi
+from aea_ledger_ethereum import LedgerApi
 from autonomy.chain.base import registry_contracts
 from autonomy.chain.config import CHAIN_PROFILES, ChainType
 from autonomy.chain.metadata import IPFS_URI_PREFIX
@@ -47,7 +45,7 @@ from operate.data.contracts.requester_activity_checker.contract import (
     RequesterActivityCheckerContract,
 )
 from operate.data.contracts.staking_token.contract import StakingTokenContract
-from operate.keys import Key, KeysManager
+from operate.keys import KeysManager
 from operate.ledger import PUBLIC_RPCS, get_currency_denom
 from operate.ledger.profiles import (
     CONTRACTS,
@@ -76,6 +74,7 @@ from operate.services.service import (
     NON_EXISTENT_TOKEN,
     OnChainData,
     SERVICE_CONFIG_PREFIX,
+    SERVICE_CONFIG_VERSION,
     Service,
 )
 from operate.services.utils.mech import deploy_mech
@@ -98,9 +97,8 @@ class ServiceManager:
     def __init__(
         self,
         path: Path,
-        keys_manager: KeysManager,
         wallet_manager: MasterWalletManager,
-        logger: t.Optional[logging.Logger] = None,
+        logger: logging.Logger,
         skip_dependency_check: t.Optional[bool] = False,
     ) -> None:
         """
@@ -112,43 +110,66 @@ class ServiceManager:
         :param logger: logging.Logger object.
         """
         self.path = path
-        self.keys_manager = keys_manager
+        self.keys_manager = KeysManager()
         self.wallet_manager = wallet_manager
-        self.logger = logger or setup_logger(name="operate.manager")
+        self.logger = logger
         self.skip_depencency_check = skip_dependency_check
 
     def setup(self) -> None:
         """Setup service manager."""
         self.path.mkdir(exist_ok=True)
 
-    def _get_all_services(self) -> t.List[Service]:
+    def get_all_service_ids(self) -> t.List[str]:
+        """
+        Get all service ids.
+
+        :return: List of service ids.
+        """
+        return [
+            path.name
+            for path in self.path.iterdir()
+            if path.is_dir() and path.name.startswith(SERVICE_CONFIG_PREFIX)
+        ]
+
+    def get_all_services(self) -> t.Tuple[t.List[Service], bool]:
+        """Get all services."""
         services = []
+        success = True
         for path in self.path.iterdir():
             if not path.name.startswith(SERVICE_CONFIG_PREFIX):
                 continue
             try:
                 service = Service.load(path=path)
+                if service.version != SERVICE_CONFIG_VERSION:
+                    self.logger.warning(
+                        f"Service {path.name} has an unsupported version: {service.version}."
+                    )
+                    success = False
+                    continue
+
                 services.append(service)
-            except ValueError as e:
-                raise e
             except Exception as e:  # pylint: disable=broad-except
                 self.logger.error(
                     f"Failed to load service: {path.name}. Exception {e}: {traceback.format_exc()}"
                 )
-                # Rename the invalid path
-                timestamp = int(time.time())
-                invalid_path = path.parent / f"invalid_{timestamp}_{path.name}"
-                os.rename(path, invalid_path)
-                self.logger.info(
-                    f"Renamed invalid service: {path.name} to {invalid_path.name}"
-                )
+                success = False
+
+        return services, success
 
-        return services
+    def validate_services(self) -> bool:
+        """
+        Validate all services.
+
+        :return: True if all services are valid, False otherwise.
+        """
+        _, success = self.get_all_services()
+        return success
 
     @property
     def json(self) -> t.List[t.Dict]:
         """Returns the list of available services."""
-        return [service.json for service in self._get_all_services()]
+        services, _ = self.get_all_services()
+        return [service.json for service in services]
 
     def exists(self, service_config_id: str) -> bool:
         """Check if service exists."""
@@ -176,14 +197,14 @@ class ServiceManager:
         self,
         hash: str,
         service_template: t.Optional[ServiceTemplate] = None,
-        keys: t.Optional[t.List[Key]] = None,
+        agent_addresses: t.Optional[t.List[str]] = None,
     ) -> Service:
         """
         Create or load a service
 
         :param hash: Service hash
         :param service_template: Service template
-        :param keys: Keys
+        :param agent_addresses: Agents' addresses to be used for the service.
         :return: Service instance
         """
         path = self.path / hash
@@ -202,21 +223,10 @@ class ServiceManager:
                 "'service_template' cannot be None when creating a new service"
             )
 
-        service = Service.new(
-            keys=keys or [],
-            storage=self.path,
-            service_template=service_template,
+        return self.create(
+            service_template=service_template, agent_addresses=agent_addresses
         )
 
-        if not service.keys:
-            service.keys = [
-                self.keys_manager.get(self.keys_manager.create())
-                for _ in range(NUM_LOCAL_AGENT_INSTANCES)
-            ]
-            service.store()
-
-        return service
-
     def load(
         self,
         service_config_id: str,
@@ -233,25 +243,24 @@ class ServiceManager:
     def create(
         self,
         service_template: ServiceTemplate,
-        keys: t.Optional[t.List[Key]] = None,
+        agent_addresses: t.Optional[t.List[str]] = None,
     ) -> Service:
         """
         Create a service
 
         :param service_template: Service template
-        :param keys: Keys
+        :param agent_addresses: Agents' addresses to be used for the service.
         :return: Service instance
         """
         service = Service.new(
-            keys=keys or [],
+            agent_addresses=agent_addresses or [],
             storage=self.path,
             service_template=service_template,
         )
 
-        if not service.keys:
-            service.keys = [
-                self.keys_manager.get(self.keys_manager.create())
-                for _ in range(NUM_LOCAL_AGENT_INSTANCES)
+        if not service.agent_addresses:
+            service.agent_addresses = [
+                self.keys_manager.create() for _ in range(NUM_LOCAL_AGENT_INSTANCES)
             ]
             service.store()
 
@@ -314,8 +323,6 @@ class ServiceManager:
         ledger_config = chain_config.ledger_config
         chain_data = chain_config.chain_data
         user_params = chain_config.chain_data.user_params
-        keys = service.keys
-        instances = [key.address for key in keys]
         ocm = self.get_on_chain_manager(ledger_config=ledger_config)
 
         # TODO fix this
@@ -427,7 +434,7 @@ class ServiceManager:
                         if user_params.use_staking
                         else user_params.cost_of_bond
                     ),
-                    threshold=user_params.threshold,
+                    threshold=len(service.agent_addresses),
                     nft=IPFSHash(user_params.nft),
                     update_token=chain_data.token if is_update else None,
                     token=(
@@ -459,8 +466,8 @@ class ServiceManager:
             agent_id = staking_params["agent_ids"][0]
             ocm.register(
                 service_id=chain_data.token,
-                instances=instances,
-                agents=[agent_id for _ in instances],
+                instances=service.agent_addresses,
+                agents=[agent_id for _ in service.agent_addresses],
                 token=(OLAS[ledger_config.chain] if user_params.use_staking else None),
             )
             on_chain_state = OnChainState.FINISHED_REGISTRATION
@@ -610,8 +617,6 @@ class ServiceManager:
         ledger_config = chain_config.ledger_config
         chain_data = chain_config.chain_data
         user_params = chain_config.chain_data.user_params
-        keys = service.keys
-        instances = [key.address for key in keys]
         wallet = self.wallet_manager.load(ledger_config.chain.ledger_type)
         sftxb = self.get_eth_safe_tx_builder(ledger_config=ledger_config)
         safe = wallet.safes[Chain(chain)]
@@ -719,11 +724,14 @@ class ServiceManager:
                 }
             )
 
-        # TODO: yet another agent specific logic for memeooorr and optimus, which should be abstracted
-        if "memeooorr" in service.name.lower() or "optimus" in service.name.lower():
-            store_path = service.path / "persistent_data"
-            store_path.mkdir(parents=True, exist_ok=True)
-            env_var_to_value.update({"STORE_PATH": os.path.join(str(store_path), "")})
+        # Set environment variables for the service
+        for dir_name, env_var_name in (
+            ("persistent_data", "STORE_PATH"),
+            ("benchmarks", "LOG_DIR"),
+        ):
+            dir_path = service.path / dir_name
+            dir_path.mkdir(parents=True, exist_ok=True)
+            env_var_to_value.update({env_var_name: str(dir_path)})
 
         service.update_env_variables_values(env_var_to_value)
 
@@ -834,7 +842,7 @@ class ServiceManager:
                                 if user_params.use_staking
                                 else user_params.cost_of_bond
                             ),
-                            threshold=user_params.threshold,
+                            threshold=len(service.agent_addresses),
                             nft=IPFSHash(user_params.nft),
                             update_token=chain_data.token,
                             token=(
@@ -884,7 +892,7 @@ class ServiceManager:
                             if user_params.use_staking
                             else user_params.cost_of_bond
                         ),
-                        threshold=user_params.threshold,
+                        threshold=len(service.agent_addresses),
                         nft=IPFSHash(user_params.nft),
                         update_token=None,
                         token=(
@@ -1013,10 +1021,10 @@ class ServiceManager:
                 self.logger.info(
                     f"Approved {token_utility_allowance} OLAS from {safe} to {token_utility}"
                 )
-                cost_of_bond = 1 * len(instances)
+                cost_of_bond = 1 * len(service.agent_addresses)
 
             self.logger.info(
-                f"Registering agent instances: {chain_data.token} -> {instances}"
+                f"Registering agent instances: {chain_data.token} -> {service.agent_addresses}"
             )
 
             native_balance = get_asset_balance(
@@ -1033,8 +1041,8 @@ class ServiceManager:
             sftxb.new_tx().add(
                 sftxb.get_register_instances_data(
                     service_id=chain_data.token,
-                    instances=instances,
-                    agents=[agent_id for _ in instances],
+                    instances=service.agent_addresses,
+                    agents=[agent_id for _ in service.agent_addresses],
                     cost_of_bond=cost_of_bond,
                 )
             ).settle()
@@ -1179,8 +1187,6 @@ class ServiceManager:
         chain_config = service.chain_configs[chain]
         ledger_config = chain_config.ledger_config
         chain_data = chain_config.chain_data
-        keys = service.keys
-        instances = [key.address for key in keys]
         wallet = self.wallet_manager.load(ledger_config.chain.ledger_type)
         safe = wallet.safes[Chain(chain)]  # type: ignore
 
@@ -1245,7 +1251,7 @@ class ServiceManager:
         # Swap service safe
         current_safe_owners = sftxb.get_service_safe_owners(service_id=chain_data.token)
         counter_current_safe_owners = Counter(s.lower() for s in current_safe_owners)
-        counter_instances = Counter(s.lower() for s in instances)
+        counter_instances = Counter(s.lower() for s in service.agent_addresses)
 
         if withdrawal_address is not None:
             # we don't drain signer yet, because the owner swapping tx may need to happen
@@ -1290,16 +1296,15 @@ class ServiceManager:
             )  # noqa: E800
 
         if withdrawal_address is not None:
+            ethereum_crypto = KeysManager().get_crypto_instance(
+                service.agent_addresses[0]
+            )
             # drain all native tokens from service signer key
             drain_eoa(
                 ledger_api=self.wallet_manager.load(
                     ledger_config.chain.ledger_type
                 ).ledger_api(chain=ledger_config.chain, rpc=ledger_config.rpc),
-                crypto=EthereumCrypto(
-                    private_key_path=service.path
-                    / "deployment"
-                    / "ethereum_private_key.txt",
-                ),
+                crypto=ethereum_crypto,
                 withdrawal_address=withdrawal_address,
                 chain_id=ledger_config.chain.id,
             )
@@ -1775,11 +1780,11 @@ class ServiceManager:
                 on_chain_state = self._get_on_chain_state(service=service, chain=chain)
                 if on_chain_state != OnChainState.DEPLOYED:
                     if chain_data.user_params.use_staking:
-                        on_chain_operations_buffer = 1 + len(service.keys)
+                        on_chain_operations_buffer = 1 + len(service.agent_addresses)
                     else:
                         on_chain_operations_buffer = (
                             chain_data.user_params.cost_of_bond
-                            * (1 + len(service.keys))
+                            * (1 + len(service.agent_addresses))
                         )
 
             asset_funding_values = (
@@ -1793,21 +1798,21 @@ class ServiceManager:
                 else fund_requirements.agent
             )
 
-            for key in service.keys:
+            for agent_address in service.agent_addresses:
                 agent_balance = get_asset_balance(
                     ledger_api=ledger_api,
                     asset_address=asset_address,
-                    address=key.address,
+                    address=agent_address,
                 )
                 self.logger.info(
-                    f"[FUNDING_JOB] Agent {key.address} Asset: {asset_address} balance: {agent_balance}"
+                    f"[FUNDING_JOB] Agent {agent_address} Asset: {asset_address} balance: {agent_balance}"
                 )
                 if agent_fund_threshold > 0:
                     self.logger.info(
                         f"[FUNDING_JOB] Required balance: {agent_fund_threshold}"
                     )
                     if agent_balance < agent_fund_threshold:
-                        self.logger.info(f"[FUNDING_JOB] Funding agent {key.address}")
+                        self.logger.info(f"[FUNDING_JOB] Funding agent {agent_address}")
                         target_balance = (
                             asset_funding_values["agent"]["topup"]
                             if asset_funding_values is not None
@@ -1825,11 +1830,11 @@ class ServiceManager:
                             min(available_balance, target_balance - agent_balance), 0
                         )
                         self.logger.info(
-                            f"[FUNDING_JOB] Transferring {to_transfer} units (asset {asset_address}) to agent {key.address}"
+                            f"[FUNDING_JOB] Transferring {to_transfer} units (asset {asset_address}) to agent {agent_address}"
                         )
                         wallet.transfer_asset(
                             asset=asset_address,
-                            to=key.address,
+                            to=agent_address,
                             amount=int(to_transfer),
                             chain=ledger_config.chain,
                             from_safe=from_safe,
@@ -1931,9 +1936,9 @@ class ServiceManager:
             or chain_data.user_params.fund_requirements[ZERO_ADDRESS].agent
         )
 
-        for key in service.keys:
-            agent_balance = ledger_api.get_balance(address=key.address)
-            self.logger.info(f"Agent {key.address} balance: {agent_balance}")
+        for agent_address in service.agent_addresses:
+            agent_balance = ledger_api.get_balance(address=agent_address)
+            self.logger.info(f"Agent {agent_address} balance: {agent_balance}")
             self.logger.info(f"Required balance: {agent_fund_threshold}")
             if agent_balance < agent_fund_threshold:
                 self.logger.info("Funding agents")
@@ -1941,10 +1946,10 @@ class ServiceManager:
                     agent_topup
                     or chain_data.user_params.fund_requirements[ZERO_ADDRESS].agent
                 )
-                self.logger.info(f"Transferring {to_transfer} units to {key.address}")
+                self.logger.info(f"Transferring {to_transfer} units to {agent_address}")
                 wallet.transfer_erc20(
                     token=token,
-                    to=key.address,
+                    to=agent_address,
                     amount=int(to_transfer),
                     chain=ledger_config.chain,
                     from_safe=from_safe,
@@ -1979,7 +1984,7 @@ class ServiceManager:
                 rpc=rpc or ledger_config.rpc,
             )
 
-    def drain_service_safe(
+    def drain_service_safe(  # pylint: disable=too-many-locals
         self,
         service_config_id: str,
         withdrawal_address: str,
@@ -1995,9 +2000,7 @@ class ServiceManager:
         chain_data = chain_config.chain_data
         wallet = self.wallet_manager.load(ledger_config.chain.ledger_type)
         ledger_api = wallet.ledger_api(chain=ledger_config.chain, rpc=ledger_config.rpc)
-        ethereum_crypto = EthereumCrypto(
-            private_key_path=service.path / "deployment" / "ethereum_private_key.txt",
-        )
+        ethereum_crypto = KeysManager().get_crypto_instance(service.agent_addresses[0])
 
         # drain ERC20 tokens from service safe
         for token_name, token_address in (
@@ -2171,40 +2174,6 @@ class ServiceManager:
         )
         return service
 
-    def migrate_service_configs(self) -> None:
-        """Migrate old service config formats to new ones, if applies."""
-
-        bafybei_count = sum(
-            1 for path in self.path.iterdir() if path.name.startswith("bafybei")
-        )
-        if bafybei_count > 1:
-            self.log_directories()
-            raise RuntimeError(
-                f"Your services folder contains {bafybei_count} folders starting with 'bafybei'. This is an unintended situation. Please contact support."
-            )
-
-        paths = list(self.path.iterdir())
-        for path in paths:
-            try:
-                if path.name.startswith(SERVICE_CONFIG_PREFIX) or path.name.startswith(
-                    "bafybei"
-                ):
-                    self.logger.info(f"migrate_service_configs {str(path)}")
-                    migrated = Service.migrate_format(path)
-                    if migrated:
-                        self.logger.info(f"Folder {str(path)} has been migrated.")
-            except Exception as e:  # pylint: disable=broad-except
-                self.logger.error(
-                    f"Failed to migrate service: {path.name}. Exception {e}: {traceback.format_exc()}"
-                )
-                # Rename the invalid path
-                timestamp = int(time.time())
-                invalid_path = path.parent / f"invalid_{timestamp}_{path.name}"
-                os.rename(path, invalid_path)
-                self.logger.info(
-                    f"Renamed invalid service: {path.name} to {invalid_path.name}"
-                )
-
     def refill_requirements(  # pylint: disable=too-many-locals,too-many-statements,too-many-nested-blocks
         self, service_config_id: str
     ) -> t.Dict:
@@ -2232,11 +2201,9 @@ class ServiceManager:
             master_safe_exists = wallet.safes.get(Chain(chain)) is not None
             master_safe = wallet.safes.get(Chain(chain), "master_safe")
 
-            agent_addresses = {key.address for key in service.keys}
+            agent_addresses = set(service.agent_addresses)
             service_safe = (
-                chain_data.multisig
-                if chain_data.multisig and chain_data.multisig != NON_EXISTENT_MULTISIG
-                else "service_safe"
+                chain_data.multisig if chain_data.multisig else "service_safe"
             )
 
             if not master_safe_exists: