olas-operate-middleware 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

operate/migration.py

@@ -22,9 +22,25 @@
 
 import json
 import logging
+import shutil
+import traceback
 from pathlib import Path
+from time import time
 
+from aea_cli_ipfs.ipfs_utils import IPFSTool
+
+from operate.constants import ZERO_ADDRESS
+from operate.operate_types import Chain, LedgerType
+from operate.services.manage import ServiceManager
+from operate.services.service import (
+    DEFAULT_TRADER_ENV_VARS,
+    NON_EXISTENT_MULTISIG,
+    SERVICE_CONFIG_PREFIX,
+    SERVICE_CONFIG_VERSION,
+    Service,
+)
 from operate.utils import create_backup
+from operate.wallet.master import LEDGER_TYPE_TO_WALLET_CLASS, MasterWalletManager
 
 
 class MigrationManager:
@@ -42,6 +58,12 @@ class MigrationManager:
         self._path = home
         self.logger = logger
 
+    def log_directories(self, path: Path) -> None:
+        """Log directories present in `path`."""
+        directories = [f"  - {str(p)}" for p in path.iterdir() if p.is_dir()]
+        directories_str = "\n".join(directories)
+        self.logger.info(f"Directories in {path}\n: {directories_str}")
+
     def migrate_user_account(self) -> None:
         """Migrates user.json"""
 
@@ -62,35 +84,280 @@ class MigrationManager:
 
         self.logger.info("[MIGRATION MANAGER] Migrated user.json.")
 
-    def migrate_wallets(self) -> None:
-        """Migrates wallets."""
+    def migrate_wallets(self, wallet_manager: MasterWalletManager) -> None:
+        """Migrate old wallet config formats to new ones, if applies."""
+        self.logger.info("Migrating wallet configs...")
 
-        path = self._path / "wallets" / "ethereum.json"
-        if not path.exists():
-            return
+        for ledger_type in LedgerType:
+            if not wallet_manager.exists(ledger_type=ledger_type):
+                continue
 
-        migrated = False
-        with open(path, "r", encoding="utf-8") as f:
-            data = json.load(f)
+            wallet_class = LEDGER_TYPE_TO_WALLET_CLASS.get(ledger_type)
+            if wallet_class is None:
+                continue
+
+            migrated = wallet_class.migrate_format(path=wallet_manager.path)
+            if migrated:
+                self.logger.info(f"Wallet {wallet_class} has been migrated.")
+
+        self.logger.info("Migrating wallet configs done.")
+
+    @staticmethod
+    def _migrate_service(  # pylint: disable=too-many-statements,too-many-locals
+        path: Path,
+    ) -> bool:
+        """Migrate the JSON file format if needed."""
+
+        if not path.is_dir():
+            return False
+
+        if not path.name.startswith(SERVICE_CONFIG_PREFIX) and not path.name.startswith(
+            "bafybei"
+        ):
+            return False
+
+        if path.name.startswith("bafybei"):
+            backup_name = f"backup_{int(time())}_{path.name}"
+            backup_path = path.parent / backup_name
+            shutil.copytree(path, backup_path)
+            deployment_path = backup_path / "deployment"
+            if deployment_path.is_dir():
+                shutil.rmtree(deployment_path)
+
+        with open(
+            path / Service._file,  # pylint: disable=protected-access
+            "r",
+            encoding="utf-8",
+        ) as file:
+            data = json.load(file)
+
+        version = data.get("version", 0)
+        if version > SERVICE_CONFIG_VERSION:
+            raise RuntimeError(
+                f"Service configuration in {path} has version {version}, which means it was created with a newer version of olas-operate-middleware. Only configuration versions <= {SERVICE_CONFIG_VERSION} are supported by this version of olas-operate-middleware."
+            )
+
+        # Complete missing env vars for trader
+        if "trader" in data["name"].lower():
+            data.setdefault("env_variables", {})
+
+            for key, value in DEFAULT_TRADER_ENV_VARS.items():
+                if key not in data["env_variables"]:
+                    data["env_variables"][key] = value
+
+            with open(
+                path / Service._file,  # pylint: disable=protected-access
+                "w",
+                encoding="utf-8",
+            ) as file:
+                json.dump(data, file, indent=2)
+
+        if version == SERVICE_CONFIG_VERSION:
+            return False
+
+        # Migration steps for older versions
+        if version == 0:
+            new_data = {
+                "version": 2,
+                "hash": data.get("hash"),
+                "keys": data.get("keys"),
+                "home_chain_id": "100",  # This is the default value for version 2 - do not change, will be corrected below
+                "chain_configs": {
+                    "100": {  # This is the default value for version 2 - do not change, will be corrected below
+                        "ledger_config": {
+                            "rpc": data.get("ledger_config", {}).get("rpc"),
+                            "type": data.get("ledger_config", {}).get("type"),
+                            "chain": data.get("ledger_config", {}).get("chain"),
+                        },
+                        "chain_data": {
+                            "instances": data.get("chain_data", {}).get(
+                                "instances", []
+                            ),
+                            "token": data.get("chain_data", {}).get("token"),
+                            "multisig": data.get("chain_data", {}).get("multisig"),
+                            "staked": data.get("chain_data", {}).get("staked", False),
+                            "on_chain_state": data.get("chain_data", {}).get(
+                                "on_chain_state", 3
+                            ),
+                            "user_params": {
+                                "staking_program_id": "pearl_alpha",
+                                "nft": data.get("chain_data", {})
+                                .get("user_params", {})
+                                .get("nft"),
+                                "threshold": data.get("chain_data", {})
+                                .get("user_params", {})
+                                .get("threshold"),
+                                "use_staking": data.get("chain_data", {})
+                                .get("user_params", {})
+                                .get("use_staking"),
+                                "cost_of_bond": data.get("chain_data", {})
+                                .get("user_params", {})
+                                .get("cost_of_bond"),
+                                "fund_requirements": data.get("chain_data", {})
+                                .get("user_params", {})
+                                .get("fund_requirements", {}),
+                                "agent_id": data.get("chain_data", {})
+                                .get("user_params", {})
+                                .get("agent_id", "14"),
+                            },
+                        },
+                    }
+                },
+                "service_path": data.get("service_path", ""),
+                "name": data.get("name", ""),
+            }
+            data = new_data
+
+        if version < 4:
+            # Add missing fields introduced in later versions, if necessary.
+            for _, chain_data in data.get("chain_configs", {}).items():
+                chain_data.setdefault("chain_data", {}).setdefault(
+                    "user_params", {}
+                ).setdefault("use_mech_marketplace", False)
+                service_name = data.get("name", "")
+                agent_id = Service.determine_agent_id(service_name)
+                chain_data.setdefault("chain_data", {}).setdefault("user_params", {})[
+                    "agent_id"
+                ] = agent_id
 
-        if "optimistic" in data.get("safes", {}):
-            data["safes"]["optimism"] = data["safes"].pop("optimistic")
-            migrated = True
+            data["description"] = data.setdefault("description", data.get("name"))
+            data["hash_history"] = data.setdefault(
+                "hash_history", {int(time()): data["hash"]}
+            )
+
+            if "service_config_id" not in data:
+                service_config_id = Service.get_new_service_config_id(path)
+                new_path = path.parent / service_config_id
+                data["service_config_id"] = service_config_id
+                path = path.rename(new_path)
+
+            old_to_new_ledgers = ["ethereum", "solana"]
+            for key_data in data["keys"]:
+                key_data["ledger"] = old_to_new_ledgers[key_data["ledger"]]
 
-        if "optimistic" in data.get("safe_chains"):
-            data["safe_chains"] = [
-                "optimism" if chain == "optimistic" else chain
-                for chain in data["safe_chains"]
+            old_to_new_chains = [
+                "ethereum",
+                "goerli",
+                "gnosis",
+                "solana",
+                "optimism",
+                "base",
+                "mode",
             ]
-            migrated = True
+            new_chain_configs = {}
+            for chain_id, chain_data in data["chain_configs"].items():
+                chain_data["ledger_config"]["chain"] = old_to_new_chains[
+                    chain_data["ledger_config"]["chain"]
+                ]
+                del chain_data["ledger_config"]["type"]
+                new_chain_configs[Chain.from_id(int(chain_id)).value] = chain_data  # type: ignore
 
-        if not migrated:
-            return
+            data["chain_configs"] = new_chain_configs
+            data["home_chain"] = data.setdefault("home_chain", Chain.from_id(int(data.get("home_chain_id", "100"))).value)  # type: ignore
+            del data["home_chain_id"]
 
-        with open(path, "w", encoding="utf-8") as f:
-            json.dump(data, f, indent=4)
+            if "env_variables" not in data:
+                if data["name"] == "valory/trader_pearl":
+                    data["env_variables"] = DEFAULT_TRADER_ENV_VARS
+                else:
+                    data["env_variables"] = {}
+
+        if version < 5:
+            new_chain_configs = {}
+            for chain, chain_data in data["chain_configs"].items():
+                fund_requirements = chain_data["chain_data"]["user_params"][
+                    "fund_requirements"
+                ]
+                if ZERO_ADDRESS not in fund_requirements:
+                    chain_data["chain_data"]["user_params"]["fund_requirements"] = {
+                        ZERO_ADDRESS: fund_requirements
+                    }
+
+                new_chain_configs[chain] = chain_data  # type: ignore
+            data["chain_configs"] = new_chain_configs
+
+        if version < 7:
+            for _, chain_data in data.get("chain_configs", {}).items():
+                if chain_data["chain_data"]["multisig"] == "0xm":
+                    chain_data["chain_data"]["multisig"] = NON_EXISTENT_MULTISIG
+
+            data["agent_addresses"] = [key["address"] for key in data["keys"]]
+            del data["keys"]
+
+        if version < 8:
+            if data["home_chain"] == "optimistic":
+                data["home_chain"] = Chain.OPTIMISM.value
+
+            if "optimistic" in data["chain_configs"]:
+                data["chain_configs"]["optimism"] = data["chain_configs"].pop(
+                    "optimistic"
+                )
+
+            for _, chain_config in data["chain_configs"].items():
+                if chain_config["ledger_config"]["chain"] == "optimistic":
+                    chain_config["ledger_config"]["chain"] = Chain.OPTIMISM.value
+
+        data["version"] = SERVICE_CONFIG_VERSION
+
+        # Redownload service path
+        if "service_path" in data:
+            package_absolute_path = path / Path(data["service_path"]).name
+            data.pop("service_path")
+        else:
+            package_absolute_path = path / data["package_path"]
+
+        if package_absolute_path.exists() and package_absolute_path.is_dir():
+            shutil.rmtree(package_absolute_path)
+
+        package_absolute_path = Path(
+            IPFSTool().download(
+                hash_id=data["hash"],
+                target_dir=path,
+            )
+        )
+        data["package_path"] = str(package_absolute_path.name)
+
+        with open(
+            path / Service._file,  # pylint: disable=protected-access
+            "w",
+            encoding="utf-8",
+        ) as file:
+            json.dump(data, file, indent=2)
+
+        return True
+
+    def migrate_services(self, service_manager: ServiceManager) -> None:
+        """Migrate old service config formats to new ones, if applies."""
+        self.log_directories(service_manager.path)
+        self.logger.info("Migrating service configs...")
+
+        bafybei_count = sum(
+            1
+            for path in service_manager.path.iterdir()
+            if path.name.startswith("bafybei")
+        )
+        if bafybei_count > 1:
+            raise RuntimeError(
+                f"Your services folder contains {bafybei_count} folders starting with 'bafybei'. This is an unintended situation. Please contact support."
+            )
+
+        paths = list(service_manager.path.iterdir())
+        for path in paths:
+            try:
+                if path.name.startswith(SERVICE_CONFIG_PREFIX) or path.name.startswith(
+                    "bafybei"
+                ):
+                    self.logger.info(f"migrate_service_configs {str(path)}")
+                    migrated = self._migrate_service(path)
+                    if migrated:
+                        self.logger.info(f"Folder {str(path)} has been migrated.")
+            except Exception as e:  # pylint: disable=broad-except
+                self.logger.error(
+                    f"Failed to migrate service: {path.name}. Exception {e}: {traceback.format_exc()}"
+                )
 
-        self.logger.info("[MIGRATION MANAGER] Migrated wallets.")
+        self.logger.info("Migrating service configs done.")
+        self.log_directories(service_manager.path)
 
     def migrate_qs_configs(self) -> None:
         """Migrates quickstart configs."""

operate/operate_types.py

@@ -28,6 +28,7 @@ from autonomy.chain.config import ChainType
 from autonomy.chain.constants import CHAIN_NAME_TO_CHAIN_ID
 from typing_extensions import TypedDict
 
+from operate.constants import NO_STAKING_PROGRAM_ID
 from operate.resource import LocalResource
 
 
@@ -187,9 +188,6 @@ class ConfigurationTemplate(TypedDict):
     nft: str
     rpc: str
     agent_id: int
-    threshold: int
-    use_staking: bool
-    use_mech_marketplace: bool
     cost_of_bond: int
     fund_requirements: t.Dict[str, FundRequirementsTemplate]
     fallback_chain_params: t.Optional[t.Dict]
@@ -254,13 +252,18 @@ class OnChainUserParams(LocalResource):
 
     staking_program_id: str
     nft: str
-    threshold: int
     agent_id: int
-    use_staking: bool
-    use_mech_marketplace: bool
     cost_of_bond: int
     fund_requirements: OnChainTokenRequirements
 
+    @property
+    def use_staking(self) -> bool:
+        """Check if staking is used."""
+        return (
+            self.staking_program_id is not None
+            and self.staking_program_id != NO_STAKING_PROGRAM_ID
+        )
+
     @classmethod
     def from_json(cls, obj: t.Dict) -> "OnChainUserParams":
         """Load a service"""

operate/quickstart/analyse_logs.py

@@ -38,7 +38,7 @@ def find_build_directory(config_file: Path, operate: "OperateApp") -> Path:
         config = json.load(f)
         config_service_hash = config.get("hash")
 
-    services = operate.service_manager()._get_all_services()
+    services = operate.service_manager().get_all_services()
     for service in services:
         if service.hash == config_service_hash:
             build_dir = service.path / DEPLOYMENT_DIR
@@ -107,9 +107,6 @@ def analyse_logs(
         print(f"Config file '{config_file}' not found.")
         sys.exit(1)
 
-    operate.service_manager().migrate_service_configs()
-    operate.wallet_manager.migrate_wallet_configs()
-
     # Auto-detect the logs directory
     build_dir = find_build_directory(config_file, operate)
     logs_dir = build_dir / "persistent_data" / "logs"

operate/quickstart/claim_staking_rewards.py

@@ -48,9 +48,6 @@ def claim_staking_rewards(operate: "OperateApp", config_path: str) -> None:
 
     print_section(f"Claim staking rewards for {template['name']}")
 
-    operate.service_manager().migrate_service_configs()
-    operate.wallet_manager.migrate_wallet_configs()
-
     # check if agent was started before
     config = load_local_config(
         operate=operate, service_name=cast(str, template["name"])

operate/quickstart/reset_configs.py

@@ -70,9 +70,6 @@ def reset_configs(operate: "OperateApp", config_path: str) -> None:
     with open(config_path, "r") as config_file:
         template = json.load(config_file)
 
-    operate.service_manager().migrate_service_configs()
-    operate.wallet_manager.migrate_wallet_configs()
-
     print_title(f"Reset your {template['name']} configurations")
 
     # check if agent was started before

operate/quickstart/reset_password.py

@@ -35,9 +35,6 @@ def reset_password(operate: "OperateApp") -> None:
     """Reset password."""
     print_title("Reset your password")
 
-    operate.service_manager().migrate_service_configs()
-    operate.wallet_manager.migrate_wallet_configs()
-
     # check if agent was started before
     if not (operate._path / "user.json").exists():
         print("No previous agent setup found. Exiting.")

operate/quickstart/reset_staking.py

@@ -22,7 +22,8 @@ import json
 import os
 from typing import TYPE_CHECKING, cast
 
-from operate.ledger.profiles import NO_STAKING_PROGRAM_ID, get_staking_contract
+from operate.constants import NO_STAKING_PROGRAM_ID
+from operate.ledger.profiles import get_staking_contract
 from operate.quickstart.run_service import (
     CUSTOM_PROGRAM_ID,
     ask_password_if_needed,
@@ -44,9 +45,6 @@ def reset_staking(operate: "OperateApp", config_path: str) -> None:
     with open(config_path, "r") as config_file:
         template = json.load(config_file)
 
-    operate.service_manager().migrate_service_configs()
-    operate.wallet_manager.migrate_wallet_configs()
-
     print_title("Reset your staking program preference")
 
     # check if agent was started before

operate/quickstart/run_service.py

@@ -34,10 +34,10 @@ from halo import Halo  # type: ignore[import]
 from web3.exceptions import Web3Exception
 
 from operate.account.user import UserAccount
-from operate.constants import IPFS_ADDRESS, OPERATE_HOME
+from operate.constants import IPFS_ADDRESS, NO_STAKING_PROGRAM_ID, OPERATE_HOME
 from operate.data import DATA_DIR
 from operate.data.contracts.staking_token.contract import StakingTokenContract
-from operate.ledger.profiles import NO_STAKING_PROGRAM_ID, STAKING, get_staking_contract
+from operate.ledger.profiles import STAKING, get_staking_contract
 from operate.operate_types import (
     Chain,
     LedgerType,
@@ -363,14 +363,12 @@ def configure_local_config(
             template["configurations"][chain] |= {
                 "staking_program_id": config.staking_program_id,
                 "rpc": config.rpc[chain],
-                "use_staking": config.staking_program_id != NO_STAKING_PROGRAM_ID,
                 "cost_of_bond": min_staking_deposit,
             }
         else:
             template["configurations"][chain] |= {
                 "staking_program_id": NO_STAKING_PROGRAM_ID,
                 "rpc": config.rpc[chain],
-                "use_staking": False,
                 "cost_of_bond": 1,
             }
 
@@ -569,7 +567,7 @@ def _ask_funds_from_requirements(
             chain_config.chain_data.multisig: "Service Safe"
             for chain_config in service.chain_configs.values()
         }
-        | {key.address: "Agent EOA" for key in service.keys}
+        | {address: "Agent EOA" for address in service.agent_addresses}
     )
 
     if not requirements["is_refill_required"] and requirements["allow_start_agent"]:

operate/quickstart/stop_service.py

@@ -45,9 +45,6 @@ def stop_service(operate: "OperateApp", config_path: str) -> None:
 
     print_title(f"Stop {template['name']} Quickstart")
 
-    operate.service_manager().migrate_service_configs()
-    operate.wallet_manager.migrate_wallet_configs()
-
     # check if agent was started before
     config = load_local_config(
         operate=operate, service_name=cast(str, template["name"])

operate/quickstart/terminate_on_chain_service.py

@@ -43,9 +43,6 @@ def terminate_service(operate: "OperateApp", config_path: str) -> None:
 
     print_title(f"Terminate {template['name']} on-chain service")
 
-    operate.service_manager().migrate_service_configs()
-    operate.wallet_manager.migrate_wallet_configs()
-
     # check if agent was started before
     config = load_local_config(
         operate=operate, service_name=cast(str, template["name"])