olas-operate-middleware 0.10.20__py3-none-any.whl → 0.11.0__py3-none-any.whl

operate/cli.py

@@ -22,6 +22,7 @@ import asyncio
 import atexit
 import multiprocessing
 import os
+import shutil
 import signal
 import traceback
 import typing as t
@@ -30,10 +31,10 @@ from concurrent.futures import ThreadPoolExecutor
 from contextlib import asynccontextmanager, suppress
 from http import HTTPStatus
 from pathlib import Path
+from time import time
 from types import FrameType
 
-import psutil
-import requests
+import autonomy.chain.tx
 from aea.helpers.logging import setup_logger
 from clea import group, params, run
 from fastapi import FastAPI, Request
@@ -47,22 +48,35 @@ from operate import __version__, services
 from operate.account.user import UserAccount
 from operate.bridge.bridge_manager import BridgeManager
 from operate.constants import (
+    AGENT_RUNNER_PREFIX,
+    DEPLOYMENT_DIR,
     KEYS_DIR,
     MIN_PASSWORD_LENGTH,
+    MSG_INVALID_MNEMONIC,
+    MSG_INVALID_PASSWORD,
+    MSG_NEW_PASSWORD_MISSING,
+    OPERATE,
     OPERATE_HOME,
     SERVICES_DIR,
     USER_JSON,
+    VERSION_FILE,
     WALLETS_DIR,
     WALLET_RECOVERY_DIR,
     ZERO_ADDRESS,
 )
 from operate.ledger.profiles import (
-    DEFAULT_MASTER_EOA_FUNDS,
+    DEFAULT_EOA_TOPUPS,
     DEFAULT_NEW_SAFE_FUNDS,
     ERC20_TOKENS,
 )
 from operate.migration import MigrationManager
-from operate.operate_types import Chain, DeploymentStatus, LedgerType
+from operate.operate_types import (
+    Chain,
+    ChainAmounts,
+    DeploymentStatus,
+    LedgerType,
+    Version,
+)
 from operate.quickstart.analyse_logs import analyse_logs
 from operate.quickstart.claim_staking_rewards import claim_staking_rewards
 from operate.quickstart.reset_configs import reset_configs
@@ -72,16 +86,38 @@ from operate.quickstart.run_service import run_service
 from operate.quickstart.stop_service import stop_service
 from operate.quickstart.terminate_on_chain_service import terminate_service
 from operate.services.deployment_runner import stop_deployment_manager
+from operate.services.funding_manager import FundingInProgressError, FundingManager
 from operate.services.health_checker import HealthChecker
+from operate.settings import Settings
 from operate.utils import subtract_dicts
 from operate.utils.gnosis import get_assets_balances
-from operate.wallet.master import MasterWalletManager
+from operate.utils.single_instance import AppSingleInstance, ParentWatchdog
+from operate.wallet.master import InsufficientFundsException, MasterWalletManager
 from operate.wallet.wallet_recovery_manager import (
     WalletRecoveryError,
     WalletRecoveryManager,
 )
 
 
+# TODO Backport to Open Autonomy
+def should_rebuild(error: str) -> bool:
+    """Check if we should rebuild the transaction."""
+    for _error in (
+        "wrong transaction nonce",
+        "OldNonce",
+        "nonce too low",
+        "replacement transaction underpriced",
+    ):
+        if _error in error:
+            return True
+    return False
+
+
+autonomy.chain.tx.ERRORS_TO_RETRY += ("replacement transaction underpriced",)
+autonomy.chain.tx.should_rebuild = should_rebuild
+# End backport to Open Autonomy
+
+
 DEFAULT_MAX_RETRIES = 3
 USER_NOT_LOGGED_IN_ERROR = JSONResponse(
     content={"error": "User not logged in."}, status_code=HTTPStatus.UNAUTHORIZED
@@ -115,17 +151,28 @@ class OperateApp:
         home: t.Optional[Path] = None,
     ) -> None:
         """Initialize object."""
-        super().__init__()
         self._path = (home or OPERATE_HOME).resolve()
         self._services = self._path / SERVICES_DIR
         self._keys = self._path / KEYS_DIR
         self.setup()
+        self._backup_operate_if_new_version()
 
         services.manage.KeysManager(
             path=self._keys,
             logger=logger,
         )
-        self.password: t.Optional[str] = os.environ.get("OPERATE_USER_PASSWORD")
+        self._password: t.Optional[str] = os.environ.get("OPERATE_USER_PASSWORD")
+        self.settings = Settings(path=self._path)
+
+        self._wallet_manager = MasterWalletManager(
+            path=self._path / WALLETS_DIR,
+            password=self.password,
+        )
+        self._wallet_manager.setup()
+        self._funding_manager = FundingManager(
+            wallet_manager=self._wallet_manager,
+            logger=logger,
+        )
 
         mm = MigrationManager(self._path, logger)
         mm.migrate_user_account()
@@ -133,6 +180,59 @@ class OperateApp:
         mm.migrate_wallets(self.wallet_manager)
         mm.migrate_qs_configs()
 
+    @property
+    def password(self) -> t.Optional[str]:
+        """Get the password."""
+        return self._password
+
+    @password.setter
+    def password(self, value: t.Optional[str]) -> None:
+        """Set the password."""
+        self._password = value
+        self._wallet_manager.password = value
+
+    def _backup_operate_if_new_version(self) -> None:
+        """Backup .operate directory if this is a new version."""
+        current_version = Version(__version__)
+        backup_required = False
+        version_file = self._path / VERSION_FILE
+        if not version_file.exists():
+            backup_required = True
+            found_version = "0.10.21"  # first version with version file
+        else:
+            found_version = Version(version_file.read_text())
+            if current_version.major > found_version.major or (
+                current_version.major == found_version.major
+                and current_version.minor > found_version.minor
+            ):
+                backup_required = True
+
+        if not backup_required:
+            return
+
+        backup_path = self._path.parent / f"{OPERATE}_v{found_version}_bak"
+        if backup_path.exists():
+            logger.info(f"Backup directory {backup_path} already exists.")
+            backup_path = (
+                self._path.parent / f"{OPERATE}_v{found_version}_bak_{int(time())}"
+            )
+
+        logger.info(f"Backing up existing {OPERATE} directory to {backup_path}")
+        shutil.copytree(self._path, backup_path)
+        version_file.write_text(str(current_version))
+
+        # remove recoverable files from the backup to save space
+        service_dir = backup_path / SERVICES_DIR
+        for service_path in service_dir.iterdir():
+            deployment_dir = service_path / DEPLOYMENT_DIR
+            if deployment_dir.exists():
+                shutil.rmtree(deployment_dir)
+
+            for agent_runner_path in service_path.glob(f"{AGENT_RUNNER_PREFIX}*"):
+                agent_runner_path.unlink()
+
+        logger.info("Backup completed.")
+
     def create_user_account(self, password: str) -> UserAccount:
         """Create a user account."""
         self.password = password
@@ -145,13 +245,13 @@ class OperateApp:
         """Updates current password"""
 
         if not new_password:
-            raise ValueError("'new_password' is required.")
+            raise ValueError(MSG_NEW_PASSWORD_MISSING)
 
         if not (
             self.user_account.is_valid(old_password)
             and self.wallet_manager.is_password_valid(old_password)
         ):
-            raise ValueError("Password is not valid.")
+            raise ValueError(MSG_INVALID_PASSWORD)
 
         wallet_manager = self.wallet_manager
         wallet_manager.password = old_password
@@ -162,11 +262,11 @@ class OperateApp:
         """Updates current password using the mnemonic"""
 
         if not new_password:
-            raise ValueError("'new_password' is required.")
+            raise ValueError(MSG_NEW_PASSWORD_MISSING)
 
         mnemonic = mnemonic.strip().lower()
         if not self.wallet_manager.is_mnemonic_valid(mnemonic):
-            raise ValueError("Seed phrase is not valid.")
+            raise ValueError(MSG_INVALID_MNEMONIC)
 
         wallet_manager = self.wallet_manager
         wallet_manager.update_password_with_mnemonic(mnemonic, new_password)
@@ -179,10 +279,16 @@ class OperateApp:
         return services.manage.ServiceManager(
             path=self._services,
             wallet_manager=self.wallet_manager,
+            funding_manager=self.funding_manager,
             logger=logger,
             skip_dependency_check=skip_dependency_check,
         )
 
+    @property
+    def funding_manager(self) -> FundingManager:
+        """Load funding manager."""
+        return self._funding_manager
+
     @property
     def user_account(self) -> t.Optional[UserAccount]:
         """Load user account."""
@@ -193,13 +299,7 @@ class OperateApp:
     @property
     def wallet_manager(self) -> MasterWalletManager:
         """Load wallet manager."""
-        manager = MasterWalletManager(
-            path=self._path / WALLETS_DIR,
-            password=self.password,
-            logger=logger,
-        )
-        manager.setup()
-        return manager
+        return self._wallet_manager
 
     @property
     def wallet_recoverey_manager(self) -> WalletRecoveryManager:
@@ -232,7 +332,7 @@ class OperateApp:
         """Json representation of the app."""
         return {
             "name": "Operate HTTP server",
-            "version": "0.1.0.rc0",
+            "version": (__version__),
             "home": str(self._path),
         }
 
@@ -252,7 +352,7 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
         logger.warning("Healthchecker is off!!!")
     operate = OperateApp(home=home)
 
-    funding_jobs: t.Dict[str, asyncio.Task] = {}
+    funding_job: t.Optional[asyncio.Task] = None
     health_checker = HealthChecker(
         operate.service_manager(), number_of_fails=number_of_fails, logger=logger
     )
@@ -272,25 +372,6 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
             raise exception
         return res
 
-    def schedule_funding_job(
-        service_config_id: str,
-        from_safe: bool = True,
-    ) -> None:
-        """Schedule a funding job."""
-        logger.info(f"Starting funding job for {service_config_id}")
-        if service_config_id in funding_jobs:
-            logger.info(f"Cancelling existing funding job for {service_config_id}")
-            cancel_funding_job(service_config_id=service_config_id)
-
-        loop = asyncio.get_running_loop()
-        funding_jobs[service_config_id] = loop.create_task(
-            operate.service_manager().funding_job(
-                service_config_id=service_config_id,
-                loop=loop,
-                from_safe=from_safe,
-            )
-        )
-
     def schedule_healthcheck_job(
         service_config_id: str,
     ) -> None:
@@ -299,13 +380,29 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
             # dont start health checker if it's switched off
             health_checker.start_for_service(service_config_id)
 
-    def cancel_funding_job(service_config_id: str) -> None:
+    def schedule_funding_job() -> None:
+        """Schedule the funding job."""
+        cancel_funding_job()  # cancel previous job if any
+        logger.info("Starting the funding job")
+
+        loop = asyncio.get_event_loop()
+        nonlocal funding_job
+        funding_job = loop.create_task(
+            operate.funding_manager.funding_job(
+                service_manager=operate.service_manager(),
+                loop=loop,
+            )
+        )
+
+    def cancel_funding_job() -> None:
         """Cancel funding job."""
-        if service_config_id not in funding_jobs:
+        nonlocal funding_job
+        if funding_job is None:
             return
-        status = funding_jobs[service_config_id].cancel()
+
+        status = funding_job.cancel()
         if not status:
-            logger.info(f"Funding job cancellation for {service_config_id} failed")
+            logger.info("Funding job cancellation failed")
 
     def pause_all_services_on_startup() -> None:
         logger.info("Stopping services on startup...")
@@ -344,7 +441,6 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
                     f"Deployment {service_config_id} stopping failed. but continue"
                 )
             logger.info(f"Cancelling funding job for {service_config_id}")
-            cancel_funding_job(service_config_id=service_config_id)
             health_checker.stop_for_service(service_config_id=service_config_id)
 
     def pause_all_services_on_exit(signum: int, frame: t.Optional[FrameType]) -> None:
@@ -363,49 +459,25 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
 
     @asynccontextmanager
     async def lifespan(app: FastAPI):
-        # Load the ML model
-        watchdog_task = set_parent_watchdog(app)
-        yield
-        # Clean up the ML models and release the resources
-
-        with suppress(Exception):
-            watchdog_task.cancel()
-
-        with suppress(Exception):
-            await watchdog_task
-
-    app = FastAPI(lifespan=lifespan)
-
-    def set_parent_watchdog(app):
         async def stop_app():
-            logger.info("Stopping services on demand...")
-
-            stop_deployment_manager()  # TODO: make it async?
+            logger.info("Stopping services due to parent death...")
+            stop_deployment_manager()
             await run_in_executor(pause_all_services)
-
-            logger.info("Stopping services on demand done.")
             app._server.should_exit = True  # pylint: disable=protected-access
-            logger.info("Stopping app.")
+            logger.info("App stopped due to parent death.")
 
-        async def check_parent_alive():
-            try:
-                logger.info(
-                    f"Parent alive check task started: ppid is {os.getppid()} and own pid is {os.getpid()}"
-                )
-                while True:
-                    parent = psutil.Process(os.getpid()).parent()
-                    if not parent:
-                        logger.info("Parent is not alive, going to stop")
-                        await stop_app()
-                        return
-                    await asyncio.sleep(3)
+        watchdog = ParentWatchdog(on_parent_exit=stop_app)
+        watchdog.start()
 
-            except Exception:  # pylint: disable=broad-except
-                logger.exception("Parent alive check crashed!")
+        yield  # --- app is running ---
 
-        loop = asyncio.get_running_loop()
-        task = loop.create_task(check_parent_alive())
-        return task
+        with suppress(Exception):
+            cancel_funding_job()
+
+        with suppress(Exception):
+            await watchdog.stop()
+
+    app = FastAPI(lifespan=lifespan)
 
     app.add_middleware(
         CORSMiddleware,
@@ -446,6 +518,11 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
         """Get API info."""
         return JSONResponse(content=operate.json)
 
+    @app.get("/api/settings")
+    async def _get_settings(request: Request) -> JSONResponse:
+        """Get settings."""
+        return JSONResponse(content=operate.settings.json)
+
     @app.get("/api/account")
     async def _get_account(request: Request) -> t.Dict:
         """Get account information."""
@@ -550,11 +627,12 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
         data = await request.json()
         if not operate.user_account.is_valid(password=data["password"]):
             return JSONResponse(
-                content={"error": "Password is not valid."},
+                content={"error": MSG_INVALID_PASSWORD},
                 status_code=HTTPStatus.UNAUTHORIZED,
             )
 
         operate.password = data["password"]
+        schedule_funding_job()
         return JSONResponse(
             content={"message": "Login successful."},
             status_code=HTTPStatus.OK,
@@ -602,14 +680,50 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
             return USER_NOT_LOGGED_IN_ERROR
         if operate.password != password:
             return JSONResponse(
-                content={"error": "Password is not valid."},
+                content={"error": MSG_INVALID_PASSWORD},
                 status_code=HTTPStatus.UNAUTHORIZED,
             )
 
+        # TODO Should fail if not provided
         ledger_type = data.get("ledger_type", LedgerType.ETHEREUM.value)
         wallet = operate.wallet_manager.load(ledger_type=LedgerType(ledger_type))
         return JSONResponse(content={"private_key": wallet.crypto.private_key})
 
+    @app.post("/api/wallet/mnemonic")
+    async def _get_mnemonic(request: Request) -> t.List[t.Dict]:
+        """Get Master EOA mnemonic."""
+        if operate.user_account is None:
+            return ACCOUNT_NOT_FOUND_ERROR
+
+        data = await request.json()
+        password = data.get("password")
+        if operate.password is None:
+            return USER_NOT_LOGGED_IN_ERROR
+        if operate.password != password:
+            return JSONResponse(
+                content={"error": "Password is not valid."},
+                status_code=HTTPStatus.UNAUTHORIZED,
+            )
+
+        try:
+            ledger_type = LedgerType(data.get("ledger_type"))
+            wallet = operate.wallet_manager.load(ledger_type=ledger_type)
+            mnemonic = wallet.decrypt_mnemonic(password=password)
+            if mnemonic is None:
+                return JSONResponse(
+                    content={"error": "Mnemonic file does not exist."},
+                    status_code=HTTPStatus.NOT_FOUND,
+                )
+            return JSONResponse(content={"mnemonic": mnemonic})
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(f"Failed to retrieve mnemonic: {e}\n{traceback.format_exc()}")
+            return JSONResponse(
+                content={
+                    "error": "Failed to retrieve mnemonic. Please check the logs."
+                },
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+            )
+
     @app.get("/api/extended/wallet")
     async def _get_wallet_safe(request: Request) -> t.List[t.Dict]:
         """Get wallets."""
@@ -709,14 +823,16 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
         )
 
         if transfer_excess_assets:
-            asset_addresses = {ZERO_ADDRESS} | {token[chain] for token in ERC20_TOKENS}
+            asset_addresses = {ZERO_ADDRESS} | {
+                token[chain] for token in ERC20_TOKENS.values()
+            }
             balances = get_assets_balances(
                 ledger_api=ledger_api,
                 addresses={wallet.address},
                 asset_addresses=asset_addresses,
                 raise_on_invalid_address=False,
             )[wallet.address]
-            initial_funds = subtract_dicts(balances, DEFAULT_MASTER_EOA_FUNDS[chain])
+            initial_funds = subtract_dicts(balances, DEFAULT_EOA_TOPUPS[chain])
 
         logger.info(f"_create_safe Computed {initial_funds=}")
 
@@ -730,10 +846,13 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
 
             transfer_txs = {}
             for asset, amount in initial_funds.items():
+                if amount <= 0:
+                    continue
+
                 logger.info(
                     f"_create_safe Transfer to={safe_address} {amount=} {chain} {asset=}"
                 )
-                tx_hash = wallet.transfer_asset(
+                tx_hash = wallet.transfer(
                     to=safe_address,
                     amount=int(amount),
                     chain=chain,
@@ -810,6 +929,82 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
             }
         )
 
+    @app.post("/api/wallet/withdraw")
+    async def _wallet_withdraw(request: Request) -> JSONResponse:
+        """Withdraw from Master Safe / master eoa"""
+
+        if operate.password is None:
+            return USER_NOT_LOGGED_IN_ERROR
+
+        data = await request.json()
+        if not operate.user_account.is_valid(password=data["password"]):
+            return JSONResponse(
+                content={"error": MSG_INVALID_PASSWORD},
+                status_code=HTTPStatus.UNAUTHORIZED,
+            )
+
+        try:
+            withdraw_assets = data.get("withdraw_assets", {})
+            to = data["to"]
+            wallet_manager = operate.wallet_manager
+            transfer_txs: t.Dict[str, t.Dict[str, t.List[str]]] = {}
+
+            # TODO: Ensure master wallet has enough funding.
+            for chain_str, tokens in withdraw_assets.items():
+                chain = Chain(chain_str)
+                wallet = wallet_manager.load(chain.ledger_type)
+                transfer_txs[chain_str] = {}
+
+                # Process ERC20 first
+                for asset, amount in tokens.items():
+                    if asset != ZERO_ADDRESS:
+                        txs = wallet.transfer_from_safe_then_eoa(
+                            to=to,
+                            amount=int(amount),
+                            chain=chain,
+                            asset=asset,
+                        )
+                        transfer_txs[chain_str][asset] = txs
+
+                # Process native last
+                if ZERO_ADDRESS in tokens:
+                    asset = ZERO_ADDRESS
+                    amount = tokens[asset]
+                    txs = wallet.transfer_from_safe_then_eoa(
+                        to=to,
+                        amount=int(amount),
+                        chain=chain,
+                        asset=asset,
+                    )
+                    transfer_txs[chain_str][asset] = txs
+
+        except InsufficientFundsException as e:
+            logger.error(f"Insufficient funds: {e}\n{traceback.format_exc()}")
+            return JSONResponse(
+                content={
+                    "error": f"Failed to withdraw funds. Insufficient funds: {e}",
+                    "transfer_txs": transfer_txs,
+                },
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(f"Failed to withdraw funds: {e}\n{traceback.format_exc()}")
+            return JSONResponse(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                content={
+                    "error": "Failed to withdraw funds. Please check the logs.",
+                    "transfer_txs": transfer_txs,
+                },
+            )
+
+        return JSONResponse(
+            content={
+                "error": None,
+                "message": "Funds withdrawn successfully.",
+                "transfer_txs": transfer_txs,
+            }
+        )
+
     @app.get("/api/v2/services")
     async def _get_services(request: Request) -> JSONResponse:
         """Get all services."""
@@ -885,6 +1080,21 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
             .get_agent_performance()
         )
 
+    @app.get("/api/v2/service/{service_config_id}/funding_requirements")
+    async def _get_funding_requirements(request: Request) -> JSONResponse:
+        """Get the service refill requirements."""
+        service_config_id = request.path_params["service_config_id"]
+
+        if not operate.service_manager().exists(service_config_id=service_config_id):
+            return service_not_found_error(service_config_id=service_config_id)
+
+        return JSONResponse(
+            content=operate.service_manager().funding_requirements(
+                service_config_id=service_config_id
+            )
+        )
+
+    # TODO deprecate
     @app.get("/api/v2/service/{service_config_id}/refill_requirements")
     async def _get_refill_requirements(request: Request) -> JSONResponse:
         """Get the service refill requirements."""
@@ -928,11 +1138,9 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
             manager.deploy_service_onchain_from_safe(
                 service_config_id=service_config_id
             )
-            manager.fund_service(service_config_id=service_config_id)
             manager.deploy_service_locally(service_config_id=service_config_id)
 
         await run_in_executor(_fn)
-        schedule_funding_job(service_config_id=service_config_id)
         schedule_healthcheck_job(service_config_id=service_config_id)
 
         return JSONResponse(
@@ -996,9 +1204,9 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
 
         await run_in_executor(deployment.stop)
         logger.info(f"Cancelling funding job for {service_config_id}")
-        cancel_funding_job(service_config_id=service_config_id)
         return JSONResponse(content=deployment.json)
 
+    # TODO Deprecate
     @app.post("/api/v2/service/{service_config_id}/onchain/withdraw")
     async def _withdraw_onchain(request: Request) -> JSONResponse:
         """Withdraw all the funds from a service."""
@@ -1028,16 +1236,20 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
                 service_manager.terminate_service_on_chain_from_safe(
                     service_config_id=service_config_id,
                     chain=chain,
+                )
+                service_manager.drain(
+                    service_config_id=service_config_id,
+                    chain_str=chain,
                     withdrawal_address=withdrawal_address,
                 )
 
-                # drain the master safe and master signer for the home chain
+                # drain the Master Safe and master signer for the home chain
                 chain = Chain(service.home_chain)
                 master_wallet = service_manager.wallet_manager.load(
                     ledger_type=chain.ledger_type
                 )
 
-                # drain the master safe
+                # drain the Master Safe
                 logger.info(
                     f"Draining the Master Safe {master_wallet.safes[chain]} on chain {chain.value} (withdrawal address {withdrawal_address})."
                 )
@@ -1067,6 +1279,136 @@ def create_app(  # pylint: disable=too-many-locals, unused-argument, too-many-st
 
         return JSONResponse(content={"error": None, "message": "Withdrawal successful"})
 
+    @app.post("/api/v2/service/{service_config_id}/terminate_and_withdraw")
+    async def _terminate_and_withdraw(request: Request) -> JSONResponse:
+        """Terminate the service and withdraw all the funds to Master Safe"""
+
+        if operate.password is None:
+            return USER_NOT_LOGGED_IN_ERROR
+
+        service_config_id = request.path_params["service_config_id"]
+        service_manager = operate.service_manager()
+        wallet_manager = operate.wallet_manager
+
+        if not service_manager.exists(service_config_id=service_config_id):
+            return service_not_found_error(service_config_id=service_config_id)
+
+        try:
+            pause_all_services()
+            service = service_manager.load(service_config_id=service_config_id)
+            for chain in service.chain_configs:
+                wallet = wallet_manager.load(Chain(chain).ledger_type)
+                master_safe = wallet.safes[Chain(chain)]
+                service_manager.terminate_service_on_chain_from_safe(
+                    service_config_id=service_config_id,
+                    chain=chain,
+                )
+                service_manager.drain(
+                    service_config_id=service_config_id,
+                    chain_str=chain,
+                    withdrawal_address=master_safe,
+                )
+
+        except InsufficientFundsException as e:
+            logger.error(
+                f"Failed to terminate service and withdraw funds. Insufficient funds: {e}\n{traceback.format_exc()}"
+            )
+            return JSONResponse(
+                content={
+                    "error": f"Failed to terminate service and withdraw funds. Insufficient funds: {e}"
+                },
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(
+                f"Terminate service and withdraw funds failed: {e}\n{traceback.format_exc()}"
+            )
+            return JSONResponse(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                content={
+                    "error": "Failed to terminate service and withdraw funds. Please check the logs."
+                },
+            )
+
+        return JSONResponse(
+            content={
+                "error": None,
+                "message": "Terminate service and withdraw funds successful",
+            }
+        )
+
+    @app.post("/api/v2/service/{service_config_id}/fund")
+    async def fund_service(  # pylint: disable=too-many-return-statements
+        request: Request,
+    ) -> JSONResponse:
+        """Fund agent or service safe via Master Safe"""
+
+        if operate.password is None:
+            return USER_NOT_LOGGED_IN_ERROR
+
+        service_config_id = request.path_params["service_config_id"]
+        service_manager = operate.service_manager()
+
+        if not service_manager.exists(service_config_id=service_config_id):
+            return service_not_found_error(service_config_id=service_config_id)
+
+        try:
+            data = await request.json()
+            service_manager.fund_service(
+                service_config_id=service_config_id,
+                amounts=ChainAmounts(
+                    {
+                        chain_str: {
+                            address: {
+                                asset: int(amount) for asset, amount in assets.items()
+                            }
+                            for address, assets in addresses.items()
+                        }
+                        for chain_str, addresses in data.items()
+                    }
+                ),
+            )
+        except ValueError as e:
+            logger.error(
+                f"Failed to fund from Master Safe: {e}\n{traceback.format_exc()}"
+            )
+            return JSONResponse(
+                content={"error": str(e)},
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+        except InsufficientFundsException as e:
+            logger.error(
+                f"Failed to fund from Master Safe. Insufficient funds: {e}\n{traceback.format_exc()}"
+            )
+            return JSONResponse(
+                content={
+                    "error": f"Failed to fund from Master Safe. Insufficient funds: {e}"
+                },
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+        except FundingInProgressError as e:
+            logger.error(
+                f"Failed to fund from Master Safe: {e}\n{traceback.format_exc()}"
+            )
+            return JSONResponse(
+                content={"error": str(e)},
+                status_code=HTTPStatus.CONFLICT,
+            )
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error(
+                f"Failed to fund from Master Safe: {e}\n{traceback.format_exc()}"
+            )
+            return JSONResponse(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                content={
+                    "error": "Failed to fund from Master Safe. Please check the logs."
+                },
+            )
+
+        return JSONResponse(
+            content={"error": None, "message": "Funded from Master Safe successfully"}
+        )
+
     @app.post("/api/bridge/bridge_refill_requirements")
     async def _bridge_refill_requirements(request: Request) -> JSONResponse:
         """Get the bridge refill requirements."""
@@ -1274,6 +1616,11 @@ def _daemon(
     ] = None,
 ) -> None:
     """Launch operate daemon."""
+    # try automatically shutdown previous instance before creating the app
+    if TRY_TO_SHUTDOWN_PREVIOUS_INSTANCE:
+        app_single_instance = AppSingleInstance(port)
+        app_single_instance.shutdown_previous_instance()
+
     app = create_app(home=home)
 
     config_kwargs = {
@@ -1293,23 +1640,6 @@ def _daemon(
             }
         )
 
-    # try automatically shutdown previous instance
-    if TRY_TO_SHUTDOWN_PREVIOUS_INSTANCE:
-        url = f"http{'s' if ssl_keyfile and ssl_certfile else ''}://{host}:{port}/shutdown"
-        logger.info(f"trying to stop  previous instance with {url}")
-        try:
-            requests.get(
-                f"https://{host}:{port}/shutdown", timeout=3, verify=False  # nosec
-            )
-        except requests.exceptions.SSLError:
-            logger.warning("SSL failed, trying HTTP fallback...")
-            try:
-                requests.get(f"http://{host}:{port}/shutdown", timeout=3)
-            except Exception:  # pylint: disable=broad-except
-                logger.exception("Failed to stop previous instance")
-        except Exception:  # pylint: disable=broad-except
-            logger.exception("Failed to stop previous instance")
-
     server = Server(Config(**config_kwargs))
     app._server = server  # pylint: disable=protected-access
     server.run()