olas-operate-middleware 0.1.0rc59__py3-none-any.whl → 0.13.2__py3-none-any.whl

operate/services/service.py

@@ -23,16 +23,19 @@ import json
 import os
 import platform
 import shutil
-import signal
 import subprocess  # nosec
 import sys
+import tempfile
 import time
 import typing as t
-from copy import copy, deepcopy
+import uuid
+from copy import copy
 from dataclasses import dataclass
+from json import JSONDecodeError
 from pathlib import Path
+from traceback import print_exc
 
-import psutil
+import requests
 from aea.configurations.constants import (
     DEFAULT_LEDGER,
     LEDGER,
@@ -40,11 +43,13 @@ from aea.configurations.constants import (
     PRIVATE_KEY_PATH_SCHEMA,
     SKILL,
 )
-from aea.configurations.data_types import PackageType
+from aea.helpers.logging import setup_logger
 from aea.helpers.yaml_utils import yaml_dump, yaml_load, yaml_load_all
 from aea_cli_ipfs.ipfs_utils import IPFSTool
 from autonomy.cli.helpers.deployment import run_deployment, stop_deployment
-from autonomy.configurations.loader import load_service_config
+from autonomy.configurations.constants import DEFAULT_SERVICE_CONFIG_FILE
+from autonomy.configurations.loader import apply_env_variables, load_service_config
+from autonomy.constants import DEFAULT_KEYS_FILE, DOCKER_COMPOSE_YAML
 from autonomy.deploy.base import BaseDeploymentGenerator
 from autonomy.deploy.base import ServiceBuilder as BaseServiceBuilder
 from autonomy.deploy.constants import (
@@ -57,39 +62,61 @@ from autonomy.deploy.constants import (
     VENVS_DIR,
 )
 from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
+from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
 from docker import from_env
 
 from operate.constants import (
-    DEPLOYMENT,
+    AGENT_FUNDS_STATUS_URL,
+    AGENT_PERSISTENT_STORAGE_ENV_VAR,
+    CONFIG_JSON,
+    DEPLOYMENT_DIR,
     DEPLOYMENT_JSON,
-    DOCKER_COMPOSE_YAML,
-    KEYS_JSON,
+    HEALTHCHECK_JSON,
+    SERVICE_SAFE_PLACEHOLDER,
+    ZERO_ADDRESS,
 )
-from operate.http.exceptions import NotAllowed
-from operate.keys import Keys
-from operate.resource import LocalResource
-from operate.services.utils import tendermint
-from operate.types import (
-    ChainType,
+from operate.keys import KeysManager
+from operate.ledger import get_default_ledger_api, get_default_rpc
+from operate.operate_http.exceptions import NotAllowed
+from operate.operate_types import (
+    AgentRelease,
+    Chain,
+    ChainAmounts,
+    ChainConfig,
+    ChainConfigs,
     DeployedNodes,
     DeploymentConfig,
     DeploymentStatus,
+    EnvVariables,
     LedgerConfig,
-    LedgerType,
+    LedgerConfigs,
     OnChainData,
-    OnChainState,
     OnChainUserParams,
+    ServiceEnvProvisionType,
+    ServiceTemplate,
 )
+from operate.resource import LocalResource
+from operate.services.deployment_runner import run_host_deployment, stop_host_deployment
+from operate.services.utils import tendermint
+from operate.utils import unrecoverable_delete
+from operate.utils.gnosis import get_asset_balance
+from operate.utils.ssl import create_ssl_certificate
 
 
+# pylint: disable=no-member,redefined-builtin,too-many-instance-attributes,too-many-locals
+
 SAFE_CONTRACT_ADDRESS = "safe_contract_address"
 ALL_PARTICIPANTS = "all_participants"
 CONSENSUS_THRESHOLD = "consensus_threshold"
+SERVICE_CONFIG_VERSION = 9
+SERVICE_CONFIG_PREFIX = "sc-"
 
+NON_EXISTENT_MULTISIG = None
+NON_EXISTENT_TOKEN = -1
 
-# pylint: disable=no-member,redefined-builtin,too-many-instance-attributes
+AGENT_TYPE_IDS = {"mech": 37, "optimus": 40, "modius": 40, "trader": 25}
 
-DUMMY_MULTISIG = "0xm"
+logger = setup_logger("operate.services.service")
 
 
 def mkdirs(build_dir: Path) -> None:
@@ -135,42 +162,6 @@ def remove_service_network(service_name: str, force: bool = True) -> None:
 class ServiceBuilder(BaseServiceBuilder):
     """Service builder patch."""
 
-    def try_update_ledger_params(self, chain: str, address: str) -> None:
-        """Try to update the ledger params."""
-
-        for override in deepcopy(self.service.overrides):
-            (
-                override,
-                component_id,
-                _,
-            ) = self.service.process_metadata(
-                configuration=override,
-            )
-
-            if (
-                component_id.package_type == PackageType.CONNECTION
-                and component_id.name == "ledger"
-            ):
-                ledger_connection_overrides = deepcopy(override)
-                break
-        else:
-            return
-
-        # TODO: Support for multiple overrides
-        ledger_connection_overrides["config"]["ledger_apis"][chain]["address"] = address
-        service_overrides = deepcopy(self.service.overrides)
-        service_overrides = [
-            override
-            for override in service_overrides
-            if override["public_id"] != str(component_id.public_id)
-            or override["type"] != PackageType.CONNECTION.value
-        ]
-
-        ledger_connection_overrides["type"] = PackageType.CONNECTION.value
-        ledger_connection_overrides["public_id"] = str(component_id.public_id)
-        service_overrides.append(ledger_connection_overrides)
-        self.service.overrides = service_overrides
-
     def try_update_runtime_params(
         self,
         multisig_address: t.Optional[str] = None,
@@ -219,9 +210,12 @@ class ServiceBuilder(BaseServiceBuilder):
                     has_multiple_overrides=has_multiple_overrides,
                 )
                 if service_id is not None:
-                    override["models"]["params"]["args"][
-                        "on_chain_service_id"
-                    ] = service_id
+                    if has_multiple_overrides:
+                        os.environ["ON_CHAIN_SERVICE_ID"] = str(service_id)
+                    else:
+                        override["models"]["params"]["args"][
+                            "on_chain_service_id"
+                        ] = service_id
 
             override["type"] = component_id.package_type.value
             override["public_id"] = str(component_id.public_id)
@@ -236,22 +230,29 @@ class ServiceHelper:
         """Initialize object."""
         self.path = path
         self.config = load_service_config(service_path=path)
+        self.config.overrides = apply_env_variables(
+            self.config.overrides, os.environ.copy()
+        )
 
-    def ledger_config(self) -> "LedgerConfig":
-        """Get ledger config."""
-        # TODO: Multiledger/Multiagent support
+    def ledger_configs(self) -> LedgerConfigs:
+        """Get ledger configs."""
+        ledger_configs = {}
         for override in self.config.overrides:
             if (
                 override["type"] == "connection"
                 and "valory/ledger" in override["public_id"]
             ):
-                (_, config), *_ = override["config"]["ledger_apis"].items()
-                return LedgerConfig(
-                    rpc=config["address"],
-                    chain=ChainType.from_id(cid=config["chain_id"]),
-                    type=LedgerType.ETHEREUM,
-                )
-        raise ValueError("No ledger config found.")
+                if 0 in override:  # take the values from the first config
+                    override = override[0]
+
+                for _, config in override["config"]["ledger_apis"].items():
+                    # TODO chain name is inferred from the chain_id. The actual id provided on service.yaml is ignored.
+                    chain = Chain.from_id(chain_id=config["chain_id"])  # type: ignore
+                    ledger_configs[chain.value] = LedgerConfig(
+                        rpc=config["address"],
+                        chain=chain,
+                    )
+        return ledger_configs
 
     def deployment_config(self) -> DeploymentConfig:
         """Returns deployment config."""
@@ -268,11 +269,30 @@ class HostDeploymentGenerator(BaseDeploymentGenerator):
     def generate_config_tendermint(self) -> "HostDeploymentGenerator":
         """Generate tendermint configuration."""
         tmhome = str(self.build_dir / "node")
+        tendermint_executable = str(
+            shutil.which("tendermint"),
+        )
+        env = {}
+        env["PATH"] = os.path.dirname(sys.executable) + ":" + os.environ["PATH"]
+        tendermint_executable = str(
+            Path(os.path.dirname(sys.executable)) / "tendermint"
+        )
+
+        if platform.system() == "Windows":
+            env["PATH"] = os.path.dirname(sys.executable) + ";" + os.environ["PATH"]
+            tendermint_executable = str(
+                Path(os.path.dirname(sys.executable)) / "tendermint.exe"
+            )
+
+        if not (getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS")):
+            # we dont run inside pyinstaller, mean DEV mode!
+            tendermint_executable = "tendermint"
+            if platform.system() == "Windows":
+                tendermint_executable = "tendermint.exe"
+
         subprocess.run(  # pylint: disable=subprocess-run-check # nosec
             args=[
-                str(
-                    shutil.which("tendermint"),
-                ),
+                tendermint_executable,
                 "--home",
                 tmhome,
                 "init",
@@ -309,6 +329,8 @@ class HostDeploymentGenerator(BaseDeploymentGenerator):
         use_acn: bool = False,
     ) -> "HostDeploymentGenerator":
         """Generate agent and tendermint configurations"""
+        self.build_dir.mkdir(exist_ok=True, parents=True)
+        (self.build_dir / "agent").mkdir(exist_ok=True, parents=True)
         agent = self.service_builder.generate_agent(agent_n=0)
         agent = {key: f"{value}" for key, value in agent.items()}
         (self.build_dir / "agent.json").write_text(
@@ -338,201 +360,6 @@ class HostDeploymentGenerator(BaseDeploymentGenerator):
         return self
 
 
-def _run_cmd(args: t.List[str], cwd: t.Optional[Path] = None) -> None:
-    """Run command in a subprocess."""
-    print(f"Running: {' '.join(args)}")
-    # print working dir
-    print(f"Working dir: {os.getcwd()}")
-    result = subprocess.run(  # pylint: disable=subprocess-run-check # nosec
-        args=args,
-        cwd=cwd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-    if result.returncode != 0:
-        raise RuntimeError(f"Error running: {args} @ {cwd}\n{result.stderr.decode()}")
-
-
-def _setup_agent(working_dir: Path) -> None:
-    """Setup agent."""
-    env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8"))
-    # Patch for trader agent
-    if "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH" in env:
-        data_dir = working_dir / "data"
-        data_dir.mkdir(exist_ok=True)
-        env["SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH"] = str(data_dir)
-
-    # TODO: Dynamic port allocation, backport to service builder
-    env["CONNECTION_ABCI_CONFIG_HOST"] = "localhost"
-    env["CONNECTION_ABCI_CONFIG_PORT"] = "26658"
-
-    for var in env:
-        # Fix tendermint connection params
-        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"):
-            env[var] = "http://localhost:8080"
-
-        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"):
-            env[var] = "http://localhost:26657"
-
-        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_P2P_URL"):
-            env[var] = "localhost:26656"
-
-        if var.endswith("MODELS_BENCHMARK_TOOL_ARGS_LOG_DIR"):
-            benchmarks_dir = working_dir / "benchmarks"
-            benchmarks_dir.mkdir(exist_ok=True, parents=True)
-            env[var] = str(benchmarks_dir.resolve())
-
-    (working_dir / "agent.json").write_text(
-        json.dumps(env, indent=4),
-        encoding="utf-8",
-    )
-
-    abin = str(Path(sys._MEIPASS) / "aea_bin")  # type: ignore # pylint: disable=protected-access
-    # Fetch agent
-    _run_cmd(
-        args=[
-            abin,
-            "init",
-            "--reset",
-            "--author",
-            "valory",
-            "--remote",
-            "--ipfs",
-            "--ipfs-node",
-            "/dns/registry.autonolas.tech/tcp/443/https",
-        ],
-        cwd=working_dir,
-    )
-    _run_cmd(
-        args=[
-            abin,
-            "fetch",
-            env["AEA_AGENT"],
-            "--alias",
-            "agent",
-        ],
-        cwd=working_dir,
-    )
-
-    # Add keys
-    shutil.copy(
-        working_dir / "ethereum_private_key.txt",
-        working_dir / "agent" / "ethereum_private_key.txt",
-    )
-    _run_cmd(
-        args=[abin, "add-key", "ethereum"],
-        cwd=working_dir / "agent",
-    )
-    _run_cmd(
-        args=[abin, "issue-certificates"],
-        cwd=working_dir / "agent",
-    )
-
-
-def _start_agent(working_dir: Path) -> None:
-    """Start agent process."""
-    env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8"))
-    aea_bin = str(Path(sys._MEIPASS) / "aea_bin")  # type: ignore  # pylint: disable=protected-access
-    process = subprocess.Popen(  # pylint: disable=consider-using-with # nosec
-        args=[aea_bin, "run"],
-        cwd=working_dir / "agent",
-        stdout=subprocess.DEVNULL,
-        stderr=subprocess.DEVNULL,
-        env={**os.environ, **env},
-        creationflags=(
-            0x00000008 if platform.system() == "Windows" else 0
-        ),  # Detach process from the main process
-    )
-    (working_dir / "agent.pid").write_text(
-        data=str(process.pid),
-        encoding="utf-8",
-    )
-
-
-def _start_tendermint(working_dir: Path) -> None:
-    """Start tendermint process."""
-    env = json.loads((working_dir / "tendermint.json").read_text(encoding="utf-8"))
-    tendermint_com = str(Path(sys._MEIPASS) / "tendermint")  # type: ignore  # pylint: disable=protected-access
-    process = subprocess.Popen(  # pylint: disable=consider-using-with # nosec
-        args=[tendermint_com],
-        cwd=working_dir,
-        stdout=subprocess.DEVNULL,
-        stderr=subprocess.DEVNULL,
-        env={**os.environ, **env},
-        creationflags=(
-            0x00000008 if platform.system() == "Windows" else 0
-        ),  # Detach process from the main process
-    )
-    (working_dir / "tendermint.pid").write_text(
-        data=str(process.pid),
-        encoding="utf-8",
-    )
-
-
-def _kill_process(pid: int) -> None:
-    """Kill process."""
-    print(f"Trying to kill process: {pid}")
-    while True:
-        if not psutil.pid_exists(pid=pid):
-            return
-        if psutil.Process(pid=pid).status() in (
-            psutil.STATUS_DEAD,
-            psutil.STATUS_ZOMBIE,
-        ):
-            return
-        try:
-            os.kill(
-                pid,
-                (
-                    signal.CTRL_C_EVENT  # type: ignore
-                    if platform.platform() == "Windows"
-                    else signal.SIGKILL
-                ),
-            )
-        except OSError:
-            return
-        time.sleep(1)
-
-
-def _stop_agent(working_dir: Path) -> None:
-    """Start process."""
-    pid = working_dir / "agent.pid"
-    if not pid.exists():
-        return
-    _kill_process(int(pid.read_text(encoding="utf-8")))
-
-
-def _stop_tendermint(working_dir: Path) -> None:
-    """Start tendermint process."""
-    pid = working_dir / "tendermint.pid"
-    if not pid.exists():
-        return
-    _kill_process(int(pid.read_text(encoding="utf-8")))
-
-
-def run_host_deployment(build_dir: Path) -> None:
-    """Run host deployment."""
-    _setup_agent(
-        working_dir=build_dir,
-    )
-    _start_tendermint(
-        working_dir=build_dir,
-    )
-    _start_agent(
-        working_dir=build_dir,
-    )
-
-
-def stop_host_deployment(build_dir: Path) -> None:
-    """Stop host deployment."""
-    _stop_agent(
-        working_dir=build_dir,
-    )
-    _stop_tendermint(
-        working_dir=build_dir,
-    )
-
-
 @dataclass
 class Deployment(LocalResource):
     """Deployment resource for a service."""
@@ -541,7 +368,7 @@ class Deployment(LocalResource):
     nodes: DeployedNodes
     path: Path
 
-    _file = "deployment.json"
+    _file = DEPLOYMENT_JSON
 
     @staticmethod
     def new(path: Path) -> "Deployment":
@@ -564,9 +391,58 @@ class Deployment(LocalResource):
         """Load a service"""
         return super().load(path)  # type: ignore
 
+    def copy_previous_agent_run_logs(self) -> None:
+        """Copy previous agent logs."""
+        source_path = self.path / DEPLOYMENT_DIR / "agent" / "log.txt"
+        destination_path = self.path / "prev_log.txt"
+        if source_path.exists():
+            shutil.copy(source_path, destination_path)
+
+    def _build_kubernetes(self, keys_manager: KeysManager, force: bool = True) -> None:
+        """Build kubernetes deployment."""
+        k8s_build = self.path / DEPLOYMENT_DIR / "abci_build_k8s"
+        if k8s_build.exists() and force:
+            shutil.rmtree(k8s_build)
+        mkdirs(build_dir=k8s_build)
+
+        service = Service.load(path=self.path)
+        keys_file = self.path / DEFAULT_KEYS_FILE
+        keys_file.write_text(
+            json.dumps(
+                [
+                    keys_manager.get_decrypted(address)
+                    for address in service.agent_addresses
+                ],
+                indent=4,
+            ),
+            encoding="utf-8",
+        )
+        builder = ServiceBuilder.from_dir(
+            path=service.package_absolute_path,
+            keys_file=keys_file,
+            number_of_agents=len(service.agent_addresses),
+        )
+        unrecoverable_delete(keys_file)
+        builder.deplopyment_type = KubernetesGenerator.deployment_type
+        (
+            KubernetesGenerator(
+                service_builder=builder,
+                build_dir=k8s_build.resolve(),
+                use_tm_testnet_setup=True,
+                image_author=builder.service.author,
+            )
+            .generate()
+            .generate_config_tendermint()
+            .write_config()
+            .populate_private_keys()
+        )
+        print(f"Kubernetes deployment built on {k8s_build.resolve()}\n")
+
     def _build_docker(
         self,
+        keys_manager: KeysManager,
         force: bool = True,
+        chain: t.Optional[str] = None,
     ) -> None:
         """Build docker deployment."""
         service = Service.load(path=self.path)
@@ -577,23 +453,20 @@ class Deployment(LocalResource):
             force=force,
         )
 
-        build = self.path / DEPLOYMENT
+        build = self.path / DEPLOYMENT_DIR
         if build.exists() and not force:
             return
         if build.exists() and force:
+            self.copy_previous_agent_run_logs()
             shutil.rmtree(build)
         mkdirs(build_dir=build)
 
-        keys_file = self.path / KEYS_JSON
+        keys_file = self.path / DEFAULT_KEYS_FILE
         keys_file.write_text(
             json.dumps(
                 [
-                    {
-                        "address": key.address,
-                        "private_key": key.private_key,
-                        "ledger": key.ledger.name.lower(),
-                    }
-                    for key in service.keys
+                    keys_manager.get_decrypted(address)
+                    for address in service.agent_addresses
                 ],
                 indent=4,
             ),
@@ -601,36 +474,42 @@ class Deployment(LocalResource):
         )
         try:
             builder = ServiceBuilder.from_dir(
-                path=service.service_path,
+                path=service.package_absolute_path,
                 keys_file=keys_file,
-                number_of_agents=len(service.keys),
+                number_of_agents=len(service.agent_addresses),
             )
+            unrecoverable_delete(keys_file)
             builder.deplopyment_type = DockerComposeGenerator.deployment_type
             builder.try_update_abci_connection_params()
+
+            if not chain:
+                chain = service.home_chain
+
+            chain_config = service.chain_configs[chain]
+            chain_data = chain_config.chain_data
+
             builder.try_update_runtime_params(
-                multisig_address=service.chain_data.multisig,
-                agent_instances=service.chain_data.instances,
-                service_id=service.chain_data.token,
+                multisig_address=chain_data.multisig,
+                agent_instances=chain_data.instances,
+                service_id=chain_data.token,
                 consensus_threshold=None,
             )
-            # TODO: Support for multiledger
-            builder.try_update_ledger_params(
-                chain=LedgerType(service.ledger_config.type).name.lower(),
-                address=service.ledger_config.rpc,
-            )
 
-            # build deployment
+            # build docker-compose deployment
             (
                 DockerComposeGenerator(
                     service_builder=builder,
                     build_dir=build.resolve(),
                     use_tm_testnet_setup=True,
+                    image_author=builder.service.author,
                 )
                 .generate()
                 .generate_config_tendermint()
                 .write_config()
                 .populate_private_keys()
             )
+            print(f"Docker Compose deployment built on {build.resolve()} \n")
+
         except Exception as e:
             shutil.rmtree(build)
             raise e
@@ -657,16 +536,17 @@ class Deployment(LocalResource):
         for node in deployment["services"]:
             if "abci" in node:
                 deployment["services"][node]["volumes"].extend(_volumes)
-                if (
-                    "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_MECH_REQUEST_PRICE=0"
-                    in deployment["services"][node]["environment"]
-                ):
-                    deployment["services"][node]["environment"].remove(
-                        "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_MECH_REQUEST_PRICE=0"
-                    )
-                    deployment["services"][node]["environment"].append(
-                        "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_MECH_REQUEST_PRICE=10000000000000000"
-                    )
+                new_mappings = []
+                for mapping in deployment["services"][node]["volumes"]:
+                    if mapping.startswith("./data"):
+                        (self.path / "persistent_data").mkdir(
+                            exist_ok=True, parents=True
+                        )
+                        mapping = mapping.replace("./data", "../persistent_data")
+
+                    new_mappings.append(mapping)
+
+                deployment["services"][node]["volumes"] = new_mappings
 
         with (build / DOCKER_COMPOSE_YAML).open("w", encoding="utf-8") as stream:
             yaml_dump(data=deployment, stream=stream)
@@ -674,15 +554,30 @@ class Deployment(LocalResource):
         self.status = DeploymentStatus.BUILT
         self.store()
 
-    def _build_host(self, force: bool = True) -> None:
+    def _build_host(
+        self,
+        keys_manager: KeysManager,
+        force: bool = True,
+        chain: t.Optional[str] = None,
+        with_tm: bool = True,
+    ) -> None:
         """Build host depployment."""
-        build = self.path / DEPLOYMENT
+        build = self.path / DEPLOYMENT_DIR
         if build.exists() and not force:
             return
 
         if build.exists() and force:
             stop_host_deployment(build_dir=build)
-            shutil.rmtree(build)
+            try:
+                # sleep needed to ensure all processes closed/killed otherwise it will block directory removal on windows
+                time.sleep(3)
+                self.copy_previous_agent_run_logs()
+                shutil.rmtree(build)
+            except:  # noqa  # pylint: disable=bare-except
+                # sleep and try again. exception if fails
+                print_exc()
+                time.sleep(3)
+                shutil.rmtree(build)
 
         service = Service.load(path=self.path)
         if service.helper.config.number_of_agents > 1:
@@ -690,87 +585,129 @@ class Deployment(LocalResource):
                 "Host deployment currently only supports single agent deployments"
             )
 
-        keys_file = self.path / KEYS_JSON
+        if not chain:
+            chain = service.home_chain
+
+        chain_config = service.chain_configs[chain]
+        chain_data = chain_config.chain_data
+
+        keys_file = self.path / DEFAULT_KEYS_FILE
         keys_file.write_text(
             json.dumps(
-                [
-                    {
-                        "address": key.address,
-                        "private_key": key.private_key,
-                        "ledger": key.ledger.name.lower(),
-                    }
-                    for key in service.keys
-                ],
+                [keys_manager.get(address).json for address in service.agent_addresses],
                 indent=4,
             ),
             encoding="utf-8",
         )
         try:
             builder = ServiceBuilder.from_dir(
-                path=service.service_path,
+                path=service.package_absolute_path,
                 keys_file=keys_file,
-                number_of_agents=len(service.keys),
+                number_of_agents=len(service.agent_addresses),
             )
             builder.deplopyment_type = HostDeploymentGenerator.deployment_type
             builder.try_update_abci_connection_params()
             builder.try_update_runtime_params(
-                multisig_address=service.chain_data.multisig,
-                agent_instances=service.chain_data.instances,
-                service_id=service.chain_data.token,
+                multisig_address=chain_data.multisig,
+                agent_instances=chain_data.instances,
+                service_id=chain_data.token,
                 consensus_threshold=None,
             )
-            # TODO: Support for multiledger
-            builder.try_update_ledger_params(
-                chain=LedgerType(service.ledger_config.type).name.lower(),
-                address=service.ledger_config.rpc,
+
+            deployement_generator = HostDeploymentGenerator(
+                service_builder=builder,
+                build_dir=build.resolve(),
+                use_tm_testnet_setup=True,
             )
+            if with_tm:
+                deployement_generator.generate_config_tendermint()
 
-            (
-                HostDeploymentGenerator(
-                    service_builder=builder,
-                    build_dir=build.resolve(),
-                    use_tm_testnet_setup=True,
-                )
-                .generate_config_tendermint()
-                .generate()
-                .populate_private_keys()
+            deployement_generator.generate()
+            deployement_generator.populate_private_keys()
+
+            # Add keys
+            shutil.copy(
+                build / "ethereum_private_key.txt",
+                build / "agent" / "ethereum_private_key.txt",
             )
 
         except Exception as e:
-            shutil.rmtree(build)
+            if build.exists():
+                shutil.rmtree(build)
             raise e
 
-        # Mech price patch.
-        agent_vars = json.loads(Path(build, "agent.json").read_text(encoding="utf-8"))
-        if "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_MECH_REQUEST_PRICE" in agent_vars:
-            agent_vars[
-                "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_MECH_REQUEST_PRICE"
-            ] = "10000000000000000"
-            Path(build, "agent.json").write_text(
-                json.dumps(agent_vars, indent=4),
-                encoding="utf-8",
-            )
-
         self.status = DeploymentStatus.BUILT
         self.store()
 
     def build(
         self,
+        keys_manager: KeysManager,
         use_docker: bool = False,
+        use_kubernetes: bool = False,
         force: bool = True,
+        chain: t.Optional[str] = None,
     ) -> None:
         """
         Build a deployment
 
-        :param use_docker: Use docker deployment
+        :param use_docker: Use a Docker Compose deployment. If True, then no host deployment.
+        :param use_kubernetes: Build Kubernetes deployment. If True, then no host deployment.
         :param force: Remove existing deployment and build a new one
+        :param chain: Chain to set runtime parameters on the deployment (home_chain if not provided).
         :return: Deployment object
         """
-        if use_docker:
-            return self._build_docker(force=force)
-        return self._build_host(force=force)
+        # TODO: Maybe remove usage of chain and use home_chain always?
+        original_env = os.environ.copy()
+        service = Service.load(path=self.path)
 
-    def start(self, use_docker: bool = False) -> None:
+        if use_docker or use_kubernetes:
+            ssl_key_path, ssl_cert_path = create_ssl_certificate(
+                ssl_dir=service.path / PERSISTENT_DATA_DIR / "ssl"
+            )
+            service.update_env_variables_values(
+                {
+                    "STORE_PATH": "/data",
+                    "SSL_KEY_PATH": (
+                        Path("/data") / "ssl" / ssl_key_path.name
+                    ).as_posix(),
+                    "SSL_CERT_PATH": (
+                        Path("/data") / "ssl" / ssl_cert_path.name
+                    ).as_posix(),
+                }
+            )
+            service.consume_env_variables()
+            if use_docker:
+                self._build_docker(keys_manager=keys_manager, force=force, chain=chain)
+            if use_kubernetes:
+                self._build_kubernetes(keys_manager=keys_manager, force=force)
+        else:
+            ssl_key_path, ssl_cert_path = create_ssl_certificate(
+                ssl_dir=service.path / DEPLOYMENT_DIR / "ssl"
+            )
+            service.update_env_variables_values(
+                {
+                    "SSL_KEY_PATH": str(ssl_key_path),
+                    "SSL_CERT_PATH": str(ssl_cert_path),
+                }
+            )
+            service.consume_env_variables()
+            is_aea = service.agent_release["is_aea"]
+            self._build_host(
+                keys_manager=keys_manager,
+                force=force,
+                chain=chain,
+                with_tm=is_aea,
+            )
+
+        os.environ.clear()
+        os.environ.update(original_env)
+
+    def start(
+        self,
+        password: str,
+        use_docker: bool = False,
+        is_aea: bool = True,
+    ) -> None:
         """Start the service"""
         if self.status != DeploymentStatus.BUILT:
             raise NotAllowed(
@@ -782,9 +719,17 @@ class Deployment(LocalResource):
 
         try:
             if use_docker:
-                run_deployment(build_dir=self.path / "deployment", detach=True)
+                run_deployment(
+                    build_dir=self.path / DEPLOYMENT_DIR,
+                    detach=True,
+                    project_name=self.path.name,
+                )
             else:
-                run_host_deployment(build_dir=self.path / "deployment")
+                run_host_deployment(
+                    build_dir=self.path / DEPLOYMENT_DIR,
+                    password=password,
+                    is_aea=is_aea,
+                )
         except Exception:
             self.status = DeploymentStatus.BUILT
             self.store()
@@ -793,7 +738,12 @@ class Deployment(LocalResource):
         self.status = DeploymentStatus.DEPLOYED
         self.store()
 
-    def stop(self, use_docker: bool = False, force: bool = False) -> None:
+    def stop(
+        self,
+        use_docker: bool = False,
+        force: bool = False,
+        is_aea: bool = True,
+    ) -> None:
         """Stop the deployment."""
         if self.status != DeploymentStatus.DEPLOYED and not force:
             return
@@ -802,16 +752,20 @@ class Deployment(LocalResource):
         self.store()
 
         if use_docker:
-            stop_deployment(build_dir=self.path / "deployment")
+            stop_deployment(
+                build_dir=self.path / DEPLOYMENT_DIR,
+                project_name=self.path.name,
+            )
         else:
-            stop_host_deployment(build_dir=self.path / "deployment")
+            stop_host_deployment(build_dir=self.path / DEPLOYMENT_DIR, is_aea=is_aea)
 
         self.status = DeploymentStatus.BUILT
         self.store()
 
     def delete(self) -> None:
         """Delete the deployment."""
-        shutil.rmtree(self.path / "deployment")
+        build = self.path / DEPLOYMENT_DIR
+        shutil.rmtree(build)
         self.status = DeploymentStatus.DELETED
         self.store()
 
@@ -820,20 +774,43 @@ class Deployment(LocalResource):
 class Service(LocalResource):
     """Service class."""
 
-    hash: str
-    keys: Keys
-    ledger_config: LedgerConfig
-    chain_data: OnChainData
-
+    name: str
+    version: int
+    service_config_id: str
     path: Path
-    service_path: Path
-
-    name: t.Optional[str] = None
+    package_path: Path
+    hash: str
+    hash_history: t.Dict[int, str]
+    agent_release: AgentRelease
+    agent_addresses: t.List[str]
+    home_chain: str
+    chain_configs: ChainConfigs
+    description: str
+    env_variables: EnvVariables
 
     _helper: t.Optional[ServiceHelper] = None
     _deployment: t.Optional[Deployment] = None
 
-    _file = "config.json"
+    _file = CONFIG_JSON
+
+    @property
+    def json(self) -> t.Dict:
+        """To dictionary object."""
+        obj = super().json
+        obj["service_public_id"] = self.service_public_id()
+        return obj
+
+    @staticmethod
+    def determine_agent_id(service_name: str) -> int:
+        """Determine the appropriate agent ID based on service name."""
+        service_name_lower = service_name.lower()
+        if "mech" in service_name_lower:
+            return AGENT_TYPE_IDS["mech"]
+        if "optimus" in service_name_lower:
+            return AGENT_TYPE_IDS["optimus"]
+        if "modius" in service_name_lower:
+            return AGENT_TYPE_IDS["modius"]
+        return AGENT_TYPE_IDS["trader"]
 
     @classmethod
     def load(cls, path: Path) -> "Service":
@@ -844,7 +821,7 @@ class Service(LocalResource):
     def helper(self) -> ServiceHelper:
         """Get service helper."""
         if self._helper is None:
-            self._helper = ServiceHelper(path=self.service_path)
+            self._helper = ServiceHelper(path=self.package_absolute_path)
         return t.cast(ServiceHelper, self._helper)
 
     @property
@@ -852,54 +829,442 @@ class Service(LocalResource):
         """Load deployment object for the service."""
         if not (self.path / DEPLOYMENT_JSON).exists():
             self._deployment = Deployment.new(path=self.path)
-        if self._deployment is None:
+        try:
             self._deployment = Deployment.load(path=self.path)
+        except JSONDecodeError:
+            self._deployment = Deployment.new(path=self.path)
         return t.cast(Deployment, self._deployment)
 
+    @property
+    def package_absolute_path(self) -> Path:
+        """Get the package_absolute_path."""
+        self._ensure_package_exists()
+        package_absolute_path = self.path / self.package_path
+        return package_absolute_path
+
+    def _ensure_package_exists(self) -> None:
+        package_absolute_path = self.path / self.package_path
+        if (
+            not package_absolute_path.exists()
+            or not (package_absolute_path / DEFAULT_SERVICE_CONFIG_FILE).exists()
+        ):
+            with tempfile.TemporaryDirectory(dir=self.path) as temp_dir:
+                package_temp_path = Path(
+                    IPFSTool().download(
+                        hash_id=self.hash,
+                        target_dir=temp_dir,
+                    )
+                )
+                target_path = self.path / package_temp_path.name
+
+                if target_path.exists():
+                    shutil.rmtree(target_path)
+
+                shutil.move(package_temp_path, target_path)
+                self.package_path = Path(target_path.name)
+                self.store()
+
     @staticmethod
-    def new(
-        hash: str,
-        keys: Keys,
-        rpc: str,
-        on_chain_user_params: OnChainUserParams,
+    def new(  # pylint: disable=too-many-locals
+        agent_addresses: t.List[str],
+        service_template: ServiceTemplate,
         storage: Path,
     ) -> "Service":
         """Create a new service."""
-        path = storage / hash
+
+        service_config_id = Service.get_new_service_config_id(storage)
+        path = storage / service_config_id
         path.mkdir()
-        service_path = Path(
+        package_absolute_path = Path(
             IPFSTool().download(
-                hash_id=hash,
+                hash_id=service_template["hash"],
                 target_dir=path,
             )
         )
-        with (service_path / "service.yaml").open("r", encoding="utf-8") as fp:
-            config, *_ = yaml_load_all(fp)
 
-        ledger_config = ServiceHelper(path=service_path).ledger_config()
-        service = Service(
-            name=config["author"] + "/" + config["name"],
-            hash=hash,
-            keys=keys,
-            ledger_config=LedgerConfig(
-                rpc=rpc,
-                type=ledger_config.type,
-                chain=ledger_config.chain,
-            ),
-            chain_data=OnChainData(
+        chain_configs = {}
+        for chain_str, config in service_template["configurations"].items():
+            chain = Chain(chain_str)
+            ledger_config = LedgerConfig(
+                rpc=get_default_rpc(Chain(chain_str)), chain=chain
+            )
+            ledger_config.rpc = config["rpc"]
+
+            chain_data = OnChainData(
                 instances=[],
-                token=-1,
-                multisig=DUMMY_MULTISIG,
-                staked=False,
-                on_chain_state=OnChainState.NOTMINTED,
-                user_params=on_chain_user_params,
-            ),
-            path=service_path.parent,
-            service_path=service_path,
+                token=NON_EXISTENT_TOKEN,
+                multisig=NON_EXISTENT_MULTISIG,
+                user_params=OnChainUserParams.from_json(config),  # type: ignore
+            )
+
+            chain_configs[chain_str] = ChainConfig(
+                ledger_config=ledger_config,
+                chain_data=chain_data,
+            )
+
+        current_timestamp = int(time.time())
+        service = Service(
+            version=SERVICE_CONFIG_VERSION,
+            service_config_id=service_config_id,
+            name=service_template["name"],
+            description=service_template["description"],
+            hash=service_template["hash"],
+            agent_addresses=agent_addresses,
+            home_chain=service_template["home_chain"],
+            hash_history={current_timestamp: service_template["hash"]},
+            chain_configs=chain_configs,
+            path=package_absolute_path.parent,
+            package_path=Path(package_absolute_path.name),
+            env_variables=service_template["env_variables"],
+            agent_release=service_template["agent_release"],
         )
         service.store()
         return service
 
-    def delete(self) -> None:
-        """Delete a service."""
-        shutil.rmtree(self.path)
+    def service_public_id(self, include_version: bool = True) -> str:
+        """Get the public id (based on the service hash)."""
+        with (self.package_absolute_path / DEFAULT_SERVICE_CONFIG_FILE).open(
+            "r", encoding="utf-8"
+        ) as fp:
+            service_yaml, *_ = yaml_load_all(fp)
+
+        public_id = f"{service_yaml['author']}/{service_yaml['name']}"
+
+        if include_version:
+            public_id += f":{service_yaml['version']}"
+
+        return public_id
+
+    @staticmethod
+    def get_service_public_id(
+        hash: str, temp_dir: t.Optional[Path] = None, include_version: bool = True
+    ) -> str:
+        """
+        Get the service public ID from IPFS based on the hash.
+
+        :param hash: The IPFS hash of the service.
+        :param dir: Optional directory path where the temporary download folder will be created.
+                    If None, a system-default temporary directory will be used.
+        :return: The public ID of the service in the format "author/name:version".
+        """
+        with tempfile.TemporaryDirectory(dir=temp_dir) as path:
+            package_path = Path(
+                IPFSTool().download(
+                    hash_id=hash,
+                    target_dir=path,
+                )
+            )
+
+            with (package_path / DEFAULT_SERVICE_CONFIG_FILE).open(
+                "r", encoding="utf-8"
+            ) as fp:
+                service_yaml, *_ = yaml_load_all(fp)
+
+            public_id = f"{service_yaml['author']}/{service_yaml['name']}"
+
+            if include_version:
+                public_id += f":{service_yaml['version']}"
+
+            return public_id
+
+    @staticmethod
+    def get_new_service_config_id(path: Path) -> str:
+        """Get a new service config id that does not clash with any directory in path."""
+        while True:
+            service_config_id = f"{SERVICE_CONFIG_PREFIX}{uuid.uuid4()}"
+            new_path = path.parent / service_config_id
+            if not new_path.exists():
+                return service_config_id
+
+    def get_latest_healthcheck(self) -> t.Dict:
+        """Return the latest stored healthcheck.json"""
+        healthcheck_json_path = self.path / HEALTHCHECK_JSON
+
+        if not healthcheck_json_path.exists():
+            return {}
+
+        try:
+            with open(healthcheck_json_path, "r", encoding="utf-8") as file:
+                return json.load(file)
+        except (IOError, json.JSONDecodeError) as e:
+            return {"error": f"Error reading healthcheck.json: {e}"}
+
+    def remove_latest_healthcheck(self) -> None:
+        """Remove the latest healthcheck.json, if it exists"""
+        healthcheck_json_path = self.path / HEALTHCHECK_JSON
+
+        if healthcheck_json_path.exists():
+            try:
+                healthcheck_json_path.unlink()
+            except Exception as e:  # pylint: disable=broad-except
+                print(f"Exception deleting {healthcheck_json_path}: {e}")
+
+    def get_agent_performance(self) -> t.Dict:
+        """Return the agent activity"""
+
+        # Default values
+        agent_performance: t.Dict[str, t.Any] = {
+            "timestamp": None,
+            "metrics": [],
+            "last_activity": None,
+            "last_chat_message": None,
+        }
+
+        agent_performance_json_path = (
+            Path(
+                self.env_variables.get(
+                    AGENT_PERSISTENT_STORAGE_ENV_VAR, {"value": "."}
+                ).get("value", ".")
+            )
+            / "agent_performance.json"
+        )
+
+        if agent_performance_json_path.exists():
+            try:
+                with open(agent_performance_json_path, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                if isinstance(data, dict):
+                    agent_performance.update(data)
+            except (json.JSONDecodeError, OSError) as e:
+                # Keep default values if file is invalid
+                print(
+                    f"Error reading file 'agent_performance.json': {e}"
+                )  # TODO Use logger
+
+        return dict(sorted(agent_performance.items()))
+
+    def update(
+        self,
+        service_template: ServiceTemplate,
+        allow_different_service_public_id: bool = False,
+        partial_update: bool = False,
+    ) -> None:
+        """Update service."""
+
+        target_hash = service_template.get("hash")
+        if target_hash:
+            target_service_public_id = Service.get_service_public_id(
+                target_hash, self.path
+            )
+
+            if not allow_different_service_public_id and (
+                self.service_public_id() != target_service_public_id
+            ):
+                raise ValueError(
+                    f"Trying to update a service with a different public id: {self.service_public_id()=} {self.hash=} {target_service_public_id=} {target_hash=}."
+                )
+
+        self.hash = service_template.get("hash", self.hash)
+
+        # hash_history - Only update if latest inserted hash is different
+        if self.hash_history[max(self.hash_history.keys())] != self.hash:
+            current_timestamp = int(time.time())
+            self.hash_history[current_timestamp] = self.hash
+
+        self.home_chain = service_template.get("home_chain", self.home_chain)
+        self.description = service_template.get("description", self.description)
+        self.name = service_template.get("name", self.name)
+
+        package_absolute_path = self.path / self.package_path
+        if package_absolute_path.exists():
+            shutil.rmtree(package_absolute_path)
+
+        package_absolute_path = Path(
+            IPFSTool().download(
+                hash_id=self.hash,
+                target_dir=self.path,
+            )
+        )
+        self.package_path = Path(package_absolute_path.name)
+
+        self.agent_release = service_template.get("agent_release", self.agent_release)
+
+        # env_variables
+        if partial_update:
+            for var, attrs in service_template.get("env_variables", {}).items():
+                self.env_variables.setdefault(var, {}).update(attrs)
+        else:
+            self.env_variables = service_template["env_variables"]
+
+        # chain_configs
+        # TODO support remove chains for non-partial updates
+        # TODO ensure all and only existing chains are passed for non-partial updates
+        ledger_configs = ServiceHelper(path=self.package_absolute_path).ledger_configs()
+        for chain, new_config in service_template.get("configurations", {}).items():
+            if chain in self.chain_configs:
+                # The template is providing a chain configuration that already
+                # exists in this service - update only the user parameters.
+                # This is to avoid losing on-chain data like safe, token, etc.
+                if partial_update:
+                    config = self.chain_configs[chain].chain_data.user_params.json
+                    config.update(new_config)
+                else:
+                    config = new_config
+
+                self.chain_configs[
+                    chain
+                ].chain_data.user_params = OnChainUserParams.from_json(
+                    config  # type: ignore
+                )
+            else:
+                # The template is providing a chain configuration that does
+                # not currently exist in this service - copy all config as
+                # when creating a new service.
+                ledger_config = ledger_configs[chain]
+                ledger_config.rpc = new_config["rpc"]
+
+                chain_data = OnChainData(
+                    instances=[],
+                    token=NON_EXISTENT_TOKEN,
+                    multisig=NON_EXISTENT_MULTISIG,
+                    user_params=OnChainUserParams.from_json(new_config),  # type: ignore
+                )
+
+                self.chain_configs[chain] = ChainConfig(
+                    ledger_config=ledger_config,
+                    chain_data=chain_data,
+                )
+
+        self.store()
+
+    def update_user_params_from_template(
+        self, service_template: ServiceTemplate
+    ) -> None:
+        """Update user params from template."""
+        for chain, config in service_template["configurations"].items():
+            self.chain_configs[
+                chain
+            ].chain_data.user_params = OnChainUserParams.from_json(
+                config  # type: ignore
+            )
+
+            self.chain_configs[chain].ledger_config.rpc = config["rpc"]
+
+        self.store()
+
+    def consume_env_variables(self) -> None:
+        """Consume (apply) environment variables.
+
+        Note that this method modifies os.environ. Consider if you need a backup of os.environ before using this method.
+        """
+        for env_var, attributes in self.env_variables.items():
+            os.environ[env_var] = str(attributes.get("value", ""))
+
+    def update_env_variables_values(
+        self, env_var_to_value: t.Dict[str, t.Any], except_if_undefined: bool = False
+    ) -> None:
+        """
+        Updates and stores the values of the env variables to override service.yaml on the deployment.
+
+        This method does not apply the variables to the environment. Use consume_env_variables to apply the
+        env variables.
+        """
+
+        updated = False
+        for var, value in env_var_to_value.items():
+            value_str = str(value)
+            attributes = self.env_variables.get(var)
+            if (
+                attributes
+                and self.env_variables[var]["provision_type"]
+                == ServiceEnvProvisionType.COMPUTED
+                and attributes["value"] != value_str
+            ):
+                attributes["value"] = value_str
+                updated = True
+            elif except_if_undefined:
+                raise ValueError(
+                    f"Trying to set value for an environment variable ({var}) not present on service configuration {self.service_config_id}."
+                )
+
+        if updated:
+            self.store()
+
+    def get_initial_funding_amounts(self) -> ChainAmounts:
+        """Get funding amounts as a dict structure."""
+        amounts = ChainAmounts()
+
+        for chain_str, chain_config in self.chain_configs.items():
+            fund_requirements = chain_config.chain_data.user_params.fund_requirements
+            service_safe = chain_config.chain_data.multisig
+
+            if service_safe is None or service_safe == ZERO_ADDRESS:
+                service_safe = SERVICE_SAFE_PLACEHOLDER
+
+            chain_amounts = amounts.setdefault(chain_str, {})
+            for asset, req in fund_requirements.items():
+                chain_amounts.setdefault(service_safe, {})[asset] = req.safe
+                for agent_address in self.agent_addresses:
+                    chain_amounts.setdefault(agent_address, {})[asset] = req.agent
+
+        return amounts
+
+    def get_balances(self) -> ChainAmounts:
+        """Get balances of the agent addresses and service safe."""
+        initial_funding_amounts = self.get_initial_funding_amounts()
+        return ChainAmounts(
+            {
+                chain_str: {
+                    address: {
+                        asset: get_asset_balance(
+                            ledger_api=get_default_ledger_api(
+                                Chain.from_string(chain_str)
+                            ),
+                            asset_address=asset,
+                            address=address,
+                            raise_on_invalid_address=False,
+                        )
+                        for asset in tokens
+                    }
+                    for address, tokens in addresses.items()
+                }
+                for chain_str, addresses in initial_funding_amounts.items()
+            }
+        )
+
+    def get_funding_requests(self) -> ChainAmounts:
+        """Get funding amounts requested by the agent."""
+        agent_response = {}
+        funding_requests = ChainAmounts()
+
+        if self.deployment.status != DeploymentStatus.DEPLOYED:
+            return funding_requests
+
+        try:
+            resp = requests.get(AGENT_FUNDS_STATUS_URL, timeout=10)
+            resp.raise_for_status()
+            agent_response = resp.json()
+        except Exception as e:  # pylint: disable=broad-except
+            logger.warning(
+                f"[FUNDING MANAGER] Cannot read url {AGENT_FUNDS_STATUS_URL}: {e}"
+            )
+
+        for chain_str, addresses in agent_response.items():
+            for address, assets in addresses.items():
+                if chain_str not in self.chain_configs:
+                    raise ValueError(
+                        f"Service {self.service_config_id} asked funding for an unknown chain {chain_str}."
+                    )
+
+                if (
+                    address not in self.agent_addresses
+                    and address != self.chain_configs[chain_str].chain_data.multisig
+                ):
+                    raise ValueError(
+                        f"Service {self.service_config_id} asked funding for an unknown address {address} on chain {chain_str}."
+                    )
+
+                funding_requests.setdefault(chain_str, {})
+                funding_requests[chain_str].setdefault(address, {})
+                for asset, amounts in assets.items():
+                    try:
+                        funding_requests[chain_str][address][asset] = int(
+                            amounts["deficit"]
+                        )
+                    except (ValueError, TypeError):
+                        logger.warning(
+                            f"[FUNDING MANAGER] Invalid funding amount {amounts['deficit']} for asset {asset} on chain {chain_str} for address {address}. Setting to 0."
+                        )
+                        funding_requests[chain_str][address][asset] = 0
+
+        return funding_requests