databricks-labs-lakebridge 0.10.0__py3-none-any.whl

databricks/labs/lakebridge/deployment/configurator.py

@@ -0,0 +1,199 @@
+import logging
+import time
+
+from databricks.labs.blueprint.tui import Prompts
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.service.catalog import Privilege, SecurableType
+from databricks.sdk.service.sql import (
+    CreateWarehouseRequestWarehouseType,
+    EndpointInfoWarehouseType,
+    SpotInstancePolicy,
+)
+
+from databricks.labs.lakebridge.helpers.metastore import CatalogOperations
+
+logger = logging.getLogger(__name__)
+
+
+class ResourceConfigurator:
+    """
+    Handles the setup of common Databricks resources like
+    catalogs, schemas, volumes, and warehouses used across remorph modules.
+    """
+
+    def __init__(self, ws: WorkspaceClient, prompts: Prompts, catalog_ops: CatalogOperations):
+        self._ws = ws
+        self._user = ws.current_user.me()
+        self._prompts = prompts
+        self._catalog_ops = catalog_ops
+
+    def prompt_for_catalog_setup(
+        self,
+    ) -> str:
+        catalog_name = self._prompts.question("Enter catalog name", default="remorph")
+        catalog = self._catalog_ops.get_catalog(catalog_name)
+        if catalog:
+            logger.info(f"Found existing catalog `{catalog_name}`")
+            return catalog_name
+        if self._prompts.confirm(f"Catalog `{catalog_name}` doesn't exist. Create it?"):
+            result = self._catalog_ops.create_catalog(catalog_name)
+            assert result.name is not None
+            return result.name
+        raise SystemExit("Cannot continue installation, without a valid catalog, Aborting the installation.")
+
+    def prompt_for_schema_setup(
+        self,
+        catalog: str,
+        default_schema_name: str,
+    ) -> str:
+        schema_name = self._prompts.question("Enter schema name", default=default_schema_name)
+        schema = self._catalog_ops.get_schema(catalog, schema_name)
+        if schema:
+            logger.info(f"Found existing schema `{schema_name}` in catalog `{catalog}`")
+            return schema_name
+        if self._prompts.confirm(f"Schema `{schema_name}` doesn't exist in catalog `{catalog}`. Create it?"):
+            result = self._catalog_ops.create_schema(schema_name, catalog)
+            assert result.name is not None
+            return result.name
+        raise SystemExit("Cannot continue installation, without a valid schema. Aborting the installation.")
+
+    def prompt_for_volume_setup(
+        self,
+        catalog: str,
+        schema: str,
+        default_volume_name: str,
+    ) -> str:
+        volume_name = self._prompts.question("Enter volume name", default=default_volume_name)
+        volume = self._catalog_ops.get_volume(catalog, schema, volume_name)
+        if volume:
+            logger.info(f"Found existing volume `{volume_name}` in catalog `{catalog}` and schema `{schema}`")
+            return volume_name
+        if self._prompts.confirm(
+            f"Volume `{volume_name}` doesn't exist in catalog `{catalog}` and schema `{schema}`. Create it?"
+        ):
+            result = self._catalog_ops.create_volume(catalog, schema, volume_name)
+            assert result.name is not None
+            return result.name
+        raise SystemExit("Cannot continue installation, without a valid volume. Aborting the installation.")
+
+    def prompt_for_warehouse_setup(self, warehouse_name_prefix: str) -> str:
+        def warehouse_type(_):
+            return _.warehouse_type.value if not _.enable_serverless_compute else "SERVERLESS"
+
+        pro_warehouses = {"[Create new PRO SQL warehouse]": "create_new"} | {
+            f"{_.name} ({_.id}, {warehouse_type(_)}, {_.state.value})": _.id
+            for _ in self._ws.warehouses.list()
+            if _.warehouse_type == EndpointInfoWarehouseType.PRO
+        }
+        warehouse_id = self._prompts.choice_from_dict(
+            "Select PRO or SERVERLESS SQL warehouse",
+            pro_warehouses,
+        )
+        if warehouse_id == "create_new":
+            new_warehouse = self._ws.warehouses.create(
+                name=f"{warehouse_name_prefix} {time.time_ns()}",
+                spot_instance_policy=SpotInstancePolicy.COST_OPTIMIZED,
+                warehouse_type=CreateWarehouseRequestWarehouseType.PRO,
+                cluster_size="Small",
+                max_num_clusters=1,
+            )
+            warehouse_id = new_warehouse.id
+        return warehouse_id
+
+    def has_necessary_catalog_access(
+        self, catalog_name: str, user_name: str, privilege_sets: tuple[set[Privilege], ...]
+    ):
+        catalog = self._catalog_ops.get_catalog(catalog_name)
+        assert catalog, f"Catalog not found {catalog_name}"
+        if self._catalog_ops.has_catalog_access(catalog, user_name, privilege_sets):
+            return True
+        missing_permissions = self._get_missing_permissions(
+            user_name, SecurableType.CATALOG, catalog.name, privilege_sets
+        )
+        logger.error(
+            f"User `{user_name}` doesn't have required privileges :: \n`{missing_permissions}`\n to access catalog `{catalog_name}` "
+        )
+        return False
+
+    def has_necessary_schema_access(
+        self, catalog_name: str, schema_name: str, user_name: str, privilege_sets: tuple[set[Privilege], ...]
+    ):
+        schema = self._catalog_ops.get_schema(catalog_name, schema_name)
+        assert schema, f"Schema not found {catalog_name}.{schema_name}"
+        if self._catalog_ops.has_schema_access(schema, user_name, privilege_sets):
+            return True
+        missing_permissions = self._get_missing_permissions(
+            user_name, SecurableType.SCHEMA, schema.full_name, privilege_sets
+        )
+        logger.error(
+            f"User `{user_name}` doesn't have required privileges :: \n`{missing_permissions}`\n to access schema `{schema.full_name}` "
+        )
+        return False
+
+    def has_necessary_volume_access(
+        self,
+        catalog_name: str,
+        schema_name: str,
+        volume_name: str,
+        user_name: str,
+        privilege_sets: tuple[set[Privilege], ...],
+    ):
+        volume = self._catalog_ops.get_volume(catalog_name, schema_name, volume_name)
+        assert volume, f"Volume not found {catalog_name}.{schema_name}.{volume_name}"
+        if self._catalog_ops.has_volume_access(volume, user_name, privilege_sets):
+            return True
+        missing_permissions = self._get_missing_permissions(
+            user_name, SecurableType.VOLUME, volume.full_name, privilege_sets
+        )
+        logger.error(
+            f"User `{user_name}` doesn't have required privileges :: \n`{missing_permissions}`\n to access volume `{volume.full_name}` "
+        )
+        return False
+
+    def _get_missing_permissions(
+        self,
+        user_name: str,
+        securable_type: SecurableType,
+        resource_name: str | None,
+        privilege_sets: tuple[set[Privilege], ...],
+    ):
+        assert resource_name, f"Catalog Resource name must be provided {resource_name}"
+        missing_permissions_list = []
+        for privilege_set in privilege_sets:
+            permissions = self._catalog_ops.has_privileges(user_name, securable_type, resource_name, privilege_set)
+            if not permissions:
+                missing_privileges = ", ".join([privilege.name for privilege in privilege_set])
+                missing_permissions_list.append(f" * '{missing_privileges}' ")
+
+        return " OR \n".join(missing_permissions_list)
+
+    def has_necessary_access(self, catalog_name: str, schema_name: str, volume_name: str | None):
+        catalog_required_privileges: tuple[set[Privilege], ...] = (
+            {Privilege.ALL_PRIVILEGES},
+            {Privilege.USE_CATALOG},
+        )
+        schema_required_privileges: tuple[set[Privilege], ...] = (
+            {Privilege.ALL_PRIVILEGES},
+            {Privilege.USE_SCHEMA, Privilege.MODIFY, Privilege.SELECT, Privilege.CREATE_VOLUME},
+            {Privilege.USE_SCHEMA, Privilege.MODIFY, Privilege.SELECT},
+        )
+        volume_required_privileges: tuple[set[Privilege], ...] = (
+            {Privilege.ALL_PRIVILEGES},
+            {Privilege.READ_VOLUME, Privilege.WRITE_VOLUME},
+        )
+
+        user_name = self._user.user_name
+        assert user_name is not None
+
+        catalog_access = self.has_necessary_catalog_access(catalog_name, user_name, catalog_required_privileges)
+        schema_access = self.has_necessary_schema_access(
+            catalog_name, schema_name, user_name, schema_required_privileges
+        )
+        required_access = catalog_access and schema_access
+        if volume_name:
+            volume_access = self.has_necessary_volume_access(
+                catalog_name, schema_name, volume_name, user_name, volume_required_privileges
+            )
+            required_access = required_access and volume_access
+        if not required_access:
+            raise SystemExit("Cannot continue installation, without necessary access. Aborting the installation.")

databricks/labs/lakebridge/deployment/dashboard.py

@@ -0,0 +1,140 @@
+import logging
+from datetime import timedelta
+from pathlib import Path
+
+from databricks.labs.blueprint.installation import Installation
+from databricks.labs.blueprint.installer import InstallState
+from databricks.labs.lsql.dashboards import DashboardMetadata, Dashboards
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.errors import (
+    InvalidParameterValue,
+    NotFound,
+    DeadlineExceeded,
+    InternalError,
+    ResourceAlreadyExists,
+)
+from databricks.sdk.retries import retried
+from databricks.sdk.service.dashboards import LifecycleState, Dashboard
+
+from databricks.labs.lakebridge.config import ReconcileConfig, ReconcileMetadataConfig
+
+logger = logging.getLogger(__name__)
+
+
+class DashboardDeployment:
+
+    def __init__(
+        self,
+        ws: WorkspaceClient,
+        installation: Installation,
+        install_state: InstallState,
+    ):
+        self._ws = ws
+        self._installation = installation
+        self._install_state = install_state
+
+    def deploy(
+        self,
+        folder: Path,
+        config: ReconcileConfig,
+    ):
+        """
+        Create dashboards from Dashboard metadata files.
+        The given folder is expected to contain subfolders each containing metadata for individual dashboards.
+
+        :param folder: Path to the base folder.
+        :param config: Configuration for reconciliation.
+        """
+        logger.info(f"Deploying dashboards from base folder {folder}")
+        parent_path = f"{self._installation.install_folder()}/dashboards"
+        try:
+            self._ws.workspace.mkdirs(parent_path)
+        except ResourceAlreadyExists:
+            logger.info(f"Dashboard parent path already exists: {parent_path}")
+
+        valid_dashboard_refs = set()
+        for dashboard_folder in folder.iterdir():
+            if not dashboard_folder.is_dir():
+                continue
+            valid_dashboard_refs.add(self._dashboard_reference(dashboard_folder))
+            dashboard = self._update_or_create_dashboard(dashboard_folder, parent_path, config.metadata_config)
+            logger.info(
+                f"Dashboard deployed with URL: {self._ws.config.host}/sql/dashboardsv3/{dashboard.dashboard_id}"
+            )
+            self._install_state.save()
+
+        self._remove_deprecated_dashboards(valid_dashboard_refs)
+
+    def _dashboard_reference(self, folder: Path) -> str:
+        return f"{folder.stem}".lower()
+
+    # InternalError and DeadlineExceeded are retried because of Lakeview internal issues
+    # These issues have been reported to and are resolved by the Lakeview team
+    # Keeping the retry for resilience
+    @retried(on=[InternalError, DeadlineExceeded], timeout=timedelta(minutes=3))
+    def _update_or_create_dashboard(
+        self,
+        folder: Path,
+        ws_parent_path: str,
+        config: ReconcileMetadataConfig,
+    ) -> Dashboard:
+        logging.info(f"Reading dashboard folder {folder}")
+        metadata = DashboardMetadata.from_path(folder).replace_database(
+            catalog=config.catalog,
+            catalog_to_replace="remorph",
+            database=config.schema,
+            database_to_replace="reconcile",
+        )
+
+        metadata.display_name = self._name_with_prefix(metadata.display_name)
+        reference = self._dashboard_reference(folder)
+        dashboard_id = self._install_state.dashboards.get(reference)
+        if dashboard_id is not None:
+            try:
+                dashboard_id = self._handle_existing_dashboard(dashboard_id, metadata.display_name)
+            except (NotFound, InvalidParameterValue):
+                logger.info(f"Recovering invalid dashboard: {metadata.display_name} ({dashboard_id})")
+                try:
+                    dashboard_path = f"{ws_parent_path}/{metadata.display_name}.lvdash.json"
+                    self._ws.workspace.delete(dashboard_path)  # Cannot recreate dashboard if file still exists
+                    logger.debug(
+                        f"Deleted dangling dashboard {metadata.display_name} ({dashboard_id}): {dashboard_path}"
+                    )
+                except NotFound:
+                    pass
+                dashboard_id = None  # Recreate the dashboard if it's reference is corrupted (manually)
+
+        dashboard = Dashboards(self._ws).create_dashboard(
+            metadata,
+            dashboard_id=dashboard_id,
+            parent_path=ws_parent_path,
+            warehouse_id=self._ws.config.warehouse_id,
+            publish=True,
+        )
+        assert dashboard.dashboard_id is not None
+        self._install_state.dashboards[reference] = dashboard.dashboard_id
+        return dashboard
+
+    def _name_with_prefix(self, name: str) -> str:
+        prefix = self._installation.product()
+        return f"{prefix.upper()}_{name}".replace(" ", "_")
+
+    def _handle_existing_dashboard(self, dashboard_id: str, display_name: str) -> str | None:
+        dashboard = self._ws.lakeview.get(dashboard_id)
+        if dashboard.lifecycle_state is None:
+            raise NotFound(f"Dashboard life cycle state: {display_name} ({dashboard_id})")
+        if dashboard.lifecycle_state == LifecycleState.TRASHED:
+            logger.info(f"Recreating trashed dashboard: {display_name} ({dashboard_id})")
+            return None  # Recreate the dashboard if it is trashed (manually)
+        return dashboard_id  # Update the existing dashboard
+
+    def _remove_deprecated_dashboards(self, valid_dashboard_refs: set[str]):
+        for ref, dashboard_id in self._install_state.dashboards.items():
+            if ref not in valid_dashboard_refs:
+                try:
+                    logger.info(f"Removing dashboard_id={dashboard_id}, as it is no longer needed.")
+                    del self._install_state.dashboards[ref]
+                    self._ws.lakeview.trash(dashboard_id)
+                except (InvalidParameterValue, NotFound):
+                    logger.warning(f"Dashboard `{dashboard_id}` doesn't exist anymore for some reason.")
+                    continue

databricks/labs/lakebridge/deployment/installation.py

@@ -0,0 +1,125 @@
+import logging
+from ast import literal_eval
+from pathlib import Path
+
+from databricks.labs.blueprint.installation import Installation
+from databricks.labs.blueprint.tui import Prompts
+from databricks.labs.blueprint.upgrades import Upgrades
+from databricks.labs.blueprint.wheels import ProductInfo, Version
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.errors import NotFound
+from databricks.sdk.mixins.compute import SemVer
+from databricks.sdk.errors.platform import InvalidParameterValue, ResourceDoesNotExist
+
+from databricks.labs.lakebridge.config import RemorphConfigs
+from databricks.labs.lakebridge.deployment.recon import ReconDeployment
+
+logger = logging.getLogger("databricks.labs.lakebridge.install")
+
+
+class WorkspaceInstallation:
+    def __init__(
+        self,
+        ws: WorkspaceClient,
+        prompts: Prompts,
+        installation: Installation,
+        recon_deployment: ReconDeployment,
+        product_info: ProductInfo,
+        upgrades: Upgrades,
+    ):
+        self._ws = ws
+        self._prompts = prompts
+        self._installation = installation
+        self._recon_deployment = recon_deployment
+        self._product_info = product_info
+        self._upgrades = upgrades
+
+    def _get_local_version_file_path(self):
+        user_home = f"{Path(__file__).home()}"
+        return Path(f"{user_home}/.databricks/labs/{self._product_info.product_name()}/state/version.json")
+
+    def _get_local_version_file(self, file_path: Path):
+        data = None
+        with file_path.open("r") as f:
+            data = literal_eval(f.read())
+        assert data, "Unable to read local version file."
+        local_installed_version = data["version"]
+        try:
+            SemVer.parse(local_installed_version)
+        except ValueError:
+            logger.warning(f"{local_installed_version} is not a valid version.")
+            local_installed_version = "v0.3.0"
+        local_installed_date = data["date"]
+        logger.debug(f"Found local installation version: {local_installed_version} {local_installed_date}")
+        return Version(
+            version=local_installed_version,
+            date=local_installed_date,
+            wheel=f"databricks_labs_remorph-{local_installed_version}-py3-none-any.whl",
+        )
+
+    def _get_ws_version(self):
+        try:
+            return self._installation.load(Version)
+        except ResourceDoesNotExist as err:
+            logger.warning(f"Unable to get Workspace Version due to: {err}")
+            return None
+
+    def _apply_upgrades(self):
+        """
+        * If remote version doesn't exist and local version exists:
+           Upload Version file to workspace to handle previous installations.
+        * If remote version or local_version exists, then only apply upgrades.
+        * No need to apply upgrades for fresh installation.
+        """
+        ws_version = self._get_ws_version()
+        local_version_path = self._get_local_version_file_path()
+        local_version = local_version_path.exists()
+        if not ws_version and local_version:
+            self._installation.save(self._get_local_version_file(local_version_path))
+
+        if ws_version or local_version:
+            try:
+                self._upgrades.apply(self._ws)
+                logger.debug("Upgrades applied successfully.")
+            except (InvalidParameterValue, NotFound) as err:
+                logger.warning(f"Unable to apply Upgrades due to: {err}")
+
+    def _upload_wheel(self):
+        wheels = self._product_info.wheels(self._ws)
+        with wheels:
+            wheel_paths = [wheels.upload_to_wsfs()]
+            wheel_paths = [f"/Workspace{wheel}" for wheel in wheel_paths]
+            return wheel_paths
+
+    def install(self, config: RemorphConfigs):
+        self._apply_upgrades()
+        wheel_paths: list[str] = self._upload_wheel()
+        if config.reconcile:
+            logger.info("Installing Remorph reconcile Metadata components.")
+            self._recon_deployment.install(config.reconcile, wheel_paths)
+
+    def uninstall(self, config: RemorphConfigs):
+        # This will remove all the Remorph modules
+        if not self._prompts.confirm(
+            "Do you want to uninstall Remorph from the workspace too, this would "
+            "remove Remorph project folder, jobs, metadata and dashboards"
+        ):
+            return
+        logger.info(f"Uninstalling Remorph from {self._ws.config.host}.")
+        try:
+            self._installation.files()
+        except NotFound:
+            logger.error(f"Check if {self._installation.install_folder()} is present. Aborting uninstallation.")
+            return
+
+        if config.transpile:
+            logging.info(
+                f"Won't remove transpile validation schema `{config.transpile.schema_name}` "
+                f"from catalog `{config.transpile.catalog_name}`. Please remove it manually."
+            )
+
+        if config.reconcile:
+            self._recon_deployment.uninstall(config.reconcile)
+
+        self._installation.remove()
+        logger.info("Uninstallation completed successfully.")

databricks/labs/lakebridge/deployment/job.py

@@ -0,0 +1,147 @@
+import dataclasses
+import logging
+from datetime import datetime, timezone, timedelta
+from typing import Any
+
+from databricks.labs.blueprint.installation import Installation
+from databricks.labs.blueprint.installer import InstallState
+from databricks.labs.blueprint.wheels import ProductInfo
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.errors import InvalidParameterValue
+from databricks.sdk.service import compute
+from databricks.sdk.service.jobs import Task, PythonWheelTask, JobCluster, JobSettings, JobParameterDefinition
+
+from databricks.labs.lakebridge.config import ReconcileConfig
+from databricks.labs.lakebridge.reconcile.constants import ReconSourceType
+
+logger = logging.getLogger(__name__)
+
+_TEST_JOBS_PURGE_TIMEOUT = timedelta(hours=1, minutes=15)
+
+
+class JobDeployment:
+    def __init__(
+        self,
+        ws: WorkspaceClient,
+        installation: Installation,
+        install_state: InstallState,
+        product_info: ProductInfo,
+    ):
+        self._ws = ws
+        self._installation = installation
+        self._install_state = install_state
+        self._product_info = product_info
+
+    def deploy_recon_job(self, name, recon_config: ReconcileConfig, remorph_wheel_path: str):
+        logger.info("Deploying reconciliation job.")
+        job_id = self._update_or_create_recon_job(name, recon_config, remorph_wheel_path)
+        logger.info(f"Reconciliation job deployed with job_id={job_id}")
+        logger.info(f"Job URL: {self._ws.config.host}#job/{job_id}")
+        self._install_state.save()
+
+    def _update_or_create_recon_job(self, name, recon_config: ReconcileConfig, remorph_wheel_path: str) -> str:
+        description = "Run the reconciliation process"
+        task_key = "run_reconciliation"
+
+        job_settings = self._recon_job_settings(name, task_key, description, recon_config, remorph_wheel_path)
+        if name in self._install_state.jobs:
+            try:
+                job_id = int(self._install_state.jobs[name])
+                logger.info(f"Updating configuration for job `{name}`, job_id={job_id}")
+                self._ws.jobs.reset(job_id, JobSettings(**job_settings))
+                return str(job_id)
+            except InvalidParameterValue:
+                del self._install_state.jobs[name]
+                logger.warning(f"Job `{name}` does not exist anymore for some reason")
+                return self._update_or_create_recon_job(name, recon_config, remorph_wheel_path)
+
+        logger.info(f"Creating new job configuration for job `{name}`")
+        new_job = self._ws.jobs.create(**job_settings)
+        assert new_job.job_id is not None
+        self._install_state.jobs[name] = str(new_job.job_id)
+        return str(new_job.job_id)
+
+    def _recon_job_settings(
+        self,
+        job_name: str,
+        task_key: str,
+        description: str,
+        recon_config: ReconcileConfig,
+        remorph_wheel_path: str,
+    ) -> dict[str, Any]:
+        latest_lts_spark = self._ws.clusters.select_spark_version(latest=True, long_term_support=True)
+        version = self._product_info.version()
+        version = version if not self._ws.config.is_gcp else version.replace("+", "-")
+        tags = {"version": f"v{version}"}
+        if self._is_testing():
+            # Add RemoveAfter tag for test job cleanup
+            date_to_remove = self._get_test_purge_time()
+            tags.update({"RemoveAfter": date_to_remove})
+
+        return {
+            "name": self._name_with_prefix(job_name),
+            "tags": tags,
+            "job_clusters": [
+                JobCluster(
+                    job_cluster_key="Remorph_Reconciliation_Cluster",
+                    new_cluster=compute.ClusterSpec(
+                        data_security_mode=compute.DataSecurityMode.USER_ISOLATION,
+                        spark_conf={},
+                        node_type_id=self._get_default_node_type_id(),
+                        autoscale=compute.AutoScale(min_workers=2, max_workers=10),
+                        spark_version=latest_lts_spark,
+                    ),
+                )
+            ],
+            "tasks": [
+                self._job_recon_task(
+                    Task(
+                        task_key=task_key,
+                        description=description,
+                        job_cluster_key="Remorph_Reconciliation_Cluster",
+                    ),
+                    recon_config,
+                    remorph_wheel_path,
+                ),
+            ],
+            "max_concurrent_runs": 2,
+            "parameters": [JobParameterDefinition(name="operation_name", default="reconcile")],
+        }
+
+    def _job_recon_task(self, jobs_task: Task, recon_config: ReconcileConfig, remorph_wheel_path: str) -> Task:
+        libraries = [
+            compute.Library(whl=remorph_wheel_path),
+        ]
+        source = recon_config.data_source
+        if source == ReconSourceType.ORACLE.value:
+            # TODO: Automatically fetch a version list for `ojdbc8`
+            oracle_driver_version = "23.4.0.24.05"
+            libraries.append(
+                compute.Library(
+                    maven=compute.MavenLibrary(f"com.oracle.database.jdbc:ojdbc8:{oracle_driver_version}"),
+                ),
+            )
+
+        return dataclasses.replace(
+            jobs_task,
+            libraries=libraries,
+            python_wheel_task=PythonWheelTask(
+                package_name="databricks_labs_remorph",
+                entry_point="reconcile",
+                parameters=["{{job.parameters.[operation_name]}}"],
+            ),
+        )
+
+    def _is_testing(self):
+        return self._product_info.product_name() != "remorph"
+
+    @staticmethod
+    def _get_test_purge_time() -> str:
+        return (datetime.now(timezone.utc) + _TEST_JOBS_PURGE_TIMEOUT).strftime("%Y%m%d%H")
+
+    def _get_default_node_type_id(self) -> str:
+        return self._ws.clusters.select_node_type(local_disk=True, min_memory_gb=16)
+
+    def _name_with_prefix(self, name: str) -> str:
+        prefix = self._installation.product()
+        return f"{prefix.upper()}_{name}".replace(" ", "_")