anyscale 0.26.64__py3-none-any.whl → 0.26.66__py3-none-any.whl

anyscale/commands/cloud_commands.py

@@ -1,4 +1,5 @@
 from io import StringIO
+import pathlib
 import re
 from typing import List, Optional
 
@@ -7,10 +8,21 @@ import yaml
 
 import anyscale
 from anyscale.cli_logger import BlockLogger
-from anyscale.client.openapi_client.models import ClusterManagementStackVersions
+from anyscale.client.openapi_client.models import (
+    AWSConfig,
+    CloudDeployment,
+    CloudProviders,
+    ClusterManagementStackVersions,
+    FileStorage,
+    GCPConfig,
+    KubernetesConfig,
+    NetworkingMode,
+    NFSMountTarget,
+    ObjectStorage,
+)
 from anyscale.client.openapi_client.models.compute_stack import ComputeStack
 from anyscale.cloud.models import CreateCloudCollaborator, CreateCloudCollaborators
-from anyscale.commands import cloud_commands_util, command_examples
+from anyscale.commands import command_examples
 from anyscale.commands.util import AnyscaleCommand, OptionPromptNull
 from anyscale.controllers.cloud_controller import CloudController
 from anyscale.util import (
@@ -231,12 +243,14 @@ def setup_cloud(  # noqa: PLR0913
 @click.option(
     "--max-items",
     required=False,
-    default=20,
+    default=None,
     type=int,
-    help="Max items to show in list.",
+    help="Maximum number of clouds to return. If not specified, all results are returned.",
     callback=validate_non_negative_arg,
 )
-def list_cloud(name: Optional[str], cloud_id: Optional[str], max_items: int,) -> None:
+def list_cloud(
+    name: Optional[str], cloud_id: Optional[str], max_items: Optional[int]
+) -> None:
     print(
         CloudController().list_clouds(
             cloud_name=name, cloud_id=cloud_id, max_items=max_items
@@ -893,6 +907,12 @@ def cloud_config_update(  # noqa: PLR0913
         "permissions after the cloud is created."
     ),
 )
+@click.option(
+    "--resource-file",
+    "-f",
+    help="Path to a YAML file defining a cloud resource. Schema: https://docs.anyscale.com/reference/cloud/#cloudresource.",
+    required=False,
+)
 def register_cloud(  # noqa: PLR0913, PLR0912, C901
     provider: str,
     region: str,
@@ -933,240 +953,323 @@ def register_cloud(  # noqa: PLR0913, PLR0912, C901
     yes: bool,
     skip_verifications: bool,
     enable_auto_add_user: bool,
+    resource_file: Optional[str],
 ) -> None:
-    missing_args: List[str] = []
-
-    # Validate K8S-only storage flags
-    if (
-        persistent_volume_claim or csi_ephemeral_volume_driver
-    ) and compute_stack != ComputeStack.K8S:
-        raise click.ClickException(
-            "--persistent-volume-claim and --csi-ephemeral-volume-driver are only supported with --compute-stack=k8s"
-        )
+    # Load CloudDeployment from the resource file if provided, otherwise build from CLI flags
+    if resource_file:
+        # Read the spec file.
+        path = pathlib.Path(resource_file)
+        if not path.exists():
+            raise click.ClickException(f"{resource_file} does not exist.")
+        if not path.is_file():
+            raise click.ClickException(f"{resource_file} is not a file.")
+
+        spec = yaml.safe_load(path.read_text())
+        try:
+            cloud_resource = CloudDeployment(**spec)
+
+            # Convert nested dict objects to model objects
+            if cloud_resource.file_storage:
+                cloud_resource.file_storage = FileStorage(**cloud_resource.file_storage)
+            if cloud_resource.object_storage:
+                cloud_resource.object_storage = ObjectStorage(
+                    **cloud_resource.object_storage
+                )
+            if cloud_resource.aws_config:
+                cloud_resource.aws_config = AWSConfig(**cloud_resource.aws_config)
+            if cloud_resource.gcp_config:
+                cloud_resource.gcp_config = GCPConfig(**cloud_resource.gcp_config)
+            if cloud_resource.kubernetes_config:
+                cloud_resource.kubernetes_config = KubernetesConfig(
+                    **cloud_resource.kubernetes_config
+                )
+
+        except Exception as e:  # noqa: BLE001
+            raise click.ClickException(f"Failed to parse cloud resource: {e}")
 
-    # Validate mutual exclusivity of storage configurations
-    storage_configs = []
-    if nfs_mount_target or nfs_mount_path:
-        storage_configs.append("NFS")
-    if persistent_volume_claim:
-        storage_configs.append("persistent volume claim")
-    if csi_ephemeral_volume_driver:
-        storage_configs.append("CSI ephemeral volume driver")
-
-    if len(storage_configs) > 1:
-        raise click.ClickException(
-            f"Storage configurations are mutually exclusive. Found: {', '.join(storage_configs)}. "
-            "Please specify only one of: --nfs-mount-target/--nfs-mount-path, --persistent-volume-claim, or --csi-ephemeral-volume-driver"
-        )
-
-    if provider == "aws":
-        if s3_bucket_id and not cloud_storage_bucket_name:
-            cloud_storage_bucket_name = s3_bucket_id
-        if efs_id and not file_storage_id:
-            file_storage_id = efs_id
-        # Check for missing required arguments for AWS clouds,
-        # based on the compute stack (not all args are required
-        # on all compute stacks).
-        required_resources = [
-            (vpc_id, "--vpc-id", (ComputeStack.VM)),
-            (subnet_ids, "--subnet-ids", (ComputeStack.VM)),
-            (anyscale_iam_role_id, "--anyscale-iam-role-id", (ComputeStack.VM),),
-            (instance_iam_role_id, "--instance-iam-role-id", (ComputeStack.VM)),
-            (security_group_ids, "--security-group-ids", (ComputeStack.VM)),
-            (
-                cloud_storage_bucket_name,
-                "--cloud-storage-bucket-name",
-                (ComputeStack.VM, ComputeStack.K8S),
-            ),
-            (kubernetes_zones, "--kubernetes-zones", (ComputeStack.K8S)),
-            (
-                anyscale_operator_iam_identity,
-                "--anyscale-operator-iam-identity",
-                (ComputeStack.K8S),
-            ),
-        ]
+    else:
+        missing_args: List[str] = []
 
-        if not allow_optional_file_storage():
-            required_resources.append(
-                (file_storage_id, "--file-storage-id", (ComputeStack.VM)),
+        # Validate K8S-only storage flags
+        if (
+            persistent_volume_claim or csi_ephemeral_volume_driver
+        ) and compute_stack != ComputeStack.K8S:
+            raise click.ClickException(
+                "--persistent-volume-claim and --csi-ephemeral-volume-driver are only supported with --compute-stack=k8s"
             )
 
-        for resource in required_resources:
-            if compute_stack in resource[2] and resource[0] is None:
-                missing_args.append(resource[1])
+        # Validate mutual exclusivity of storage configurations
+        storage_configs = []
+        if nfs_mount_target or nfs_mount_path:
+            storage_configs.append("NFS")
+        if persistent_volume_claim:
+            storage_configs.append("persistent volume claim")
+        if csi_ephemeral_volume_driver:
+            storage_configs.append("CSI ephemeral volume driver")
 
-        if len(missing_args) > 0:
-            raise click.ClickException(f"Please provide a value for {missing_args}")
-
-        CloudController().register_aws_cloud(
-            region=region,
-            compute_stack=compute_stack,
-            name=name,
-            vpc_id=vpc_id,
-            subnet_ids=subnet_ids.split(",") if subnet_ids else [],
-            efs_id=file_storage_id,
-            anyscale_iam_role_id=anyscale_iam_role_id,
-            instance_iam_role_id=instance_iam_role_id,
-            security_group_ids=security_group_ids.split(",")
-            if security_group_ids
-            else [],
-            cloud_storage_bucket_name=cloud_storage_bucket_name,
-            memorydb_cluster_id=memorydb_cluster_id,
-            functional_verify=functional_verify,
-            kubernetes_zones=kubernetes_zones.split(",") if kubernetes_zones else [],
-            anyscale_operator_iam_identity=anyscale_operator_iam_identity,
-            private_network=private_network,
-            cluster_management_stack_version=ClusterManagementStackVersions.V2,
-            yes=yes,
-            skip_verifications=skip_verifications,
-            auto_add_user=enable_auto_add_user,
-            external_id=external_id,
-            persistent_volume_claim=persistent_volume_claim,
-            csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
-        )
-    elif provider == "gcp":
-        if filestore_instance_id and not file_storage_id:
-            file_storage_id = filestore_instance_id
-        # Keep the parameter naming ({resource}_name or {resource}_id) consistent with GCP to reduce confusion for customers
-        # Check if all required parameters are provided
-        # memorystore_instance_name and host_project_id are optional for GCP clouds
-        required_resources = [
-            (project_id, "--project-id", (ComputeStack.VM)),
-            (vpc_name, "--vpc-name", (ComputeStack.VM)),
-            (subnet_names, "--subnet-names", (ComputeStack.VM)),
-            (
-                anyscale_service_account_email,
-                "--anyscale-service-account-email",
-                (ComputeStack.VM),
-            ),
-            (
-                instance_service_account_email,
-                "--instance-service-account-email",
-                (ComputeStack.VM),
-            ),
-            (provider_name, "--provider-name", (ComputeStack.VM)),
-            (firewall_policy_names, "--firewall-policy-names", (ComputeStack.VM)),
-            (
-                cloud_storage_bucket_name,
-                "--cloud-storage-bucket-name",
-                (ComputeStack.VM, ComputeStack.K8S),
-            ),
-            (kubernetes_zones, "--kubernetes-zones", (ComputeStack.K8S)),
-            (
-                anyscale_operator_iam_identity,
-                "--anyscale-operator-iam-identity",
-                (ComputeStack.K8S),
-            ),
-        ]
+        if len(storage_configs) > 1:
+            raise click.ClickException(
+                f"Storage configurations are mutually exclusive. Found: {', '.join(storage_configs)}. "
+                "Please specify only one of: --nfs-mount-target/--nfs-mount-path, --persistent-volume-claim, or --csi-ephemeral-volume-driver"
+            )
 
-        if not allow_optional_file_storage():
-            required_resources.extend(
-                [
+        if provider == "aws":
+            if s3_bucket_id and not cloud_storage_bucket_name:
+                cloud_storage_bucket_name = s3_bucket_id
+            if efs_id and not file_storage_id:
+                file_storage_id = efs_id
+            # Check for missing required arguments for AWS clouds,
+            # based on the compute stack (not all args are required
+            # on all compute stacks).
+            required_resources = [
+                (vpc_id, "--vpc-id", (ComputeStack.VM)),
+                (subnet_ids, "--subnet-ids", (ComputeStack.VM)),
+                (anyscale_iam_role_id, "--anyscale-iam_role-id", (ComputeStack.VM),),
+                (instance_iam_role_id, "--instance-iam-role-id", (ComputeStack.VM)),
+                (security_group_ids, "--security-group-ids", (ComputeStack.VM)),
+                (
+                    cloud_storage_bucket_name,
+                    "--cloud-storage-bucket-name",
+                    (ComputeStack.VM, ComputeStack.K8S),
+                ),
+                (kubernetes_zones, "--kubernetes-zones", (ComputeStack.K8S)),
+                (
+                    anyscale_operator_iam_identity,
+                    "--anyscale-operator-iam-identity",
+                    (ComputeStack.K8S),
+                ),
+            ]
+
+            if not allow_optional_file_storage():
+                required_resources.append(
                     (file_storage_id, "--file-storage-id", (ComputeStack.VM)),
-                    (filestore_location, "--filestore-location", (ComputeStack.VM)),
-                ]
+                )
+
+            for resource in required_resources:
+                if compute_stack in resource[2] and resource[0] is None:
+                    missing_args.append(resource[1])
+
+            if len(missing_args) > 0:
+                raise click.ClickException(f"Please provide a value for {missing_args}")
+
+            cloud_resource = CloudDeployment(
+                compute_stack=compute_stack,
+                provider=CloudProviders.AWS,
+                region=region,
+                networking_mode=NetworkingMode.PRIVATE
+                if private_network
+                else NetworkingMode.PUBLIC,
+                object_storage=ObjectStorage(bucket_name=cloud_storage_bucket_name),
+                file_storage=FileStorage(
+                    file_storage_id=file_storage_id,
+                    persistent_volume_claim=persistent_volume_claim,
+                    csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
+                )
+                if file_storage_id
+                or persistent_volume_claim
+                or csi_ephemeral_volume_driver
+                else None,
+                aws_config=AWSConfig(
+                    vpc_id=vpc_id,
+                    subnet_ids=subnet_ids.split(",") if subnet_ids else [],
+                    security_group_ids=security_group_ids.split(",")
+                    if security_group_ids
+                    else [],
+                    anyscale_iam_role_id=anyscale_iam_role_id,
+                    external_id=external_id,
+                    cluster_iam_role_id=instance_iam_role_id,
+                    memorydb_cluster_name=memorydb_cluster_id,
+                ),
+                kubernetes_config=KubernetesConfig(
+                    anyscale_operator_iam_identity=anyscale_operator_iam_identity,
+                    zones=kubernetes_zones.split(",") if kubernetes_zones else [],
+                )
+                if compute_stack == ComputeStack.K8S
+                else None,
             )
 
-        for resource in required_resources:
-            if compute_stack in resource[2] and resource[0] is None:
-                missing_args.append(resource[1])
-
-        if len(missing_args) > 0:
-            raise click.ClickException(f"Please provide a value for {missing_args}")
-
-        if project_id and project_id[0].isdigit():
-            # project ID should start with a letter
-            raise click.ClickException(
-                "Please provide a valid project ID. Note that project ID is not project number, see https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin for details."
+        elif provider == "gcp":
+            if filestore_instance_id and not file_storage_id:
+                file_storage_id = filestore_instance_id
+            # Keep the parameter naming ({resource}_name or {resource}_id) consistent with GCP to reduce confusion for customers
+            # Check if all required parameters are provided
+            # memorystore_instance_name and host_project_id are optional for GCP clouds
+            required_resources = [
+                (project_id, "--project-id", (ComputeStack.VM)),
+                (vpc_name, "--vpc-name", (ComputeStack.VM)),
+                (subnet_names, "--subnet-names", (ComputeStack.VM)),
+                (
+                    anyscale_service_account_email,
+                    "--anyscale-service-account-email",
+                    (ComputeStack.VM),
+                ),
+                (
+                    instance_service_account_email,
+                    "--instance-service-account-email",
+                    (ComputeStack.VM),
+                ),
+                (provider_name, "--provider-name", (ComputeStack.VM)),
+                (firewall_policy_names, "--firewall-policy-names", (ComputeStack.VM)),
+                (
+                    cloud_storage_bucket_name,
+                    "--cloud-storage-bucket-name",
+                    (ComputeStack.VM, ComputeStack.K8S),
+                ),
+                (kubernetes_zones, "--kubernetes-zones", (ComputeStack.K8S)),
+                (
+                    anyscale_operator_iam_identity,
+                    "--anyscale-operator-iam-identity",
+                    (ComputeStack.K8S),
+                ),
+            ]
+
+            if not allow_optional_file_storage():
+                required_resources.extend(
+                    [
+                        (file_storage_id, "--file-storage-id", (ComputeStack.VM)),
+                        (filestore_location, "--filestore-location", (ComputeStack.VM)),
+                    ]
+                )
+
+            for resource in required_resources:
+                if compute_stack in resource[2] and resource[0] is None:
+                    missing_args.append(resource[1])
+
+            if len(missing_args) > 0:
+                raise click.ClickException(f"Please provide a value for {missing_args}")
+
+            cloud_resource = CloudDeployment(
+                compute_stack=compute_stack,
+                provider=CloudProviders.GCP,
+                region=region,
+                networking_mode=NetworkingMode.PRIVATE
+                if private_network
+                else NetworkingMode.PUBLIC,
+                object_storage=ObjectStorage(bucket_name=cloud_storage_bucket_name),
+                file_storage=FileStorage(
+                    file_storage_id="projects/{}/locations/{}/instances/{}".format(
+                        project_id, filestore_location, file_storage_id
+                    )
+                    if file_storage_id
+                    else None,
+                    persistent_volume_claim=persistent_volume_claim,
+                    csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
+                )
+                if file_storage_id
+                or persistent_volume_claim
+                or csi_ephemeral_volume_driver
+                else None,
+                gcp_config=GCPConfig(
+                    project_id=project_id,
+                    host_project_id=host_project_id,
+                    provider_name=provider_name,
+                    vpc_name=vpc_name,
+                    subnet_names=subnet_names.split(",") if subnet_names else [],
+                    firewall_policy_names=firewall_policy_names.split(",")
+                    if firewall_policy_names
+                    else [],
+                    anyscale_service_account_email=anyscale_service_account_email,
+                    cluster_service_account_email=instance_service_account_email,
+                    memorystore_instance_name=memorystore_instance_name,
+                ),
+                kubernetes_config=KubernetesConfig(
+                    anyscale_operator_iam_identity=anyscale_operator_iam_identity,
+                    zones=kubernetes_zones.split(",") if kubernetes_zones else [],
+                )
+                if compute_stack == ComputeStack.K8S
+                else None,
             )
 
-        if (
-            compute_stack != ComputeStack.K8S
-            and re.search(
-                "projects/[0-9]*/locations/global/workloadIdentityPools/.+/providers/.+",
-                provider_name,
-            )
-            is None
-        ):
-            raise click.ClickException(
-                "Please provide a valid, fully qualified provider name. Example: projects/<project number>/locations/global/workloadIdentityPools/<pool name>/providers/<provider id>"
+        elif provider in ("azure", "generic"):
+            # For the 'generic' provider type, for the time being, most fields are optional; only 'name', 'provider', and 'compute-stack' are required.
+            if not name:
+                raise click.ClickException("Please provide a value for --name.")
+
+            if compute_stack != ComputeStack.K8S:
+                raise click.ClickException(
+                    "--compute-stack=k8s must be passed to register this Anyscale cloud."
+                )
+
+            # Handle parsing / conversion of nfs_mount_targets.
+            mount_targets: List[NFSMountTarget] = []
+            for target in nfs_mount_target or []:
+                parts = [part.strip() for part in target.split(",")]
+                if len(parts) == 1:
+                    mount_targets.append(NFSMountTarget(address=parts[0]))
+                elif len(parts) == 2:
+                    mount_targets.append(
+                        NFSMountTarget(address=parts[1], zone=parts[0])
+                    )
+                else:
+                    raise click.ClickException(
+                        f"Invalid mount target {target}; expected (zone,address) tuple or a singular address."
+                    )
+
+            cloud_provider = (
+                CloudProviders.AZURE if provider == "azure" else CloudProviders.GENERIC
             )
 
-        if (
-            memorystore_instance_name is not None
-            and not cloud_commands_util.validate_memorystore_instance_name(
-                memorystore_instance_name
-            )
-        ):
-            raise click.ClickException(
-                "Please provide a valid memorystore instance name. Example: projects/<project number>/locations/<location>/instances/<instance id>"
+            cloud_resource = CloudDeployment(
+                compute_stack=ComputeStack.K8S,
+                provider=cloud_provider,
+                region=region or "default",
+                object_storage=ObjectStorage(
+                    bucket_name=cloud_storage_bucket_name,
+                    region=cloud_storage_bucket_region or region,
+                    endpoint=cloud_storage_bucket_endpoint,
+                )
+                if cloud_storage_bucket_name
+                else None,
+                file_storage=FileStorage(
+                    mount_targets=mount_targets,
+                    mount_path=nfs_mount_path,
+                    persistent_volume_claim=persistent_volume_claim,
+                    csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
+                )
+                if mount_targets
+                or persistent_volume_claim
+                or csi_ephemeral_volume_driver
+                else None,
+                kubernetes_config=KubernetesConfig(
+                    zones=kubernetes_zones.split(",") if kubernetes_zones else [],
+                ),
             )
 
-        if host_project_id is not None and host_project_id[0].isdigit():
-            # project ID should start with a letter
+        else:
             raise click.ClickException(
-                "Please provide a valid project ID for `--host-project-id`. Note that project ID is not project number, see https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin for details."
+                f"Invalid Cloud provider: {provider}. Available providers are [aws, gcp, azure, generic]."
             )
 
+    if provider == "aws":
+        CloudController().register_aws_cloud(
+            name=name,
+            cloud_resource=cloud_resource,
+            functional_verify=functional_verify,
+            cluster_management_stack_version=ClusterManagementStackVersions.V2,
+            yes=yes,
+            skip_verifications=skip_verifications,
+            auto_add_user=enable_auto_add_user,
+        )
+    elif provider == "gcp":
         CloudController().register_gcp_cloud(
-            region=region,
             name=name,
-            compute_stack=compute_stack,
-            project_id=project_id,
-            vpc_name=vpc_name,
-            subnet_names=subnet_names.split(",") if subnet_names else [],
-            filestore_instance_id=file_storage_id,
-            filestore_location=filestore_location,
-            anyscale_service_account_email=anyscale_service_account_email,
-            instance_service_account_email=instance_service_account_email,
-            # TODO (allenyin): use provider_name instead of provider_id everywhere.
-            provider_id=provider_name,
-            firewall_policy_names=firewall_policy_names.split(",")
-            if firewall_policy_names
-            else [],
-            cloud_storage_bucket_name=cloud_storage_bucket_name,
-            memorystore_instance_name=memorystore_instance_name,
-            host_project_id=host_project_id,
-            kubernetes_zones=kubernetes_zones.split(",") if kubernetes_zones else [],
-            anyscale_operator_iam_identity=anyscale_operator_iam_identity,
+            cloud_resource=cloud_resource,
             functional_verify=functional_verify,
-            private_network=private_network,
             cluster_management_stack_version=ClusterManagementStackVersions.V2,
             yes=yes,
             skip_verifications=skip_verifications,
             auto_add_user=enable_auto_add_user,
-            persistent_volume_claim=persistent_volume_claim,
-            csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
         )
     elif provider in ("azure", "generic"):
-        # For the 'generic' provider type, for the time being, most fields are optional; only 'name', 'provider', and 'compute-stack' are required.
-        if not name:
-            raise click.ClickException("Please provide a value for --name.")
-
-        if compute_stack != ComputeStack.K8S:
-            raise click.ClickException(
-                "--compute-stack=k8s must be passed to register this Anyscale cloud."
-            )
-
         CloudController().register_azure_or_generic_cloud(
             name=name,
             provider=provider,
+            cloud_resource=cloud_resource,
             auto_add_user=enable_auto_add_user,
-            region=region,
-            cloud_storage_bucket_name=cloud_storage_bucket_name,
-            cloud_storage_bucket_endpoint=cloud_storage_bucket_endpoint,
-            cloud_storage_bucket_region=cloud_storage_bucket_region,
-            nfs_mount_targets=list(nfs_mount_target) if nfs_mount_target else [],
-            nfs_mount_path=nfs_mount_path,
-            persistent_volume_claim=persistent_volume_claim,
-            csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
-            kubernetes_zones=kubernetes_zones.split(",") if kubernetes_zones else [],
-            anyscale_operator_iam_identity=anyscale_operator_iam_identity,
         )
-
     else:
         raise click.ClickException(
-            f"Invalid Cloud provider: {provider}. Available providers are [aws, gcp]."
+            f"Invalid Cloud provider: {provider}. Available providers are [aws, gcp, azure, generic]."
         )
 
 
@@ -1325,9 +1428,10 @@ def cloud_edit(  # noqa: PLR0913
             )
         if (
             memorystore_instance_name is not None
-            and not cloud_commands_util.validate_memorystore_instance_name(
-                memorystore_instance_name
+            and re.search(
+                "projects/.+/locations/.+/instances/.+", memorystore_instance_name
             )
+            is None
         ):
             raise click.ClickException(
                 "Please provide a valid memorystore instance name. Example: projects/<project number>/locations/<location>/instances/<instance id>"