anyscale 0.26.50__py3-none-any.whl → 0.26.52__py3-none-any.whl

anyscale/controllers/cloud_controller.py

@@ -118,6 +118,7 @@ from anyscale.utils.cloud_utils import (
     modify_memorydb_parameter_group,
     validate_aws_credentials,
     verify_anyscale_access,
+    wait_for_aws_lb_resource_termination,
     wait_for_gcp_lb_resource_termination,
 )
 from anyscale.utils.imports.gcp import (
@@ -1424,18 +1425,6 @@ class CloudController(BaseController):
                 cloud_id, CloudProviders.AWS, functions_to_verify, yes,
             )
 
-    def get_cloud_deployment(
-        self, cloud_id: str, cloud_deployment_id: str
-    ) -> CloudDeployment:
-        try:
-            return self.api_client.get_cloud_deployment_api_v2_clouds_cloud_id_deployment_get(
-                cloud_id=cloud_id, cloud_deployment_id=cloud_deployment_id,
-            ).result
-        except Exception as e:  # noqa: BLE001
-            raise ClickException(
-                f"Failed to get cloud deployment {cloud_deployment_id} for cloud {cloud_id}. Error: {e}"
-            )
-
     # Avoid displaying fields with empty values (since the values for optional fields default to None).
     def _remove_empty_values(self, d):
         if isinstance(d, dict):
@@ -1476,29 +1465,6 @@ class CloudController(BaseController):
             ],
         }
 
-    def get_cloud_deployment_dict_by_name(
-        self, cloud_name: str, cloud_deployment_name: Optional[str]
-    ) -> Dict[str, Any]:
-        cloud_id, _ = get_cloud_id_and_name(self.api_client, cloud_name=cloud_name)
-
-        result = self.get_cloud_deployments(cloud_id)
-        deployments = result.get("deployments", [])
-        if len(deployments) == 0:
-            raise ClickException(f"Cloud {cloud_name} has no cloud deployments.")
-
-        if cloud_deployment_name is None:
-            if len(deployments) > 1:
-                self.log.warning(
-                    f"Cloud {cloud_name} has {len(deployments)} deployments, only the primary deployment will be returned."
-                )
-            return deployments[0]
-
-        for deployment in deployments:
-            if deployment.get("name") == cloud_deployment_name:
-                return deployment
-
-        raise ClickException(f"Cloud deployment {cloud_deployment_name} not found.")
-
     def update_aws_anyscale_iam_role(
         self,
         cloud_id: str,
@@ -1775,85 +1741,103 @@ class CloudController(BaseController):
             f"Successfully created cloud deployment{' ' + new_deployment.name if new_deployment.name else ''} in cloud {existing_spec['name']}!"
         )
 
-    def update_cloud_deployment(  # noqa: PLR0912
+    def update_cloud_deployments(  # noqa: PLR0912, C901
         self,
-        cloud: str,
-        spec_file: str,
+        cloud_name: Optional[str],
+        cloud_id: Optional[str],
+        resources_file: str,
         skip_verification: bool = False,
         yes: bool = False,
     ):
+        if not cloud_id:
+            cloud_id, _ = get_cloud_id_and_name(self.api_client, cloud_name=cloud_name)
+        assert cloud_id
+
         # Read the spec file.
-        path = pathlib.Path(spec_file)
+        path = pathlib.Path(resources_file)
         if not path.exists():
-            raise ClickException(f"{spec_file} does not exist.")
+            raise ClickException(f"{resources_file} does not exist.")
         if not path.is_file():
-            raise ClickException(f"{spec_file} is not a file.")
+            raise ClickException(f"{resources_file} is not a file.")
 
         spec = yaml.safe_load(path.read_text())
-        try:
-            updated_deployment = CloudDeployment(**spec)
-        except Exception as e:  # noqa: BLE001
-            raise ClickException(f"Failed to parse cloud deployment: {e}")
 
-        if not updated_deployment.cloud_deployment_id:
+        # Get the existing spec.
+        existing_spec = self.get_cloud_deployments(cloud_id=cloud_id)
+
+        if len(existing_spec["deployments"]) > len(spec):
             raise ClickException(
-                "The cloud deployment must include a cloud_deployment_id."
+                "Please use `anyscale cloud deployment delete` to remove cloud deployments."
             )
-
-        # Get the existing cloud deployment.
-        cloud_id, _ = get_cloud_id_and_name(self.api_client, cloud_name=cloud)
-        existing_deployment = self.get_cloud_deployment(
-            cloud_id=cloud_id,
-            cloud_deployment_id=updated_deployment.cloud_deployment_id,
-        )
-        if (
-            updated_deployment.provider == CloudProviders.PCP
-            or existing_deployment.provider == CloudProviders.PCP
-        ):
+        if len(existing_spec["deployments"]) < len(spec):
             raise ClickException(
-                "Please use the `anyscale machine-pool` CLI to update machine pools."
+                "Please use `anyscale cloud deployment create` to add cloud deployments."
             )
 
-        # Diff the existing and new cloud deployments.
-        diff = self._generate_diff(
-            self._remove_empty_values(existing_deployment.to_dict()),
-            self._remove_empty_values(updated_deployment.to_dict()),
-        )
+        # Diff the existing and new specs
+        diff = self._generate_diff(existing_spec["deployments"], spec)
         if not diff:
             self.log.info("No changes detected.")
             return
 
-        # Preprocess the deployment if necessary.
-        if updated_deployment.provider == CloudProviders.AWS:
-            self._preprocess_aws(cloud_id=cloud_id, deployment=updated_deployment)
-        elif updated_deployment.provider == CloudProviders.GCP:
-            self._preprocess_gcp(deployment=updated_deployment)
-        # Skip verification for Kubernetes stacks or if explicitly requested
-        if updated_deployment.compute_stack == ComputeStack.K8S:
-            self.log.info("Skipping verification for Kubernetes compute stack.")
-        elif not skip_verification and not self.verify_cloud_deployment(
-            cloud_id=cloud_id, cloud_deployment=updated_deployment
-        ):
-            raise ClickException(
-                f"Verification failed for cloud deployment {updated_deployment.name}."
-            )
+        existing_deployments = {
+            deployment["cloud_deployment_id"]: CloudDeployment(**deployment)
+            for deployment in existing_spec["deployments"]
+        }
+
+        updated_deployments: List[CloudDeployment] = []
+        for d in spec:
+            try:
+                deployment = CloudDeployment(**d)
+            except Exception as e:  # noqa: BLE001
+                raise ClickException(f"Failed to parse deployment: {e}")
+
+            if not deployment.cloud_deployment_id:
+                raise ClickException(
+                    "All cloud deployments must include a cloud_deployment_id."
+                )
+            if deployment.cloud_deployment_id not in existing_deployments:
+                raise ClickException(
+                    f"Cloud deployment {deployment.cloud_deployment_id} not found."
+                )
+            if deployment.provider == CloudProviders.PCP:
+                raise ClickException(
+                    "Please use the `anyscale machine-pool` CLI to update machine pools."
+                )
+            if deployment != existing_deployments[deployment.cloud_deployment_id]:
+                updated_deployments.append(deployment)
 
         # Log the diff and confirm.
         self.log.info(f"Detected the following changes:\n{diff}")
 
-        confirm("Would you like to proceed with updating this cloud deployment?", yes)
+        confirm("Would you like to proceed with updating this cloud?", yes)
+
+        # Preprocess the deployments if necessary.
+        for deployment in updated_deployments:
+            if deployment.provider == CloudProviders.AWS:
+                self._preprocess_aws(cloud_id=cloud_id, deployment=deployment)
+            elif deployment.provider == CloudProviders.GCP:
+                self._preprocess_gcp(deployment=deployment)
+
+            # Skip verification for Kubernetes stacks or if explicitly requested
+            if deployment.compute_stack == ComputeStack.K8S:
+                self.log.info("Skipping verification for Kubernetes compute stack.")
+            elif not skip_verification and not self.verify_cloud_deployment(
+                cloud_id=cloud_id, cloud_deployment=deployment
+            ):
+                raise ClickException(
+                    f"Verification failed for cloud deployment {deployment.name or deployment.cloud_deployment_id}."
+                )
 
-        # Update the deployment.
+        # Update the deployments.
         try:
-            self.api_client.update_cloud_deployment_api_v2_clouds_cloud_id_update_deployment_put(
-                cloud_id=cloud_id, cloud_deployment=updated_deployment,
+            self.api_client.update_cloud_deployments_api_v2_clouds_cloud_id_deployments_put(
+                cloud_id=cloud_id, cloud_deployment=updated_deployments,
             )
         except Exception as e:  # noqa: BLE001
-            raise ClickException(f"Failed to update cloud deployment: {e}")
+            raise ClickException(f"Failed to update cloud deployments: {e}")
 
-        self.log.info(
-            f"Successfully updated cloud deployment {updated_deployment.name or updated_deployment.cloud_deployment_id} in cloud {cloud}."
-        )
+        self.log.info(f"Successfully updated cloud {cloud_name or cloud_id}.")
 
     def remove_cloud_deployment(
         self, cloud_name: str, deployment_name: str, yes: bool,
@@ -2516,6 +2500,8 @@ class CloudController(BaseController):
         cloud_storage_bucket_region: Optional[str] = None,
         nfs_mount_targets: Optional[List[str]] = None,
         nfs_mount_path: Optional[str] = None,
+        persistent_volume_claim: Optional[str] = None,
+        csi_ephemeral_volume_driver: Optional[str] = None,
         kubernetes_zones: Optional[List[str]] = None,
         anyscale_operator_iam_identity: Optional[str] = None,
     ) -> None:
@@ -2587,6 +2573,8 @@ class CloudController(BaseController):
                             or region,
                             nfs_mount_targets=mount_targets,
                             nfs_mount_path=nfs_mount_path,
+                            persistent_volume_claim=persistent_volume_claim,
+                            csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
                         ),
                     ),
                 )
@@ -2660,6 +2648,8 @@ class CloudController(BaseController):
         compute_stack: ComputeStack = ComputeStack.VM,
         kubernetes_zones: Optional[List[str]] = None,
         anyscale_operator_iam_identity: Optional[str] = None,
+        persistent_volume_claim: Optional[str] = None,
+        csi_ephemeral_volume_driver: Optional[str] = None,
     ):
         functions_to_verify = self._validate_functional_verification_args(
             functional_verify
@@ -2823,6 +2813,8 @@ class CloudController(BaseController):
                 kubernetes_zones=kubernetes_zones,
                 kubernetes_dataplane_identity=anyscale_operator_iam_identity,
                 cloud_storage_bucket_name=cloud_storage_bucket_name,
+                persistent_volume_claim=persistent_volume_claim,
+                csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
             )
 
             # Verification is only performed for VM compute stack.
@@ -3186,6 +3178,8 @@ class CloudController(BaseController):
         compute_stack: ComputeStack = ComputeStack.VM,
         kubernetes_zones: Optional[List[str]] = None,
         anyscale_operator_iam_identity: Optional[str] = None,
+        persistent_volume_claim: Optional[str] = None,
+        csi_ephemeral_volume_driver: Optional[str] = None,
     ):
         functions_to_verify = self._validate_functional_verification_args(
             functional_verify
@@ -3335,6 +3329,8 @@ class CloudController(BaseController):
                 kubernetes_zones=kubernetes_zones,
                 kubernetes_dataplane_identity=anyscale_operator_iam_identity,
                 cloud_storage_bucket_name=cloud_storage_bucket_name,
+                persistent_volume_claim=persistent_volume_claim,
+                csi_ephemeral_volume_driver=csi_ephemeral_volume_driver,
             )
 
             # Verification is only performed for VM compute stack.
@@ -3525,6 +3521,15 @@ class CloudController(BaseController):
         # Clean up cloud resources
         try:
             if cloud_provider == CloudProviders.AWS:
+                if not (cloud.is_aioa or cloud.compute_stack == ComputeStack.K8S):
+                    # Delete services resources
+                    self.delete_aws_lb_cfn_stack(cloud=cloud)
+                    with self.log.spinner("Deleting load balancing resources..."):
+                        wait_for_aws_lb_resource_termination(
+                            api_client=self.api_client, cloud_id=cloud_id
+                        )
+                    self.delete_aws_tls_certificates(cloud=cloud)
+
                 self.delete_all_aws_resources(cloud)
             elif cloud_provider == CloudProviders.GCP:
                 with self.log.spinner("Deleting load balancing resources..."):
@@ -3585,10 +3590,6 @@ class CloudController(BaseController):
             # No resources to delete for hosted and k8s clouds
             return True
 
-        # Delete services resources
-        self.delete_aws_lb_cfn_stack(cloud=cloud)
-        self.delete_aws_tls_certificates(cloud=cloud)
-
         # Clean up cloud resources
         if cloud.is_bring_your_own_resource is False:  # managed cloud
             # Delete AWS cloud resources by deleting the cfn stack
@@ -3601,7 +3602,6 @@ class CloudController(BaseController):
         return True
 
     def delete_aws_lb_cfn_stack(self, cloud: CloudWithCloudResource) -> bool:
-
         tag_name = "anyscale-cloud-id"
         key_name = cloud.id
 

anyscale/job/_private/job_sdk.py

@@ -82,19 +82,34 @@ class PrivateJobSDK(WorkloadSDK):
             - 'working_dir' will be set to '.'.
             - 'pip' will be set to the workspace-managed requirements file.
         """
-        runtime_env: Dict[str, Any] = {}
-        [runtime_env] = self.override_and_upload_local_dirs(
-            [runtime_env],
-            working_dir_override=config.working_dir,
-            excludes_override=config.excludes,
-            cloud_id=cloud_id,
-            autopopulate_in_workspace=autopopulate_in_workspace,
-            additional_py_modules=config.py_modules,
-            py_executable_override=config.py_executable,
-            has_multiple_cloud_deployments=self._compute_config_has_multiple_cloud_deployments(
-                compute_config=config.compute_config, cloud=config.cloud
-            ),
+        cloud_deployments = self._get_compute_config_cloud_deployments(
+            compute_config=config.compute_config, cloud=config.cloud
         )
+        assert len(cloud_deployments) > 0
+
+        runtime_env: Dict[str, Any] = {}
+        if len(cloud_deployments) == 1:
+            [runtime_env] = self.override_and_upload_local_dirs_single_deployment(
+                [runtime_env],
+                working_dir_override=config.working_dir,
+                excludes_override=config.excludes,
+                cloud_id=cloud_id,
+                autopopulate_in_workspace=autopopulate_in_workspace,
+                additional_py_modules=config.py_modules,
+                py_executable_override=config.py_executable,
+                cloud_deployment=cloud_deployments[0],
+            )
+        else:
+            [runtime_env] = self.override_and_upload_local_dirs_multi_deployment(
+                [runtime_env],
+                working_dir_override=config.working_dir,
+                excludes_override=config.excludes,
+                cloud_id=cloud_id,
+                autopopulate_in_workspace=autopopulate_in_workspace,
+                additional_py_modules=config.py_modules,
+                py_executable_override=config.py_executable,
+                cloud_deployments=cloud_deployments,
+            )
         [runtime_env] = self.override_and_load_requirements_files(
             [runtime_env],
             requirements_override=config.requirements,
@@ -106,15 +121,15 @@ class PrivateJobSDK(WorkloadSDK):
 
         return runtime_env or None
 
-    def _compute_config_has_multiple_cloud_deployments(
+    def _get_compute_config_cloud_deployments(
         self, compute_config: Union[ComputeConfigType, str, None], cloud: Optional[str]
-    ) -> bool:
+    ) -> List[Optional[str]]:
         if isinstance(compute_config, ComputeConfig):
             # single-deployment compute config
-            return False
+            return [compute_config.cloud_deployment]
 
         if isinstance(compute_config, MultiDeploymentComputeConfig):
-            return len(compute_config.configs) > 1
+            return [config.cloud_deployment for config in compute_config.configs]
 
         compute_config_id = self._resolve_compute_config_id(
             compute_config=compute_config, cloud=cloud
@@ -125,10 +140,13 @@ class PrivateJobSDK(WorkloadSDK):
                 f"The compute config '{compute_config_id}' does not exist."
             )
 
-        return (
-            compute_template.config.deployment_configs
-            and len(compute_template.config.deployment_configs) > 1
-        )
+        if compute_template.config.deployment_configs is None:
+            return [None]
+
+        return [
+            config.cloud_deployment
+            for config in compute_template.config.deployment_configs
+        ]
 
     def get_default_name(self) -> str:
         """Get a default name for the job.

anyscale/project/__init__.py

@@ -1,6 +1,7 @@
 from typing import List, Optional
 
 from anyscale._private.anyscale_client import AnyscaleClientInterface
+from anyscale._private.models.model_base import ResultIterator
 from anyscale._private.sdk import sdk_docs
 from anyscale._private.sdk.base_sdk import Timer
 from anyscale.cli_logger import BlockLogger
@@ -8,9 +9,30 @@ from anyscale.project._private.project_sdk import PrivateProjectSDK
 from anyscale.project.commands import (
     _ADD_COLLABORATORS_DOCSTRINGS,
     _ADD_COLLABORATORS_EXAMPLE,
+    _CREATE_PROJECT_DOCSTRINGS,
+    _CREATE_PROJECT_EXAMPLE,
+    _DELETE_PROJECT_DOCSTRINGS,
+    _DELETE_PROJECT_EXAMPLE,
+    _GET_DEFAULT_PROJECT_DOCSTRINGS,
+    _GET_DEFAULT_PROJECT_EXAMPLE,
+    _GET_PROJECT_DOCSTRINGS,
+    _GET_PROJECT_EXAMPLE,
+    _LIST_PROJECTS_DOCSTRINGS,
+    _LIST_PROJECTS_EXAMPLE,
     add_collaborators,
+    create,
+    delete,
+    get,
+    get_default,
+    list,
+)
+from anyscale.project.models import (
+    CreateProjectCollaborator,
+    Project,
+    ProjectPermissionLevel,
+    ProjectSortField,
+    ProjectSortOrder,
 )
-from anyscale.project.models import CreateProjectCollaborator
 
 
 class ProjectSDK:
@@ -33,3 +55,81 @@ class ProjectSDK:
         """Batch add collaborators to a project.
         """
         self._private_sdk.add_collaborators(cloud, project, collaborators)
+
+    @sdk_docs(
+        doc_py_example=_GET_PROJECT_EXAMPLE, arg_docstrings=_GET_PROJECT_DOCSTRINGS,
+    )
+    def get(self, project_id: str) -> Project:  # noqa: F811
+        """Get details of a project."""
+        return self._private_sdk.get(project_id)
+
+    @sdk_docs(
+        doc_py_example=_LIST_PROJECTS_EXAMPLE, arg_docstrings=_LIST_PROJECTS_DOCSTRINGS,
+    )
+    def list(  # noqa: F811
+        self,
+        *,
+        name_contains: Optional[str] = None,
+        creator_id: Optional[str] = None,
+        parent_cloud_id: Optional[str] = None,
+        include_defaults: bool = True,
+        max_items: Optional[int] = None,
+        page_size: Optional[int] = None,
+        sort_field: Optional[ProjectSortField] = None,
+        sort_order: Optional[ProjectSortOrder] = None,
+    ) -> ResultIterator[Project]:
+        """List all projects with optional filters.
+
+        Returns
+        -------
+        ResultIterator[Project]
+            An iterator yielding Project objects. Fetches pages
+            lazily as needed. Use a `for` loop to iterate or `list()`
+            to consume all at once.
+        """
+        return self._private_sdk.list(
+            name_contains=name_contains,
+            creator_id=creator_id,
+            parent_cloud_id=parent_cloud_id,
+            include_defaults=include_defaults,
+            max_items=max_items,
+            page_size=page_size,
+            sort_field=sort_field,
+            sort_order=sort_order,
+        )
+
+    @sdk_docs(
+        doc_py_example=_CREATE_PROJECT_EXAMPLE,
+        arg_docstrings=_CREATE_PROJECT_DOCSTRINGS,
+    )
+    def create(  # noqa: F811
+        self,
+        name: str,
+        parent_cloud_id: str,
+        *,
+        description: Optional[str] = None,
+        initial_cluster_config: Optional[str] = None,
+    ) -> str:
+        """Create a project."""
+        return self._private_sdk.create(
+            name,
+            description or "",
+            parent_cloud_id,
+            initial_cluster_config=initial_cluster_config,
+        )
+
+    @sdk_docs(
+        doc_py_example=_DELETE_PROJECT_EXAMPLE,
+        arg_docstrings=_DELETE_PROJECT_DOCSTRINGS,
+    )
+    def delete(self, project_id: str):  # noqa: F811
+        """Delete a project."""
+        self._private_sdk.delete(project_id)
+
+    @sdk_docs(
+        doc_py_example=_GET_DEFAULT_PROJECT_EXAMPLE,
+        arg_docstrings=_GET_DEFAULT_PROJECT_DOCSTRINGS,
+    )
+    def get_default(self, parent_cloud_id: str) -> Project:  # noqa: F811
+        """Get the default project for a cloud."""
+        return self._private_sdk.get_default(parent_cloud_id)

anyscale/project/_private/project_sdk.py

@@ -1,11 +1,23 @@
-from typing import List
+from typing import List, Optional
 
+from anyscale._private.models.model_base import ResultIterator
 from anyscale._private.sdk.base_sdk import BaseSDK
 from anyscale.client.openapi_client import (
     CreateUserProjectCollaborator,
     CreateUserProjectCollaboratorValue,
+    Project as OpenAPIProject,
+    ProjectListResponse,
+    WriteProject,
 )
-from anyscale.project.models import CreateProjectCollaborator
+from anyscale.project.models import (
+    CreateProjectCollaborator,
+    Project,
+    ProjectSortField,
+    ProjectSortOrder,
+)
+
+
+MAX_PAGE_SIZE = 50
 
 
 class PrivateProjectSDK(BaseSDK):
@@ -25,3 +37,79 @@ class PrivateProjectSDK(BaseSDK):
                 for collaborator in collaborators
             ],
         )
+
+    def get(self, project_id: str) -> Optional[Project]:
+        project: OpenAPIProject = self.client.get_project(project_id)
+        return Project.from_dict(project.to_dict()) if project else None
+
+    def list(
+        self,
+        *,
+        # filters
+        name_contains: Optional[str] = None,
+        creator_id: Optional[str] = None,
+        parent_cloud_id: Optional[str] = None,
+        include_defaults: bool = True,
+        # pagination
+        max_items: Optional[int] = None,
+        page_size: Optional[int] = None,
+        # sorting
+        sort_field: Optional[ProjectSortField] = None,
+        sort_order: Optional[ProjectSortOrder] = None,
+    ) -> ResultIterator[Project]:
+        if max_items is not None and (max_items <= 0):
+            raise ValueError("'max_items' must be greater than 0.")
+
+        if page_size is not None and (page_size <= 0 or page_size > MAX_PAGE_SIZE):
+            raise ValueError(
+                f"'page_size' must be between 1 and {MAX_PAGE_SIZE}, inclusive."
+            )
+
+        def _fetch_page(token: Optional[str],) -> ProjectListResponse:
+            return self.client.list_projects(
+                name_contains=name_contains,
+                creator_id=creator_id,
+                parent_cloud_id=parent_cloud_id,
+                include_defaults=include_defaults,
+                sort_field=str(sort_field) if sort_field else None,
+                sort_order=str(sort_order) if sort_order else None,
+                paging_token=token,
+                count=page_size,
+            )
+
+        def _openapi_project_to_sdk_project(openapi_project: OpenAPIProject) -> Project:
+            return Project.from_dict(openapi_project.to_dict())
+
+        return ResultIterator(
+            page_token=None,
+            max_items=max_items,
+            fetch_page=_fetch_page,
+            parse_fn=_openapi_project_to_sdk_project,
+        )
+
+    def create(
+        self,
+        name: str,
+        description: str,
+        parent_cloud_id: str,
+        *,
+        initial_cluster_config: Optional[str] = None,
+    ) -> str:
+        project = self.client.create_project(
+            project=WriteProject(
+                name=name,
+                description=description,
+                parent_cloud_id=parent_cloud_id,
+                initial_cluster_config=initial_cluster_config,
+            )
+        )
+        return project.id  # type: ignore
+
+    def delete(
+        self, project_id: str,
+    ):
+        self.client.delete_project(project_id)
+
+    def get_default(self, parent_cloud_id: str,) -> Project:
+        project: OpenAPIProject = self.client.get_default_project(parent_cloud_id)
+        return Project.from_dict(project.to_dict())