anyscale 0.26.66__py3-none-any.whl → 0.26.68__py3-none-any.whl

anyscale/controllers/cloud_controller.py

@@ -4193,48 +4193,49 @@ class CloudController(BaseController):
         """
         command_parts = [
             "helm upgrade <release-name> anyscale/anyscale-operator",
-            f"  --set-string cloudDeploymentId={cloud_deployment_id}",
-            f"  --set-string cloudProvider={provider}",
+            f"  --set-string global.cloudDeploymentId={cloud_deployment_id}",
+            f"  --set-string global.cloudProvider={provider}",
         ]
 
         # Add region for most providers (not for generic)
         if region and provider != "generic":
-            command_parts.append(f"  --set-string region={region}")
+            command_parts.append(f"  --set-string global.region={region}")
 
         # Add provider-specific parameters
         if provider == "gcp" and operator_iam_identity:
             command_parts.append(
-                f"  --set-string operatorIamIdentity={operator_iam_identity}"
+                f"  --set-string global.auth.iamIdentity={operator_iam_identity}"
             )
         elif provider == "azure":
             if operator_iam_identity:
                 command_parts.append(
-                    f"  --set-string operatorIamIdentity={operator_iam_identity}"
+                    f"  --set-string global.auth.iamIdentity={operator_iam_identity}"
                 )
             if anyscale_cli_token:
                 command_parts.append(
-                    f"  --set-string anyscaleCliToken={anyscale_cli_token}"
+                    f"  --set-string global.auth.anyscaleCliToken={anyscale_cli_token}"
                 )
             else:
                 command_parts.append(
-                    "  --set-string anyscaleCliToken=$ANYSCALE_CLI_TOKEN"
+                    "  --set-string global.auth.anyscaleCliToken=$ANYSCALE_CLI_TOKEN"
                 )
         elif provider == "generic":
             if anyscale_cli_token:
                 command_parts.append(
-                    f"  --set-string anyscaleCliToken={anyscale_cli_token}"
+                    f"  --set-string global.auth.anyscaleCliToken={anyscale_cli_token}"
                 )
             else:
                 command_parts.append(
-                    "  --set-string anyscaleCliToken=$ANYSCALE_CLI_TOKEN"
+                    "  --set-string global.auth.anyscaleCliToken=$ANYSCALE_CLI_TOKEN"
                 )
 
         # Add common parameters
         command_parts.extend(
             [
-                "  --set-string workloadServiceAccountName=anyscale-operator",
+                "  --set-string workloads.serviceAccount.name=anyscale-operator",
                 "  --namespace <namespace>",
                 "  --create-namespace",
+                "  --wait",
                 "  -i",
             ]
         )

anyscale/controllers/kubernetes_verifier.py

@@ -789,20 +789,59 @@ class OperatorVerifier:
 
     def _fetch_config_data(self, local_port: int) -> OperatorConfigData:
         """Fetch config data from operator."""
-        response = requests.get(
-            f"http://localhost:{local_port}{OPERATOR_CONFIG_ENDPOINT}",
-            timeout=HTTP_REQUEST_TIMEOUT,
-        )
+        max_retries = 6
+        retry_delay = 5
 
-        config_data = None
-        config_error = None
+        for attempt in range(max_retries):
+            response = requests.get(
+                f"http://localhost:{local_port}{OPERATOR_CONFIG_ENDPOINT}",
+                timeout=HTTP_REQUEST_TIMEOUT,
+            )
 
-        if response.status_code == 200:
-            try:
-                config_data = response.json()
-            except json.JSONDecodeError as e:
-                config_error = str(e)
+            config_data = None
+            config_error = None
+
+            if response.status_code == 200:
+                try:
+                    config_data = response.json()
+
+                    # Check if iamIdentity is present in the config
+                    # This is necessary because the operator may not have the iamIdentity field
+                    # immediately available after startup while it is being bootstrapped (connecting to KCP, storing registry secrets, etc.)
+                    if config_data and "iamIdentity" in config_data:
+                        return OperatorConfigData(
+                            status_code=response.status_code,
+                            response_text=response.text,
+                            config_data=config_data,
+                            config_error=config_error,
+                        )
 
+                    # If iamIdentity is missing and we have retries left, wait and retry
+                    if attempt < max_retries - 1:
+                        self.log.info(
+                            f"Operator config endpoint returned 200 but iamIdentity not found. "
+                            f"Retrying in {retry_delay} seconds... (attempt {attempt + 1}/{max_retries})"
+                        )
+                        time.sleep(retry_delay)
+                        continue
+                    else:
+                        # Last attempt and still no iamIdentity, return what we have
+                        self.log.warning(
+                            f"iamIdentity not found after {max_retries} attempts"
+                        )
+
+                except json.JSONDecodeError as e:
+                    config_error = str(e)
+            else:
+                # Non-200 status code, return immediately without retrying
+                return OperatorConfigData(
+                    status_code=response.status_code,
+                    response_text=response.text,
+                    config_data=config_data,
+                    config_error=config_error,
+                )
+
+        # Return the last response (this will have config_data but no iamIdentity)
         return OperatorConfigData(
             status_code=response.status_code,
             response_text=response.text,
@@ -1375,6 +1414,13 @@ class KubernetesCloudDeploymentVerifier:
         ):
             cloud_deployment.file_storage = FileStorage(**cloud_deployment.file_storage)
 
+        if cloud_deployment.kubernetes_config is not None and isinstance(
+            cloud_deployment.kubernetes_config, dict
+        ):
+            cloud_deployment.kubernetes_config = OpenAPIKubernetesConfig(
+                **cloud_deployment.kubernetes_config
+            )
+
         try:
             return self._run_verification_steps(cloud_deployment)
 
@@ -1519,6 +1565,14 @@ class KubernetesCloudDeploymentVerifier:
 
     def _get_kubectl_config(self):
         """Get kubectl context and operator namespace from user"""
+        # If k8s_config is already set, skip prompting
+        if self.k8s_config is not None:
+            self.log.info(
+                f"Using configured context='{self.k8s_config.context}', "
+                f"namespace='{self.k8s_config.operator_namespace}'"
+            )
+            return
+
         # Check if kubectl is available
         temp_kubectl = KubectlOperations("", self.log)
         if not temp_kubectl.check_kubectl_available():

anyscale/utils/cloudformation_utils.py

@@ -0,0 +1,364 @@
+"""
+CloudFormation utility functions for creating and managing stacks.
+
+This module provides reusable utilities for CloudFormation stack operations
+including creation, waiting, and output retrieval.
+"""
+
+import time
+from typing import Any, Dict, Optional
+
+import boto3
+from botocore.exceptions import ClientError
+from click import ClickException
+
+from anyscale.cli_logger import BlockLogger
+
+
+class CloudFormationUtils:
+    """Utility class for CloudFormation operations."""
+
+    def __init__(self, logger: Optional[BlockLogger] = None):
+        self.log = logger or BlockLogger()
+
+    def create_and_wait_for_stack(  # noqa: PLR0913
+        self,
+        stack_name: str,
+        template_body: str,
+        parameters: list,
+        region: str,
+        capabilities: Optional[list] = None,
+        timeout_seconds: int = 600,
+        boto3_session: Optional[boto3.Session] = None,
+        update_if_exists: bool = False,
+    ) -> Dict[str, Any]:
+        """
+        Create a CloudFormation stack and wait for it to complete.
+
+        Args:
+            stack_name: Name of the CloudFormation stack
+            template_body: CloudFormation template body
+            parameters: Stack parameters
+            region: AWS region
+            capabilities: Stack capabilities (default: CAPABILITY_NAMED_IAM, CAPABILITY_AUTO_EXPAND)
+            timeout_seconds: Timeout in seconds (default: 600)
+            boto3_session: Optional boto3 session (creates new if not provided)
+            update_if_exists: Whether to update stack if it already exists (default: False)
+
+        Returns:
+            CloudFormation stack information
+
+        Raises:
+            ClickException: If stack creation fails or times out
+        """
+        if boto3_session is None:
+            boto3_session = boto3.Session(region_name=region)
+
+        cfn_client = boto3_session.client("cloudformation", region_name=region)
+
+        if capabilities is None:
+            capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"]
+
+        # Create the stack
+        self.log.info(f"Creating CloudFormation stack: {stack_name}")
+        try:
+            cfn_client.create_stack(
+                StackName=stack_name,
+                TemplateBody=template_body,
+                Parameters=parameters,
+                Capabilities=capabilities,
+            )
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            if error_code == "AlreadyExistsException" and update_if_exists:
+                self.log.warning(f"Stack {stack_name} already exists, updating...")
+                try:
+                    cfn_client.update_stack(
+                        StackName=stack_name,
+                        TemplateBody=template_body,
+                        Parameters=parameters,
+                        Capabilities=capabilities,
+                    )
+                except ClientError as update_error:
+                    raise ClickException(
+                        f"Failed to update existing stack: {update_error}"
+                    )
+            else:
+                raise ClickException(f"Failed to create CloudFormation stack: {e}")
+
+        # Wait for stack completion
+        return self._wait_for_stack_completion(
+            cfn_client, stack_name, region, timeout_seconds
+        )
+
+    def _wait_for_stack_completion(
+        self, cfn_client, stack_name: str, region: str, timeout_seconds: int
+    ) -> Dict[str, Any]:
+        """Wait for CloudFormation stack to complete with timeout."""
+        stack_url = f"https://{region}.console.aws.amazon.com/cloudformation/home?region={region}#/stacks/stackinfo?stackId={stack_name}"
+        self.log.info(f"Track progress at: {stack_url}")
+
+        with self.log.spinner("Creating cloud resources through CloudFormation..."):
+            start_time = time.time()
+            end_time = start_time + timeout_seconds
+
+            while time.time() < end_time:
+                try:
+                    stacks = cfn_client.describe_stacks(StackName=stack_name)
+                    cfn_stack = stacks["Stacks"][0]
+                    stack_status = cfn_stack["StackStatus"]
+
+                    if stack_status in (
+                        "CREATE_FAILED",
+                        "ROLLBACK_COMPLETE",
+                        "ROLLBACK_IN_PROGRESS",
+                        "UPDATE_FAILED",
+                    ):
+                        # Clean up resources if stack failed
+                        self._cleanup_failed_stack(cfn_client, stack_name, region)
+
+                        # Get error details
+                        error_details = self._get_stack_error_details(
+                            cfn_client, stack_name
+                        )
+                        raise ClickException(
+                            f"CloudFormation stack failed: {stack_status}. "
+                            f"Error details: {error_details}. "
+                            f"Check the stack at: {stack_url}"
+                        )
+
+                    if stack_status in ("CREATE_COMPLETE", "UPDATE_COMPLETE"):
+                        self.log.info(
+                            f"CloudFormation stack {stack_name} completed successfully"
+                        )
+                        return cfn_stack
+
+                    # Still in progress
+                    time.sleep(5)
+
+                except ClientError as e:
+                    if "does not exist" in str(e):
+                        raise ClickException(
+                            f"CloudFormation stack {stack_name} not found"
+                        )
+                    raise ClickException(f"Error checking CloudFormation stack: {e}")
+
+            # Timeout
+            raise ClickException(
+                f"CloudFormation stack {stack_name} timed out after {timeout_seconds} seconds. "
+                f"Check the stack at: {stack_url}"
+            )
+
+    def _cleanup_failed_stack(self, cfn_client, stack_name: str, region: str) -> None:
+        """Clean up resources from a failed CloudFormation stack."""
+        try:
+            # Try to get stack outputs to find resources to clean up
+            stacks = cfn_client.describe_stacks(StackName=stack_name)
+            stack = stacks["Stacks"][0]
+
+            # Look for S3 bucket in outputs
+            for output in stack.get("Outputs", []):
+                if "Bucket" in output.get("OutputKey", ""):
+                    bucket_name = output.get("OutputValue")
+                    if bucket_name:
+                        self._cleanup_s3_bucket(bucket_name, region)
+
+        except Exception as e:  # noqa: BLE001
+            self.log.warning(f"Failed to cleanup resources from failed stack: {e}")
+
+    def _cleanup_s3_bucket(self, bucket_name: str, region: str) -> None:
+        """Clean up S3 bucket created by failed stack."""
+        try:
+            s3_client = boto3.client("s3", region_name=region)
+            s3_client.delete_bucket(Bucket=bucket_name)
+            self.log.info(f"Successfully deleted S3 bucket: {bucket_name}")
+        except ClientError as e:
+            if e.response["Error"]["Code"] != "NoSuchBucket":
+                self.log.error(f"Failed to delete S3 bucket {bucket_name}: {e}")
+        except Exception as e:  # noqa: BLE001
+            self.log.error(f"Failed to delete S3 bucket {bucket_name}: {e}")
+
+    def _get_stack_error_details(self, cfn_client, stack_name: str) -> str:
+        """Get detailed error information from CloudFormation stack events."""
+        try:
+            events = cfn_client.describe_stack_events(StackName=stack_name)
+            error_events = [
+                event
+                for event in events.get("StackEvents", [])
+                if event.get("ResourceStatus", "").endswith("_FAILED")
+            ]
+
+            if error_events:
+                latest_error = error_events[0]
+                return (
+                    f"Resource: {latest_error.get('LogicalResourceId', 'Unknown')}, "
+                    f"Status: {latest_error.get('ResourceStatus', 'Unknown')}, "
+                    f"Reason: {latest_error.get('ResourceStatusReason', 'Unknown')}"
+                )
+            else:
+                return "No detailed error information available"
+
+        except Exception as e:  # noqa: BLE001
+            return f"Failed to get error details: {e}"
+
+    def get_stack_outputs(
+        self,
+        stack_name: str,
+        region: str,
+        boto3_session: Optional[boto3.Session] = None,
+    ) -> Dict[str, str]:
+        """
+        Get all outputs from a CloudFormation stack.
+
+        Args:
+            stack_name: Name of the CloudFormation stack
+            region: AWS region
+            boto3_session: Optional boto3 session (creates new if not provided)
+
+        Returns:
+            Dictionary of output key-value pairs
+
+        Raises:
+            ClickException: If stack not found or error retrieving outputs
+        """
+        if boto3_session is None:
+            boto3_session = boto3.Session(region_name=region)
+
+        cfn_client = boto3_session.client("cloudformation", region_name=region)
+
+        try:
+            stacks = cfn_client.describe_stacks(StackName=stack_name)
+            stack = stacks["Stacks"][0]
+            outputs = {}
+
+            for output in stack.get("Outputs", []):
+                outputs[output["OutputKey"]] = output["OutputValue"]
+
+            return outputs
+        except ClientError as e:
+            if "does not exist" in str(e):
+                raise ClickException(f"CloudFormation stack {stack_name} not found")
+            raise ClickException(f"Failed to get CloudFormation outputs: {e}")
+
+    def get_stack_output(
+        self,
+        stack_name: str,
+        output_key: str,
+        region: str,
+        boto3_session: Optional[boto3.Session] = None,
+    ) -> str:
+        """
+        Get a specific output value from a CloudFormation stack.
+
+        Args:
+            stack_name: Name of the CloudFormation stack
+            output_key: Key of the output to retrieve
+            region: AWS region
+            boto3_session: Optional boto3 session (creates new if not provided)
+
+        Returns:
+            Output value
+
+        Raises:
+            ClickException: If stack not found or output key not found
+        """
+        outputs = self.get_stack_outputs(stack_name, region, boto3_session)
+
+        if output_key not in outputs:
+            available_keys = list(outputs.keys())
+            raise ClickException(
+                f"Output key '{output_key}' not found in stack {stack_name}. "
+                f"Available keys: {available_keys}"
+            )
+
+        return outputs[output_key]
+
+    def delete_stack(
+        self,
+        stack_name: str,
+        region: str,
+        boto3_session: Optional[boto3.Session] = None,
+    ) -> None:
+        """
+        Delete a CloudFormation stack.
+
+        Args:
+            stack_name: Name of the CloudFormation stack
+            region: AWS region
+            boto3_session: Optional boto3 session (creates new if not provided)
+
+        Raises:
+            ClickException: If stack deletion fails
+        """
+        if boto3_session is None:
+            boto3_session = boto3.Session(region_name=region)
+
+        cfn_client = boto3_session.client("cloudformation", region_name=region)
+
+        try:
+            self.log.info(f"Deleting CloudFormation stack: {stack_name}")
+            cfn_client.delete_stack(StackName=stack_name)
+
+            # Wait for deletion to complete
+            with self.log.spinner("Deleting CloudFormation stack..."):
+                start_time = time.time()
+                timeout_seconds = 300  # 5 minutes for deletion
+                end_time = start_time + timeout_seconds
+
+                while time.time() < end_time:
+                    try:
+                        cfn_client.describe_stacks(StackName=stack_name)
+                        time.sleep(5)  # Still exists, wait
+                    except ClientError as e:
+                        if "does not exist" in str(e):
+                            self.log.info(
+                                f"CloudFormation stack {stack_name} deleted successfully"
+                            )
+                            return
+                        raise ClickException(f"Error checking stack deletion: {e}")
+
+                raise ClickException(
+                    f"Stack deletion timed out after {timeout_seconds} seconds"
+                )
+
+        except ClientError as e:
+            if "does not exist" in str(e):
+                self.log.info(f"Stack {stack_name} does not exist")
+                return
+            raise ClickException(f"Failed to delete CloudFormation stack: {e}")
+
+
+# Convenience functions for backward compatibility
+def create_cloudformation_stack(
+    stack_name: str,
+    template_body: str,
+    parameters: list,
+    region: str,
+    logger: Optional[BlockLogger] = None,
+    **kwargs,
+) -> Dict[str, Any]:
+    """Convenience function to create a CloudFormation stack."""
+    utils = CloudFormationUtils(logger)
+    return utils.create_and_wait_for_stack(
+        stack_name, template_body, parameters, region, **kwargs
+    )
+
+
+def get_cloudformation_outputs(
+    stack_name: str, region: str, logger: Optional[BlockLogger] = None, **kwargs
+) -> Dict[str, str]:
+    """Convenience function to get CloudFormation stack outputs."""
+    utils = CloudFormationUtils(logger)
+    return utils.get_stack_outputs(stack_name, region, **kwargs)
+
+
+def get_cloudformation_output(
+    stack_name: str,
+    output_key: str,
+    region: str,
+    logger: Optional[BlockLogger] = None,
+    **kwargs,
+) -> str:
+    """Convenience function to get a specific CloudFormation stack output."""
+    utils = CloudFormationUtils(logger)
+    return utils.get_stack_output(stack_name, output_key, region, **kwargs)

anyscale/version.py

@@ -1 +1 @@
-__version__ = "0.26.66"
+__version__ = "0.26.68"

{anyscale-0.26.66.dist-info → anyscale-0.26.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anyscale
-Version: 0.26.66
+Version: 0.26.68
 Summary: Command Line Interface for Anyscale
 Author: Anyscale Inc.
 License: AS License

{anyscale-0.26.66.dist-info → anyscale-0.26.68.dist-info}/RECORD

@@ -28,7 +28,7 @@ anyscale/snapshot.py,sha256=UGJT5C1s_4xmQxjWODK5DFpGxHRBX5jOCdSCqXESH8E,1685
 anyscale/tables.py,sha256=TV4F2uLnwehvbkAfaP7iuLlT2wLIo6ORH2LVdRGXW5g,2840
 anyscale/telemetry.py,sha256=U90C2Vgx48z9PMTI6EbzHFbP3jWnDUutbIfMPBb8-SI,14711
 anyscale/util.py,sha256=3jfGH2CyDmXrSg-3owSF7F8FnqKbLJ-aHUbepyCKuqQ,42903
-anyscale/version.py,sha256=zj93ulo-laQiiTnH5BhCREtuFiM0mXZERQJbP5vaSNk,24
+anyscale/version.py,sha256=lpQs-Szxlm150B9mnwXj1al8clhb5Zo-ZL5AiFSuzk8,24
 anyscale/workspace_utils.py,sha256=OViE88CnIF5ruVxd3kazQ0Mf2BxqtMq6wx-XQ5A2cp8,1204
 anyscale/_private/anyscale_client/README.md,sha256=kSfI2Jfw5RHZWYtu0di3XtdSCx0d2pSwKMfjmDvw7Tg,3770
 anyscale/_private/anyscale_client/__init__.py,sha256=807Blx3RHQeS8BmKZcsOQQ4dYoKlCnpm6Bdsif2CrHg,337
@@ -106,7 +106,7 @@ anyscale/background/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 anyscale/background/job_runner.py,sha256=LTuv9JOahyv6C9i7DLQAONgQF6--FfYZEmJrKy-sUG8,2687
 anyscale/client/.gitignore,sha256=JZyvYEtT2DCSK9V5Joi6lQofhMik4PXiJRCWsg7SvqI,807
 anyscale/client/.openapi-generator-ignore,sha256=pu2PTide7pJtJ-DFLzDy0cTYQJRlrB-8RRH3zGLeUds,1040
-anyscale/client/README.md,sha256=qCVY-Th4K0QhR-tPgXsHF5-mjgTWupf2nHSsYWe73pk,130129
+anyscale/client/README.md,sha256=yVThjY7_FVCDVT5NRRJ-05L9BheiL0Kr01CVnSie_6I,131991
 anyscale/client/git_push.sh,sha256=EDCZOTTiLxbtPHmiU63qC99rGH67B7dhdPZdNUKivF0,1827
 anyscale/client/requirements.txt,sha256=dkVKYUStC5h_g_87SH7pRdhXCj7ySozAJMGAFEzGgFc,126
 anyscale/client/setup.cfg,sha256=l7bdKSIedeBhhoDtupsBwx1xPrlBf2yYeTH7a8kMga4,28
@@ -114,14 +114,14 @@ anyscale/client/setup.py,sha256=tSxqw1kAL1B9adnrnOarjnQfSbwGmnTr_kg8ZXhlm5A,1109
 anyscale/client/test-requirements.txt,sha256=sTjmDTj5W9fh1ZAeo8UT2EBdeGDBNttj_PHiPBXg1D4,111
 anyscale/client/tox.ini,sha256=M6L3UmvAdvU65LsoAF-Oi7oRjwZlCJZn8I7ofdXn5Ok,156
 anyscale/client/.openapi-generator/VERSION,sha256=J0RzX-4u4jfin1kviKtmncjUePyjHm2kyvmkobOrt_E,5
-anyscale/client/openapi_client/__init__.py,sha256=b-c1lof23LY0MYsuBNLux_e1KPw87ueD5ltlQPOLnlg,58726
+anyscale/client/openapi_client/__init__.py,sha256=mOXEUcUD_s5VokuYaTXHOhFQKSvsFWVZ7kxmmJp5SMo,59881
 anyscale/client/openapi_client/api_client.py,sha256=d8Un6j2Ny2vlS2qBXPVFj6_ql0k36DFahpWt_28TfCk,25563
 anyscale/client/openapi_client/configuration.py,sha256=Dd5XrlHwv-wxnf0C35PG_-HBQoY3Yaz6hKrmkZz-m0E,12363
 anyscale/client/openapi_client/exceptions.py,sha256=3egwsXQG2j_vARbqgBxUO1xSltAhpfiHTYVP7VXTvU0,3792
 anyscale/client/openapi_client/rest.py,sha256=Ehj37v7GHW6SXV067Hze5HE42ayKaGi6a6ZlkR7u3Lg,12501
 anyscale/client/openapi_client/api/__init__.py,sha256=i8u7BI2xX1GrXTL3hN0pKpYIlnT-D_uDxH2ElOfYG1I,141
-anyscale/client/openapi_client/api/default_api.py,sha256=bRwzeBTOW5dCQAcGa41eMFu9KEUiW5j64j3DqDtJWGI,2117163
-anyscale/client/openapi_client/models/__init__.py,sha256=C-fwaLsEv15SLgFnX8KGiFZndsVFApWVwVjLc7hXaRA,58236
+anyscale/client/openapi_client/api/default_api.py,sha256=JGRjn6bzNkl2ysAD-cNlfloEioWDI3KXzZdROJjUU_U,2151488
+anyscale/client/openapi_client/models/__init__.py,sha256=M51ftYXV7OZ-7DiL_hWnQXKH18zDvIjwYA4vgB0YFx4,59391
 anyscale/client/openapi_client/models/access_config.py,sha256=b2mA0qtuTA5PFbp6C61Jc_T2zUMaojM1v32IhZo0MfY,3648
 anyscale/client/openapi_client/models/actor_status.py,sha256=6xyX_aIqURj2raBdY9DmBxsdDACFrqqYvElGiM6YG2E,2813
 anyscale/client/openapi_client/models/admin_create_user.py,sha256=9DPr8D0lKgoEZ3Z2kGsAd8L7ocFCiP6woOGLVs8SRb8,7251
@@ -481,6 +481,21 @@ anyscale/client/openapi_client/models/kubernetes_manager_registration_response.p
 anyscale/client/openapi_client/models/kubernetesmanagerregistrationresponse_response.py,sha256=8UH867yGDAOgZrfXdkRzVkfTLViyTRQIZFBFd0vghu0,3825
 anyscale/client/openapi_client/models/lb_resource.py,sha256=UAjfCjXZAvAn64WSltl-VWw1WyjMrCi73A1jtY2F4To,3739
 anyscale/client/openapi_client/models/lbresource_response.py,sha256=AgJpUmMDDuF6ohHI2LbPoXICUKeBKWskgjaOcr59t8E,3528
+anyscale/client/openapi_client/models/lineage_artifact.py,sha256=1ACZuH2FaYMW3XFGAX-IyupnvwngguaTeRfA3hTeoSE,12118
+anyscale/client/openapi_client/models/lineage_artifact_sort_field.py,sha256=pOicxc4zT9fjSKG58AAiacJzo335d5y2SUdShrSm8dI,2922
+anyscale/client/openapi_client/models/lineage_artifact_type.py,sha256=rZV5nndrPi0P75fpsg6ZnGRDUxwk0OFWNef22840Kl0,2854
+anyscale/client/openapi_client/models/lineage_direction.py,sha256=tvIjSwnt4MSxuczUBI_KlI7gHiMCB76skHSdx_N8oIc,2848
+anyscale/client/openapi_client/models/lineage_graph.py,sha256=yn34T-j0-PWdvIMCTemoOLyGZ5UQgu2HYnimmODd_Ys,5216
+anyscale/client/openapi_client/models/lineage_graph_node.py,sha256=KWGpK42UCjEhe7bhR79CIJmuZdRL3NP4pwC0ig7Fs6E,13946
+anyscale/client/openapi_client/models/lineage_node_type.py,sha256=bi8gje1x01FtL0l1PzN76aQbBCeOkLWk_e2YXd3jdwY,2850
+anyscale/client/openapi_client/models/lineage_workload.py,sha256=F_iwZlFliVRPROmTwQZ_kwrzJ4kevnXfTY6q40ro3dk,11115
+anyscale/client/openapi_client/models/lineage_workload_sort_field.py,sha256=u5seM0kQtRVrEAhbQ38MlMvGYbIBTwUB52Yc7pyr7n0,2922
+anyscale/client/openapi_client/models/lineage_workload_type.py,sha256=kjZlfsIo6rRarGpnahuZEX1zM-qyqe3drQb1QO3xkZI,2914
+anyscale/client/openapi_client/models/lineageartifact_list_response.py,sha256=t4QnqXbEkoADCzpm4q23EP9VPQueTLWKmDUJ6msKZZE,4422
+anyscale/client/openapi_client/models/lineageartifact_response.py,sha256=VEkgCnS8A65Ia6opIQLeAVYPakrI5M0tNVRq-VZ8BQc,3583
+anyscale/client/openapi_client/models/lineagegraph_response.py,sha256=YwM0XPkLdrWL5hTKYT1FzMTPvOJDJHAQaWqHmkXmHzk,3550
+anyscale/client/openapi_client/models/lineageworkload_list_response.py,sha256=q6aoNXijJ7oggjGym0YkWtm_L55FCCUp4M86PbZLOGI,4422
+anyscale/client/openapi_client/models/lineageworkload_response.py,sha256=ORICo_b71QEmKnT4EapJ02unA0AFf8y0_0JjNG9ROOA,3583
 anyscale/client/openapi_client/models/list_databricks_connections.py,sha256=XRCO-FYXXWEcQ7qm1gL4uu7XP_1HSiwAx-AoNB4AtJA,3749
 anyscale/client/openapi_client/models/list_machine_pools_response.py,sha256=lH_AGSraAalhusdwbb31uSCNBobNIdTA8mTVPkA_X7Y,3842
 anyscale/client/openapi_client/models/list_machines_response.py,sha256=8aS1GNXSzf_j9MaKoRjWxJ-7xQoe1_4wS5oX3qUZEi0,3607
@@ -834,7 +849,7 @@ anyscale/cloud/_private/cloud_sdk.py,sha256=5TBGyGSjMI4jLOnSle1WWC6za0psP9xgTGWU
 anyscale/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anyscale/commands/aggregated_instance_usage_commands.py,sha256=TRP1X3hdIWbKg9V20VtazlDXsYAeV--M0DH3-Z5tnj4,2293
 anyscale/commands/auth_commands.py,sha256=X1g6Yu9kqgPb4HLODlZTYEk8G5AVLeyizPIgagWx-p0,1026
-anyscale/commands/cloud_commands.py,sha256=iNuDD8LoJe4y12bZpCDiYCs2YNxRC1rkLuzmgt7Rj_k,58604
+anyscale/commands/cloud_commands.py,sha256=NdB4O_Nee-ute0lXWs7xIotdoVuoyHUvBbYXueODFGk,60185
 anyscale/commands/cluster_commands.py,sha256=taNcffyFfqJ1MgOQd0cz9kzRXWFTdp-wfLPM4l_2tBc,13487
 anyscale/commands/cluster_env_commands.py,sha256=KNWylyE8Ew1sDi7yu2Tp4RLcRu2_KJJJIzVGRyPflJo,3899
 anyscale/commands/command_examples.py,sha256=pSj1DyHwlzmSxw-Ko5RUTQ57uFA7v7Jgur0s7K55bts,25182
@@ -859,6 +874,7 @@ anyscale/commands/schedule_commands.py,sha256=Bw2aKp_w6xcuRSVVi9FLdUjRVCr8_v4Tt2
 anyscale/commands/service_account_commands.py,sha256=JlppTmcaDofJn7TYqd3PW4hDGykdEN56fr0PT3qgBIg,3271
 anyscale/commands/service_commands.py,sha256=DBrPkW7JH3kCsr038aQStm7AF1u-n80sLjwc2Hjifks,34014
 anyscale/commands/session_commands_hidden.py,sha256=APEypnUB1yV2Rr6wdSFWy1vQbAnn-lOn0rU2enF5JdM,6200
+anyscale/commands/setup_k8s.py,sha256=MGiCtRrkMEDcQa0C7Sc_3YZKzwaABuAk82k3URGXz8Q,58248
 anyscale/commands/user_commands.py,sha256=C-i1dGpdhboywN_2XgPS2BekKx2y6LZq8c8gvS0S-tY,1259
 anyscale/commands/util.py,sha256=fIob29G1jZ4VU9PKVSJnT6hqX7wAqsS4L9sApsMZcNs,8585
 anyscale/commands/workspace_commands.py,sha256=5JoF5xftkwkxHidp8xyj9POrlZE_WxwSKtZGnKCbDV4,18597
@@ -878,7 +894,7 @@ anyscale/connect_utils/start_interactive_session.py,sha256=m-RCH0e_bQBF6dAOskbP9
 anyscale/controllers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anyscale/controllers/auth_controller.py,sha256=hDY2sPvUP8pvh8PnlDYH5rCHjQes2v3b_KBVjMbrzeE,5127
 anyscale/controllers/base_controller.py,sha256=1QFJoScFUV7YTzpKarhwPOc1SvI-xqX3TZmwxKonW6I,1998
-anyscale/controllers/cloud_controller.py,sha256=PYugEBT2UV0_pPYI6FCn7J6QI2F_LW5LxliIHK1ta3o,201971
+anyscale/controllers/cloud_controller.py,sha256=1mZIQ4OVIu0mQ4ibKVw8pJskiNzf0ZdDnc4FIGBS8IE,202079
 anyscale/controllers/cloud_file_storage_utils.py,sha256=ifaqClEybTgxhqGWHYoH1vrlbxwjRuO-De_3666R2O4,6987
 anyscale/controllers/cloud_functional_verification_controller.py,sha256=uro10r981CTlt2zlisUCmNYljbrYBiWzh7vrqrcur3I,33475
 anyscale/controllers/cluster_controller.py,sha256=Sb5wVjrjpycg5iqmENAVtZ4iy9Kr6kM97_ck-KH85LM,28745
@@ -888,7 +904,7 @@ anyscale/controllers/config_controller.py,sha256=VsfdARHxo4tMLfeiYkTNOMGW3sIcNhV
 anyscale/controllers/experimental_integrations_controller.py,sha256=_22_hAQCJIMg3E10s8xajoFF6Lf1HqVlAdAVt0Rh2DY,3889
 anyscale/controllers/job_controller.py,sha256=sNQGzSLtp6e8PSbrmMrW_dp3pYytS8KCGcE-YjaNz5I,25425
 anyscale/controllers/jobs_bg_controller.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anyscale/controllers/kubernetes_verifier.py,sha256=rs7aNepuy4wZgteAtJOqGM9HwA0cH_ndrOAp_LpqPeY,58724
+anyscale/controllers/kubernetes_verifier.py,sha256=0mGeE8i8ptWdGwVDOFFWuxMzqXpYNVZeDixbSUuKx2k,61300
 anyscale/controllers/list_controller.py,sha256=oaOS6oo2TKPpXhGjs_laxuIVKinv3FwYfHt1CIzeTuU,11621
 anyscale/controllers/logs_controller.py,sha256=x5GBUVdPYhbWRA3RfMQZJi3hBS2i35RkgzROfmY47h4,17647
 anyscale/controllers/machine_controller.py,sha256=WauNi7Spzt2TFZrIN4PwULzgJZBVDFlytzFqpDQ106w,1188
@@ -1140,6 +1156,7 @@ anyscale/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anyscale/utils/cli_version_check_util.py,sha256=U2KU-NRf09pcs-ZZYUm1GQxs7Btp6N4Fv6CQeNoEeTM,2223
 anyscale/utils/cloud_update_utils.py,sha256=c53AQyg4OJBkNqTD-8hHvEpdTwtins1rTycifBbQbWI,34106
 anyscale/utils/cloud_utils.py,sha256=DtUkaIvKeX64Scia62fjnK61_gNMJr_H2e73C-ywF7k,12228
+anyscale/utils/cloudformation_utils.py,sha256=G4QJbnwve8ZQu7qzrT7ZVZ32zqf_bDxBDZBNZEtFkZg,13839
 anyscale/utils/cluster_debug.py,sha256=P_-20L4Rt4LVAIGX-y9F61z5XYO5QbNtqf2MYJxuy_g,6391
 anyscale/utils/connect_helpers.py,sha256=hlTKnaL27pABWPoW5nA5dpJW2ZG_2k7dXjnqtdzoBjo,5956
 anyscale/utils/deprecation_util.py,sha256=2i1SNQIziOZXO24IuBZZUEi4ild1bqk0gfZWl2KgpXw,1160
@@ -1172,10 +1189,10 @@ anyscale/workspace/__init__.py,sha256=Innbm5ZhCyADEVBiYSo_vbpKwUNcMzVSAfxIGKOYe6
 anyscale/workspace/commands.py,sha256=b1sqNseoPj-1VXznqQOLe0V_a663bOTvJX-TaOMJa1Y,14590
 anyscale/workspace/models.py,sha256=uiMqoJRQNRgTcOIIsysSrtlHMtnI7paUWS34EN626Cg,10016
 anyscale/workspace/_private/workspace_sdk.py,sha256=2CMeYfJt0UtIFCocDn1ukw1iI5esKHdopLe6duEs-qE,27599
-anyscale-0.26.66.dist-info/licenses/LICENSE,sha256=UOPu974Wzsna6frFv1mu4VrZgNdZT7lbcNPzo5ue3qs,3494
-anyscale-0.26.66.dist-info/licenses/NOTICE,sha256=gHqDhSnUYlRXX-mDOL5FtE7774oiKyV_HO80qM3r9Xo,196
-anyscale-0.26.66.dist-info/METADATA,sha256=hPPRjsbemUyxKUKtECR6o5sQmIknnbRGC0YLVhdrfoU,3231
-anyscale-0.26.66.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-anyscale-0.26.66.dist-info/entry_points.txt,sha256=NqO18sCZn6zG6J0S38itjcN00s7aE3C3v3k5lMAfCLk,51
-anyscale-0.26.66.dist-info/top_level.txt,sha256=g3NVNS8Oh0NZwbFFgeX696C5MZZkS5dqV2NqcsbDRJE,9
-anyscale-0.26.66.dist-info/RECORD,,
+anyscale-0.26.68.dist-info/licenses/LICENSE,sha256=UOPu974Wzsna6frFv1mu4VrZgNdZT7lbcNPzo5ue3qs,3494
+anyscale-0.26.68.dist-info/licenses/NOTICE,sha256=gHqDhSnUYlRXX-mDOL5FtE7774oiKyV_HO80qM3r9Xo,196
+anyscale-0.26.68.dist-info/METADATA,sha256=HkQPlzY48CjPRep1GyHX3OdjWnHzBKpNKXPFZQhxxN4,3231
+anyscale-0.26.68.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anyscale-0.26.68.dist-info/entry_points.txt,sha256=NqO18sCZn6zG6J0S38itjcN00s7aE3C3v3k5lMAfCLk,51
+anyscale-0.26.68.dist-info/top_level.txt,sha256=g3NVNS8Oh0NZwbFFgeX696C5MZZkS5dqV2NqcsbDRJE,9
+anyscale-0.26.68.dist-info/RECORD,,