anyscale 0.26.50__py3-none-any.whl → 0.26.52__py3-none-any.whl

anyscale/_private/workload/workload_sdk.py

@@ -112,7 +112,7 @@ class WorkloadSDK(BaseSDK):
 
         return new_runtime_envs
 
-    def override_and_upload_local_dirs(
+    def override_and_upload_local_dirs_single_deployment(  # noqa: PLR0912
         self,
         runtime_envs: List[Dict[str, Any]],
         *,
@@ -122,7 +122,7 @@ class WorkloadSDK(BaseSDK):
         autopopulate_in_workspace: bool = True,
         additional_py_modules: Optional[List[str]] = None,
         py_executable_override: Optional[str] = None,
-        has_multiple_cloud_deployments: bool = False,
+        cloud_deployment: Optional[str] = None,
     ) -> List[Dict[str, Any]]:
         """Returns modified runtime_envs with all local dirs converted to remote URIs.
 
@@ -142,15 +142,13 @@ class WorkloadSDK(BaseSDK):
             if target in local_path_to_uri:
                 return local_path_to_uri[target]
 
-            if has_multiple_cloud_deployments:
-                self.logger.warning(
-                    "For compute configurations with multiple cloud deployments, local directories will only be uploaded to the object storage of the primary cloud deployment."
-                )
-
             self.logger.info(f"Uploading local dir '{target}' to cloud storage.")
             assert cloud_id is not None
             uri = self._client.upload_local_dir_to_cloud_storage(
-                target, cloud_id=cloud_id, excludes=excludes,
+                target,
+                cloud_id=cloud_id,
+                excludes=excludes,
+                cloud_deployment=cloud_deployment,
             )
             local_path_to_uri[target] = uri
             return uri
@@ -192,6 +190,103 @@ class WorkloadSDK(BaseSDK):
 
         return new_runtime_envs
 
+    def override_and_upload_local_dirs_multi_deployment(  # noqa: PLR0912
+        self,
+        runtime_envs: List[Dict[str, Any]],
+        *,
+        working_dir_override: Optional[str],
+        excludes_override: Optional[List[str]],
+        cloud_deployments: List[Optional[str]],
+        cloud_id: Optional[str] = None,
+        autopopulate_in_workspace: bool = True,
+        additional_py_modules: Optional[List[str]] = None,
+        py_executable_override: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """Returns modified runtime_envs with all local dirs converted to remote bucket paths,
+        stored in the "relative_working_dir" and "relative_py_modules" fields.
+
+        The precedence for overrides is: explicit overrides passed in > fields in the existing
+        runtime_envs > workspace defaults (if `autopopulate_in_workspace == True`).
+
+        Each unique local directory across these fields will be uploaded once to cloud storage,
+        then all occurrences of it in the config will be replaced with the corresponding remote URI.
+        """
+        new_runtime_envs = copy.deepcopy(runtime_envs)
+
+        local_path_to_bucket_path: Dict[str, str] = {}
+
+        def _upload_dir_memoized(target: str, *, excludes: Optional[List[str]]) -> str:
+            if target in local_path_to_bucket_path:
+                return local_path_to_bucket_path[target]
+
+            self.logger.info(
+                f"Uploading local dir '{target}' to object storage for all {len(cloud_deployments)} cloud deployments in the compute config."
+            )
+            assert cloud_id is not None
+            bucket_path = self._client.upload_local_dir_to_cloud_storage_multi_deployment(
+                target,
+                cloud_id=cloud_id,
+                excludes=excludes,
+                cloud_deployments=cloud_deployments,
+            )
+            local_path_to_bucket_path[target] = bucket_path
+            return bucket_path
+
+        for runtime_env in new_runtime_envs:
+            # Extend, don't overwrite, excludes if it's provided.
+            if excludes_override is not None:
+                existing_excludes = runtime_env.get("excludes", None) or []
+                runtime_env["excludes"] = existing_excludes + excludes_override
+
+            final_excludes = runtime_env.get("excludes", [])
+
+            new_working_dir = None
+            if working_dir_override is not None:
+                new_working_dir = working_dir_override
+            elif "working_dir" in runtime_env:
+                new_working_dir = runtime_env["working_dir"]
+            elif autopopulate_in_workspace and self._client.inside_workspace():
+                new_working_dir = "."
+
+            if new_working_dir is not None:
+                if is_dir_remote_uri(new_working_dir):
+                    runtime_env["working_dir"] = new_working_dir
+                else:
+                    runtime_env["relative_working_dir"] = _upload_dir_memoized(
+                        new_working_dir, excludes=final_excludes
+                    )
+                    runtime_env.pop("working_dir", None)
+
+            if additional_py_modules:
+                existing_py_modules = runtime_env.get("py_modules", [])
+                runtime_env["py_modules"] = existing_py_modules + additional_py_modules
+
+            final_py_modules = runtime_env.get("py_modules", None)
+            if final_py_modules is not None:
+                py_modules = [
+                    py_module
+                    for py_module in final_py_modules
+                    if is_dir_remote_uri(py_module)
+                ]
+                if len(py_modules) > 0:
+                    runtime_env["py_modules"] = py_modules
+                else:
+                    # If there are no py_modules, remove the field.
+                    runtime_env.pop("py_modules", None)
+
+                relative_py_modules = [
+                    _upload_dir_memoized(py_module, excludes=final_excludes)
+                    for py_module in final_py_modules
+                    if not is_dir_remote_uri(py_module)
+                ]
+                if len(relative_py_modules) > 0:
+                    runtime_env["relative_py_modules"] = relative_py_modules
+
+            if py_executable_override:
+                runtime_env["py_executable"] = py_executable_override
+
+        return new_runtime_envs
+
     def _resolve_compute_config_id(
         self,
         compute_config: Union[str, ComputeConfigType, None],

anyscale/client/README.md

@@ -209,9 +209,12 @@ Class | Method | HTTP request | Description
 *DefaultApi* | [**get_job_logs_download_v2_api_v2_logs_job_logs_download_v2_job_id_get**](docs/DefaultApi.md#get_job_logs_download_v2_api_v2_logs_job_logs_download_v2_job_id_get) | **GET** /api/v2/logs/job_logs_download_v2/{job_id} | Get Job Logs Download V2
 *DefaultApi* | [**get_job_queue_api_v2_job_queues_job_queue_id_get**](docs/DefaultApi.md#get_job_queue_api_v2_job_queues_job_queue_id_get) | **GET** /api/v2/job_queues/{job_queue_id} | Get Job Queue
 *DefaultApi* | [**get_jobs_api_v2_dataset_runs_jobs_get**](docs/DefaultApi.md#get_jobs_api_v2_dataset_runs_jobs_get) | **GET** /api/v2/dataset_runs/jobs | Get Jobs
+*DefaultApi* | [**get_latest_cloud_deployment_api_v2_sessions_cluster_id_latest_cloud_deployment_get**](docs/DefaultApi.md#get_latest_cloud_deployment_api_v2_sessions_cluster_id_latest_cloud_deployment_get) | **GET** /api/v2/sessions/{cluster_id}/latest_cloud_deployment | Get Latest Cloud Deployment
 *DefaultApi* | [**get_lb_resource_api_v2_clouds_with_cloud_resource_gcp_router_cloud_id_get_lb_resource_post**](docs/DefaultApi.md#get_lb_resource_api_v2_clouds_with_cloud_resource_gcp_router_cloud_id_get_lb_resource_post) | **POST** /api/v2/clouds_with_cloud_resource_gcp_router/{cloud_id}/get_lb_resource | Get Lb Resource
+*DefaultApi* | [**get_lb_resource_api_v2_clouds_with_cloud_resource_router_cloud_id_get_lb_resource_post**](docs/DefaultApi.md#get_lb_resource_api_v2_clouds_with_cloud_resource_router_cloud_id_get_lb_resource_post) | **POST** /api/v2/clouds_with_cloud_resource_router/{cloud_id}/get_lb_resource | Get Lb Resource
 *DefaultApi* | [**get_log_files_api_v2_logs_get_log_files_post**](docs/DefaultApi.md#get_log_files_api_v2_logs_get_log_files_post) | **POST** /api/v2/logs/get_log_files | Get Log Files
 *DefaultApi* | [**get_manage_billing_url_api_v2_organization_billing_manage_billing_url_get**](docs/DefaultApi.md#get_manage_billing_url_api_v2_organization_billing_manage_billing_url_get) | **GET** /api/v2/organization_billing/manage_billing_url | Get Manage Billing Url
+*DefaultApi* | [**get_metric_names_api_v2_metrics_names_get**](docs/DefaultApi.md#get_metric_names_api_v2_metrics_names_get) | **GET** /api/v2/metrics/names | Get Metric Names
 *DefaultApi* | [**get_metronome_customer_info_api_v2_metronome_customer_info_organization_id_get**](docs/DefaultApi.md#get_metronome_customer_info_api_v2_metronome_customer_info_organization_id_get) | **GET** /api/v2/metronome_customer_info/{organization_id} | Get Metronome Customer Info
 *DefaultApi* | [**get_metronome_embedded_usage_dashboard_api_v2_organization_billing_metronome_embedded_dashboard_url_dashboard_type_get**](docs/DefaultApi.md#get_metronome_embedded_usage_dashboard_api_v2_organization_billing_metronome_embedded_dashboard_url_dashboard_type_get) | **GET** /api/v2/organization_billing/metronome_embedded_dashboard_url/{dashboard_type} | Get Metronome Embedded Usage Dashboard
 *DefaultApi* | [**get_metronome_embedded_usage_dashboard_by_organization_api_v2_organization_billing_organization_id_metronome_embedded_dashboard_url_dashboard_type_get**](docs/DefaultApi.md#get_metronome_embedded_usage_dashboard_by_organization_api_v2_organization_billing_organization_id_metronome_embedded_dashboard_url_dashboard_type_get) | **GET** /api/v2/organization_billing/{organization_id}/metronome_embedded_dashboard_url/{dashboard_type} | Get Metronome Embedded Usage Dashboard By Organization
@@ -308,6 +311,7 @@ Class | Method | HTTP request | Description
 *DefaultApi* | [**put_workspace_proxied_dataplane_artifacts_api_v2_experimental_workspaces_workspace_id_proxied_dataplane_artifacts_put**](docs/DefaultApi.md#put_workspace_proxied_dataplane_artifacts_api_v2_experimental_workspaces_workspace_id_proxied_dataplane_artifacts_put) | **PUT** /api/v2/experimental_workspaces/{workspace_id}/proxied_dataplane_artifacts | Put Workspace Proxied Dataplane Artifacts
 *DefaultApi* | [**put_workspace_wandb_run_details_api_v2_integrations_workspace_wandb_run_details_workspace_id_put**](docs/DefaultApi.md#put_workspace_wandb_run_details_api_v2_integrations_workspace_wandb_run_details_workspace_id_put) | **PUT** /api/v2/integrations/workspace_wandb_run_details/{workspace_id} | Put Workspace Wandb Run Details
 *DefaultApi* | [**query_aggregated_logs_api_v2_logs_query_aggregated_logs_get**](docs/DefaultApi.md#query_aggregated_logs_api_v2_logs_query_aggregated_logs_get) | **GET** /api/v2/logs/query_aggregated_logs | Query Aggregated Logs
+*DefaultApi* | [**query_api_v2_logs_query_get**](docs/DefaultApi.md#query_api_v2_logs_query_get) | **GET** /api/v2/logs/query | Query
 *DefaultApi* | [**query_api_v2_metrics_query_get**](docs/DefaultApi.md#query_api_v2_metrics_query_get) | **GET** /api/v2/metrics/query | Query
 *DefaultApi* | [**query_range_api_v2_metrics_query_range_get**](docs/DefaultApi.md#query_range_api_v2_metrics_query_range_get) | **GET** /api/v2/metrics/query_range | Query Range
 *DefaultApi* | [**receive_cli_usage_api_v2_cli_usage_post**](docs/DefaultApi.md#receive_cli_usage_api_v2_cli_usage_post) | **POST** /api/v2/cli_usage/ | Receive Cli Usage
@@ -486,6 +490,7 @@ Class | Method | HTTP request | Description
  - [ClouddatabucketpresigneduploadinfoResponse](docs/ClouddatabucketpresigneduploadinfoResponse.md)
  - [ClouddatabucketpresignedurlresponseResponse](docs/ClouddatabucketpresignedurlresponseResponse.md)
  - [ClouddeploymentListResponse](docs/ClouddeploymentListResponse.md)
+ - [ClouddeploymentResponse](docs/ClouddeploymentResponse.md)
  - [ClouddeploymentconfigResponse](docs/ClouddeploymentconfigResponse.md)
  - [CloudoverviewdashboardResponse](docs/CloudoverviewdashboardResponse.md)
  - [CloudregionandzonesResponse](docs/CloudregionandzonesResponse.md)

anyscale/client/openapi_client/__init__.py

@@ -130,6 +130,7 @@ from openapi_client.models.cloudcollaborator_list_response import Cloudcollabora
 from openapi_client.models.clouddatabucketpresigneduploadinfo_response import ClouddatabucketpresigneduploadinfoResponse
 from openapi_client.models.clouddatabucketpresignedurlresponse_response import ClouddatabucketpresignedurlresponseResponse
 from openapi_client.models.clouddeployment_list_response import ClouddeploymentListResponse
+from openapi_client.models.clouddeployment_response import ClouddeploymentResponse
 from openapi_client.models.clouddeploymentconfig_response import ClouddeploymentconfigResponse
 from openapi_client.models.cloudoverviewdashboard_response import CloudoverviewdashboardResponse
 from openapi_client.models.cloudregionandzones_response import CloudregionandzonesResponse