anyscale 0.24.62__tar.gz → 0.24.64__tar.gz

{anyscale-0.24.62 → anyscale-0.24.64}/BUILD.bazel

@@ -21,6 +21,8 @@ py_library(
     name = "anyscale",
     srcs = glob(["anyscale/**/*.py"]),
     data = [
+        "anyscale/_private/docgen/api.md",
+        "anyscale/_private/docgen/models.md",
         "anyscale/anyscale-cloud-setup.yaml",
         "anyscale/anyscale-cloud-setup-gcp.yaml",
         "anyscale/anyscale-cloud-setup-gcp-oa.yaml",

{anyscale-0.24.62 → anyscale-0.24.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anyscale
-Version: 0.24.62
+Version: 0.24.64
 Summary: Command Line Interface for Anyscale
 Author: Anyscale Inc.
 License: AS License

{anyscale-0.24.62 → anyscale-0.24.64}/anyscale/_private/anyscale_client/anyscale_client.py

@@ -11,6 +11,7 @@ from urllib.parse import urlparse
 
 from openapi_client.exceptions import ApiException
 import requests
+from rich.style import Style
 
 from anyscale._private.anyscale_client.common import (
     AnyscaleClientInterface,
@@ -19,6 +20,10 @@ from anyscale._private.anyscale_client.common import (
     RUNTIME_ENV_PACKAGE_FORMAT,
 )
 from anyscale._private.models.image_uri import ImageURI
+from anyscale._private.utils.progress_util import (
+    FileDownloadProgress,
+    ProgressFileReader,
+)
 from anyscale.api_utils.common_utils import source_cloud_id_and_project_id
 from anyscale.api_utils.logs_util import _download_log_from_s3_url_sync
 from anyscale.authenticate import AuthenticationBlock, get_auth_api_client
@@ -35,8 +40,11 @@ from anyscale.client.openapi_client.models import (
     ComputeTemplateConfig,
     ComputeTemplateQuery,
     CreateComputeTemplate,
+    CreateDataset,
     CreateExperimentalWorkspace,
     CreateInternalProductionJob,
+    Dataset,
+    DatasetUpload,
     DecoratedComputeTemplate,
     DeletedPlatformFineTunedModel,
     ExperimentalWorkspace,
@@ -80,7 +88,9 @@ from anyscale.sdk.anyscale_client.models.sort_by_clause_jobs_sort_field import (
     SortByClauseJobsSortField,
 )
 from anyscale.sdk.anyscale_client.models.sort_order import SortOrder
+from anyscale.sdk.anyscale_client.models.update_cluster import UpdateCluster
 from anyscale.sdk.anyscale_client.rest import ApiException as ExternalApiException
+from anyscale.shared_anyscale_utils.bytes_util import Bytes
 from anyscale.shared_anyscale_utils.conf import ANYSCALE_HOST
 from anyscale.shared_anyscale_utils.latest_ray_version import LATEST_RAY_VERSION
 from anyscale.util import (
@@ -357,18 +367,29 @@ class AnyscaleClient(AnyscaleClientInterface):
     def _get_project_id_by_name(
         self, *, parent_cloud_id: Optional[str] = None, name: Optional[str] = None
     ) -> str:
+        """Resolves `name`, a project name, to a project ID.
+
+        Args:
+            parent_cloud_id: If specified, return the project that has this `parent_cloud_id`. \
+                Else (`None`), return the first project with the given `name`. \
+                Defaults to `None`.
+            name: The name of the project.
+        """
         # Find if project with name already exists
-        existing_projects = self._internal_api_client.find_project_by_project_name_api_v2_projects_find_by_name_get(
+        matching_projects = self._internal_api_client.find_project_by_project_name_api_v2_projects_find_by_name_get(
             name
         ).results
-        if len(existing_projects) == 0:
+        if len(matching_projects) == 0:
             raise ValueError(f"Project '{name}' was not found.")
         else:
-            for project in existing_projects:
-                if project.parent_cloud_id == parent_cloud_id:
+            for project in matching_projects:
+                if (
+                    parent_cloud_id is None
+                    or project.parent_cloud_id == parent_cloud_id
+                ):
                     return project.id
             raise ValueError(
-                f"{len(existing_projects)} project(s) found with name '{name}' and none matched cloud_id '{parent_cloud_id}'"
+                f"{len(matching_projects)} project(s) found with name '{name}' and none matched cloud_id '{parent_cloud_id}'"
             )
 
     def _get_project_id_by_cloud_id(
@@ -1110,29 +1131,94 @@ class AnyscaleClient(AnyscaleClientInterface):
     def upload_dataset(
         self,
         dataset_file: str,
-        existing_dataset_id_or_name: Optional[str],
+        name: Optional[str],
         description: Optional[str],
-        cloud_id: Optional[str],
-        project_id: Optional[str],
-    ):
+        cloud: Optional[str],
+        project: Optional[str],
+    ) -> Dataset:
+        # Resolve `~/.../file` to `/home/user/.../file`
+        dataset_file = os.path.expanduser(dataset_file)
+
         if not os.path.isfile(dataset_file):
             raise ValueError(f"Path '{dataset_file}' is not a valid file.")
+        dataset_file_size = os.path.getsize(dataset_file)
+        if dataset_file_size > 5 * Bytes.GB:
+            raise ValueError(
+                f"File '{dataset_file}' is too large to upload. The maximum size is 5 GB."
+            )
+        project_id = self._get_project_id_by_name(name=project) if project else None
+        cloud_id = self.get_cloud_id(cloud_name=cloud) if cloud else None
 
-        cloud_id, project_id = source_cloud_id_and_project_id(
-            internal_api=self._internal_api_client,
-            external_api=self._external_api_client,
-            cloud_id=cloud_id,
-            project_id=project_id,
-        )
-        print(existing_dataset_id_or_name, description)
+        with FileDownloadProgress() as progress:
+            task_id = progress.add_task(
+                description=f"Creating an upload request for '{dataset_file}'",
+                total=dataset_file_size,
+            )
+            _, project_id = source_cloud_id_and_project_id(
+                internal_api=self._internal_api_client,
+                external_api=self._external_api_client,
+                cloud_id=cloud_id,
+                project_id=project_id,
+            )
+            dataset_upload: DatasetUpload = self._internal_api_client.create_dataset_upload_api_v2_datasets_upload_post(
+                create_dataset=CreateDataset(
+                    filename=os.path.basename(dataset_file),
+                    description=description,
+                    name=name,
+                    project_id=project_id,
+                )
+            ).result
+
+            progress.update(task_id, description=f"Uploading '{dataset_file}'")
+
+            with open(dataset_file, "rb") as file_reader:
+                progress_reader = ProgressFileReader(file_reader, progress, task_id)
+                response = requests.put(
+                    dataset_upload.upload_url, data=progress_reader,
+                )
+                response.raise_for_status()
+
+            progress.update(task_id, completed=os.path.getsize(dataset_file))
+            progress.console.print(
+                "Upload complete!", style=Style(bold=True, color="green")
+            )
+        return dataset_upload.dataset
 
     def download_dataset(
-        self, dataset_id_or_name: str, version: Optional[int]  # noqa: ARG002
+        self, name: str, version: Optional[int], project: Optional[str]
     ) -> bytes:
-        # TODO: replace with self._external_api_client.get_dataset_download_url()
-        dataset_bytes_url = "https://example.com/dataset_bytes"
-        response = requests.get(dataset_bytes_url)
-        dataset_bytes = response.content
+        if project:
+            project_id = self._get_project_id_by_name(name=project)
+        else:
+            _, project_id = source_cloud_id_and_project_id(
+                internal_api=self._internal_api_client,
+                external_api=self._external_api_client,
+                cloud_id=None,
+                project_id=None,
+            )
+        with FileDownloadProgress() as progress:
+            task_id = progress.add_task(
+                description=f"Getting download info for '{name}'",
+            )
+            download_url: str = self._internal_api_client.get_dataset_download_url_api_v2_datasets_download_get(
+                name, version=version, project_id=project_id,
+            )
+            progress.update(task_id, description=f"Downloading '{name}'")
+            response = requests.get(download_url, stream=True)
+            total_size = int(response.headers.get("content-length", 0))
+            progress.update(task_id, total=total_size)
+
+            # For CLI, consider writing to disk instead of loading the entire file into memory.
+            dataset_bytes = b""
+            for data in response.iter_content(Bytes.MB):
+                dataset_bytes += data
+                progress.update(task_id, advance=len(data))
+
+            progress.update(task_id, completed=total_size)
+            progress.console.print(
+                "Download complete!", style=Style(bold=True, color="green")
+            )
+
         return dataset_bytes
 
     def get_finetuned_model(
@@ -1193,6 +1279,40 @@ class AnyscaleClient(AnyscaleClientInterface):
             workspace = resp.results[0]
             return workspace
 
+    @handle_api_exceptions
+    def update_workspace(
+        self,
+        *,
+        workspace_id: str,
+        name: Optional[str] = None,
+        compute_config_id: Optional[str] = None,
+        cluster_environment_build_id: Optional[str] = None,
+        idle_timeout_minutes: Optional[int] = None,
+    ):
+        workspace = self.get_workspace(id=workspace_id)
+        if not workspace:
+            raise ValueError(f"Workspace '{workspace_id}' not found.")
+
+        if name:
+            # Update the workspace name with workspaces patch API
+            self._internal_api_client.patch_workspace_api_v2_experimental_workspaces_workspace_id_patch(
+                workspace_id=workspace_id,
+                json_patch_operation=[
+                    {"op": "replace", "path": "/name", "value": name,},
+                ],
+            )
+
+        if compute_config_id or cluster_environment_build_id or idle_timeout_minutes:
+            # Update cluster with external cluster API
+            self._external_api_client.update_cluster(
+                cluster_id=workspace.cluster_id,
+                update_cluster=UpdateCluster(
+                    idle_timeout_minutes=idle_timeout_minutes,
+                    cluster_environment_build_id=cluster_environment_build_id,
+                    cluster_compute_id=compute_config_id,
+                ),
+            )
+
     @handle_api_exceptions
     def update_workspace_dependencies_offline_only(
         self, workspace_id: str, requirements: List[str]

{anyscale-0.24.62 → anyscale-0.24.64}/anyscale/_private/anyscale_client/common.py

@@ -7,6 +7,7 @@ from anyscale.client.openapi_client.models import (
     ComputeTemplateConfig,
     CreateExperimentalWorkspace,
     CreateInternalProductionJob,
+    Dataset,
     DecoratedComputeTemplate,
     DeletedPlatformFineTunedModel,
     FineTunedModel,
@@ -411,17 +412,17 @@ class AnyscaleClientInterface(ABC):
     def upload_dataset(
         self,
         dataset_file: str,
-        existing_dataset_id_or_name: Optional[str],
+        name: Optional[str],
         description: Optional[str],
-        cloud_id: Optional[str],
-        project_id: Optional[str],
-    ):
+        cloud: Optional[str],
+        project: Optional[str],
+    ) -> Dataset:
         """See docstring for `anyscale.llm.dataset.upload`."""
         raise NotImplementedError
 
     @abstractmethod
     def download_dataset(
-        self, dataset_id_or_name: str, version: Optional[int]
+        self, name: str, version: Optional[int], project: Optional[str]
     ) -> bytes:
         """See docstring for `anyscale.llm.dataset.download`."""
 
@@ -509,3 +510,16 @@ class AnyscaleClientInterface(ABC):
         self, cloud_id: Optional[str], project_id: Optional[str], max_items: int,
     ) -> List[FineTunedModel]:
         raise NotImplementedError
+
+    @abstractmethod
+    def update_workspace(
+        self,
+        *,
+        workspace_id: str,
+        name: Optional[str] = None,
+        compute_config_id: Optional[str] = None,
+        cluster_environment_build_id: Optional[str] = None,
+        idle_timeout_minutes: Optional[int] = None,
+    ):
+        """Update a workspace."""
+        raise NotImplementedError

{anyscale-0.24.62 → anyscale-0.24.64}/anyscale/_private/anyscale_client/fake_anyscale_client.py

@@ -15,6 +15,7 @@ from anyscale.client.openapi_client.models import (
     ComputeTemplateConfig,
     CreateExperimentalWorkspace,
     CreateInternalProductionJob,
+    Dataset,
     DecoratedComputeTemplate,
     DeletedPlatformFineTunedModel,
     ExperimentalWorkspace,
@@ -821,15 +822,15 @@ class FakeAnyscaleClient(AnyscaleClientInterface):
     def upload_dataset(
         self,
         dataset_file: str,  # noqa: ARG002
-        existing_dataset_id_or_name: Optional[str],  # noqa: ARG002
+        name: Optional[str],  # noqa: ARG002
         description: Optional[str],  # noqa: ARG002
-        cloud_id: Optional[str],  # noqa: ARG002
-        project_id: Optional[str],  # noqa: ARG002
-    ):
+        cloud: Optional[str],  # noqa: ARG002
+        project: Optional[str],  # noqa: ARG002
+    ) -> Dataset:
         pass
 
     def download_dataset(
-        self, dataset_id_or_name: str, version: Optional[int]  # noqa: ARG002
+        self, name: str, version: Optional[int], project: Optional[str]  # noqa: ARG002
     ) -> bytes:
         return b""
 
@@ -968,3 +969,33 @@ class FakeAnyscaleClient(AnyscaleClientInterface):
             )
             for _ in range(max_items)
         ]
+
+    def update_workspace(
+        self,
+        *,
+        workspace_id: str,
+        name: Optional[str] = None,
+        compute_config_id: Optional[str] = None,
+        cluster_environment_build_id: Optional[str] = None,
+        idle_timeout_minutes: Optional[int] = None,
+    ):
+        if workspace_id not in self._workspaces:
+            raise ValueError(f"Workspace '{workspace_id}' not found.")
+
+        workspace = self._workspaces[workspace_id]
+
+        if name:
+            workspace.name = name
+
+        if compute_config_id:
+            workspace.compute_config_id = compute_config_id
+
+        if cluster_environment_build_id:
+            workspace.environment_id = cluster_environment_build_id
+
+        if idle_timeout_minutes:
+            workspace.idle_timeout_minutes = idle_timeout_minutes
+
+            compute_config = self.get_compute_config(workspace.compute_config_id)
+            assert compute_config is not None
+            compute_config.idle_timeout_minutes = idle_timeout_minutes

{anyscale-0.24.62 → anyscale-0.24.64}/anyscale/_private/docgen/__main__.py

@@ -14,6 +14,7 @@ from anyscale.commands import (
     image_commands,
     job_commands,
     logs_commands,
+    machine_pool_commands,
     schedule_commands,
     service_account_commands,
     service_commands,
@@ -21,6 +22,7 @@ from anyscale.commands import (
     workspace_commands_v2,
 )
 from anyscale.compute_config.models import (
+    CloudDeployment,
     ComputeConfig,
     ComputeConfigVersion,
     HeadNodeConfig,
@@ -87,14 +89,47 @@ ALL_MODULES = [
             JobRunState,
             JobLogMode,
         ],
+        # The following commands are legacy
+        legacy_sdk_commands={
+            "create_job": anyscale.job.submit,
+            "get_production_job": anyscale.job.status,
+            "get_session_for_job": anyscale.job.status,
+            "terminate_job": anyscale.job.terminate,
+            "fetch_job_logs": anyscale.job.get_logs,
+            "fetch_production_job_logs": anyscale.job.get_logs,
+            "get_job_logs_download": anyscale.job.get_logs,
+            "get_job_logs_stream": anyscale.job.get_logs,
+        },
+        legacy_sdk_models=[
+            "BaseJobStatus",
+            "CreateClusterComputeConfig",
+            "CreateJobQueueConfig",
+            "CreateProductionJob",
+            "CreateProductionJobConfig",
+            "HaJobGoalStates",
+            "HaJobStates",
+            "Job",
+            "JobListResponse",
+            "JobQueueConfig",
+            "JobQueueExecutionMode",
+            "JobQueueSpec",
+            "JobRunType",
+            "JobStatus",
+            "JobsSortField",
+            "ProductionJob",
+            "ProductionJobConfig",
+            "ProductionJobStateTransition",
+            "ProductionjobListResponse",
+            "ProductionjobResponse",
+            "RayRuntimeEnvConfig",
+            "SortByClauseJobsSortField",
+        ],
     ),
     Module(
         title="Schedule",
         filename="schedule-api.md",
         cli_prefix="anyscale schedule",
         cli_commands=[
-            schedule_commands.create,
-            schedule_commands.update,
             schedule_commands.apply,
             schedule_commands.list,
             schedule_commands.pause,
@@ -111,6 +146,15 @@ ALL_MODULES = [
             anyscale.schedule.trigger,
         ],
         models=[ScheduleConfig, ScheduleState, ScheduleStatus],
+        # The following commands are legacy
+        legacy_sdk_commands={
+            "create_or_update_schedule": anyscale.schedule.apply,
+            "get_schedule": anyscale.schedule.status,
+            "pause_schedule": anyscale.schedule.set_state,
+            "run_schedule": anyscale.schedule.trigger,
+        },
+        legacy_cli_prefix="anyscale schedule",
+        legacy_cli_commands={schedule_commands.create, schedule_commands.update,},
     ),
     Module(
         title="Service",
@@ -161,6 +205,7 @@ ALL_MODULES = [
             HeadNodeConfig,
             WorkerNodeGroupConfig,
             MarketType,
+            CloudDeployment,
             ComputeConfigVersion,
         ],
     ),
@@ -253,6 +298,19 @@ ALL_MODULES = [
             workspace_commands.push,
         ],
     ),
+    Module(
+        title="Machine Pool",
+        filename="machine-pool.md",
+        cli_prefix="anyscale machine-pool",
+        cli_commands=[
+            machine_pool_commands.create_machine_pool,
+            machine_pool_commands.delete_machine_pool,
+            machine_pool_commands.list_machine_pools,
+        ],
+        sdk_prefix="anyscale.machine_pool",
+        sdk_commands=[],
+        models=[],
+    ),
 ]