anyscale 0.24.36__tar.gz → 0.24.38__tar.gz

{anyscale-0.24.36 → anyscale-0.24.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anyscale
-Version: 0.24.36
+Version: 0.24.38
 Summary: Command Line Interface for Anyscale
 Author: Anyscale Inc.
 License: AS License

anyscale-0.24.38/anyscale/__init__.py

@@ -0,0 +1,105 @@
+import inspect
+import logging
+import os
+from sys import path
+from typing import Any, Dict, List, Optional
+
+from anyscale import version
+
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler())
+logger.setLevel(os.environ.get("ANYSCALE_LOGLEVEL", "WARN"))
+
+anyscale_dir = os.path.dirname(os.path.abspath(__file__))
+path.append(os.path.join(anyscale_dir, "client"))
+path.append(os.path.join(anyscale_dir, "sdk"))
+
+import anyscale
+from anyscale import compute_config, image, integrations, job, service
+from anyscale._private.anyscale_client import AnyscaleClient, AnyscaleClientInterface
+from anyscale._private.sdk.base_sdk import Timer
+from anyscale.authenticate import AuthenticationBlock
+from anyscale.cli_logger import BlockLogger
+from anyscale.cluster import get_job_submission_client_cluster_info
+from anyscale.cluster_compute import get_cluster_compute_from_name
+from anyscale.compute_config import ComputeConfigSDK
+from anyscale.connect import ClientBuilder
+from anyscale.job import JobSDK
+from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK
+from anyscale.service import ServiceSDK
+
+
+# Note: indentation here matches that of connect.py::ClientBuilder.
+BUILDER_HELP_FOOTER = """
+        See ``anyscale.ClientBuilder`` for full documentation of
+        this experimental feature."""
+
+# Auto-add all Anyscale connect builder functions to the top-level.
+for attr, _ in inspect.getmembers(ClientBuilder, inspect.isfunction):
+    if attr.startswith("_"):
+        continue
+
+    def _new_builder(attr: str) -> Any:
+        target = getattr(ClientBuilder, attr)
+
+        def new_session_builder(*a: List[Any], **kw: Dict[str, Any]) -> Any:
+            builder = ClientBuilder()
+            return target(builder, *a, **kw)
+
+        new_session_builder.__name__ = attr
+        new_session_builder.__doc__ = target.__doc__ + BUILDER_HELP_FOOTER
+        new_session_builder.__signature__ = inspect.signature(target)  # type: ignore
+
+        return new_session_builder
+
+    globals()[attr] = _new_builder(attr)
+
+__version__ = version.__version__
+
+ANYSCALE_ENV = os.environ.copy()
+
+
+class Anyscale:
+    def __init__(
+        self, *, auth_token: Optional[str] = None, _host: Optional[str] = None,
+    ):
+        auth_block = AuthenticationBlock(
+            cli_token=auth_token, host=_host, raise_structured_exception=True,
+        )
+        self._anyscale_client = AnyscaleClient(
+            api_clients=(auth_block.anyscale_api_client, auth_block.api_client),
+            host=_host,
+        )
+        self._job_sdk = JobSDK(client=self._anyscale_client)
+        self._service_sdk = ServiceSDK(client=self._anyscale_client)
+        self._compute_config_sdk = ComputeConfigSDK(client=self._anyscale_client)
+
+    @classmethod
+    def _init_private(
+        cls, *, client: AnyscaleClientInterface, logger: BlockLogger, timer: Timer
+    ):
+        # Private constructor used to inject fakes for testing.
+        obj = cls.__new__(cls)
+        super(Anyscale, obj).__init__()
+        obj._anyscale_client = client  # noqa: SLF001
+        obj._job_sdk = JobSDK(client=client, logger=logger, timer=timer)  # noqa: SLF001
+        obj._service_sdk = ServiceSDK(  # noqa: SLF001
+            client=client, logger=logger, timer=timer
+        )
+        obj._compute_config_sdk = ComputeConfigSDK(  # noqa: SLF001
+            client=client, logger=logger, timer=timer
+        )
+        return obj
+
+    @property
+    def job(self) -> JobSDK:  # noqa: F811
+        return self._job_sdk
+
+    @property
+    def service(self) -> ServiceSDK:  # noqa: F811
+        return self._service_sdk
+
+    @property
+    def compute_config(self) -> ComputeConfigSDK:  # noqa: F811
+        return self._compute_config_sdk

{anyscale-0.24.36 → anyscale-0.24.38}/anyscale/_private/anyscale_client/anyscale_client.py

@@ -1,10 +1,11 @@
 from abc import ABC, abstractmethod
+import asyncio
 import io
 import logging
 import os
 import pathlib
 import time
-from typing import Any, Callable, Dict, IO, List, Optional, Tuple
+from typing import Any, Callable, cast, Dict, Generator, IO, List, Optional, Tuple
 from urllib.parse import urlparse
 
 import requests
@@ -16,8 +17,9 @@ from anyscale._private.anyscale_client.common import (
     RUNTIME_ENV_PACKAGE_FORMAT,
 )
 from anyscale._private.models.image_uri import ImageURI
+from anyscale.api_utils.job_logs_util import _get_job_logs_from_storage_bucket_streaming
 from anyscale.authenticate import AuthenticationBlock, get_auth_api_client
-from anyscale.cli_logger import BlockLogger
+from anyscale.cli_logger import BlockLogger, LogsLogger
 from anyscale.client.openapi_client.api.default_api import DefaultApi as InternalApi
 from anyscale.client.openapi_client.models import (
     ArchiveStatus,
@@ -45,17 +47,16 @@ from anyscale.sdk.anyscale_client.models import (
     ClusterComputeConfig,
     ClusterEnvironment,
     ClusterEnvironmentBuild,
+    ClusterenvironmentbuildListResponse,
     ClusterEnvironmentBuildStatus,
+    ClusterEnvironmentsQuery,
+    CreateClusterEnvironment,
+    CreateClusterEnvironmentBuild,
     Job as APIJobRun,
     Project,
     RollbackServiceModel,
     ServiceModel,
-)
-from anyscale.sdk.anyscale_client.models.create_cluster_environment import (
-    CreateClusterEnvironment,
-)
-from anyscale.sdk.anyscale_client.models.create_cluster_environment_build import (
-    CreateClusterEnvironmentBuild,
+    TextQuery,
 )
 from anyscale.sdk.anyscale_client.models.jobs_query import JobsQuery
 from anyscale.sdk.anyscale_client.models.jobs_sort_field import JobsSortField
@@ -154,7 +155,7 @@ class AnyscaleClient(AnyscaleClientInterface):
         self._gcs_client = gcs_client
 
         # Cached IDs and models to avoid duplicate lookups.
-        self._default_project_id_from_cloud_id: Dict[str, str] = {}
+        self._default_project_id_from_cloud_id: Dict[Optional[str], str] = {}
         self._cloud_id_cache: Dict[Optional[str], str] = {}
         self._current_workspace_cluster: Optional[Cluster] = None
         self._logger = logger or BlockLogger()
@@ -322,10 +323,43 @@ class AnyscaleClient(AnyscaleClientInterface):
 
         return self._current_workspace_cluster
 
-    def get_project_id(self, parent_cloud_id: str) -> str:
+    def _get_project_id_by_name(
+        self, *, parent_cloud_id: Optional[str] = None, name: Optional[str] = None
+    ) -> str:
+        # Find if project with name already exists
+        existing_projects = self._internal_api_client.find_project_by_project_name_api_v2_projects_find_by_name_get(
+            name
+        ).results
+        if len(existing_projects) == 0:
+            raise ValueError(f"Project '{name}' was not found.")
+        else:
+            for project in existing_projects:
+                if project.parent_cloud_id == parent_cloud_id:
+                    return project.id
+            raise ValueError(
+                f"{len(existing_projects)} project(s) found with name '{name}' and none matched cloud_id '{parent_cloud_id}'"
+            )
+
+    def _get_project_id_by_cloud_id(
+        self, *, parent_cloud_id: Optional[str] = None,
+    ) -> str:
         workspace_cluster = self.get_current_workspace_cluster()
         if workspace_cluster is not None:
-            return workspace_cluster.project_id
+            if (
+                workspace_cluster.cluster_compute_config is not None
+                and workspace_cluster.cluster_compute_config.cloud_id == parent_cloud_id
+            ):
+                return workspace_cluster.project_id
+            elif workspace_cluster.cluster_compute_id is not None:
+                workspace_cluster_compute = self.get_compute_config(
+                    workspace_cluster.cluster_compute_id
+                )
+                if (
+                    workspace_cluster_compute is not None
+                    and workspace_cluster_compute.config is not None
+                    and workspace_cluster_compute.config.cloud_id == parent_cloud_id
+                ):
+                    return workspace_cluster.project_id
 
         if self._default_project_id_from_cloud_id.get(parent_cloud_id) is None:
             # Cloud isolation organizations follow the permissions model in https://docs.anyscale.com/organization-and-user-account/access-controls
@@ -340,6 +374,16 @@ class AnyscaleClient(AnyscaleClientInterface):
 
         return self._default_project_id_from_cloud_id[parent_cloud_id]
 
+    def get_project_id(
+        self, *, parent_cloud_id: Optional[str] = None, name: Optional[str] = None
+    ) -> str:
+        if name is not None:
+            return self._get_project_id_by_name(
+                parent_cloud_id=parent_cloud_id, name=name
+            )
+        else:
+            return self._get_project_id_by_cloud_id(parent_cloud_id=parent_cloud_id)
+
     def _get_cloud_id_for_compute_config_id(self, compute_config_id: str) -> str:
         cluster_compute: ClusterCompute = self._external_api_client.get_cluster_compute(
             compute_config_id
@@ -556,15 +600,35 @@ class AnyscaleClient(AnyscaleClientInterface):
         ).result
         return result.id
 
-    def _get_cluster_env_by_name(self, name: str) -> Optional[ClusterEnvironment]:
-        cluster_envs = self._external_api_client.search_cluster_environments(
-            {
-                "name": {"equals": name},
-                "paging": {"count": 1},
-                "include_anonymous": True,
-            }
-        ).results
-        return cluster_envs[0] if cluster_envs else None
+    def get_cluster_env_by_name(self, name: str) -> Optional[ClusterEnvironment]:
+        resp = self._external_api_client.search_cluster_environments(
+            ClusterEnvironmentsQuery(
+                name=TextQuery(equals=name),  # pyright: ignore reportGeneralTypeIssues
+                paging=PageQuery(count=1),
+                include_anonymous=True,
+            )
+        )
+        if resp.results:
+            return resp.results[0]
+        return None
+
+    def list_cluster_env_builds(
+        self, cluster_env_id: str,
+    ) -> Generator[ClusterEnvironmentBuild, None, None]:
+        paging_token = None
+        while True:
+            resp: ClusterenvironmentbuildListResponse = self._external_api_client.list_cluster_environment_builds(
+                cluster_environment_id=cluster_env_id,
+                count=self.LIST_ENDPOINT_COUNT,
+                paging_token=paging_token,
+                desc=True,
+            )
+            for build in resp.results:
+                yield build
+
+            if resp.metadata.next_paging_token is None:
+                break
+            paging_token = resp.metadata.next_paging_token
 
     def _wait_for_build_to_succeed(
         self, build_id: str, poll_interval_seconds=3, timeout_secs=3600,
@@ -599,7 +663,7 @@ class AnyscaleClient(AnyscaleClientInterface):
     def _find_or_create_cluster_env(
         self, cluster_env_name: str, anonymous: bool
     ) -> ClusterEnvironment:
-        existing_cluster_env = self._get_cluster_env_by_name(cluster_env_name)
+        existing_cluster_env = self.get_cluster_env_by_name(cluster_env_name)
         if existing_cluster_env is not None:
             return existing_cluster_env
 
@@ -631,11 +695,18 @@ class AnyscaleClient(AnyscaleClientInterface):
             ):
                 return build.id
 
-        build_op = self._external_api_client.create_cluster_environment_build(
-            CreateClusterEnvironmentBuild(
-                cluster_environment_id=cluster_env.id, containerfile=containerfile,
-            )
-        ).result
+        try:
+            build_op = self._external_api_client.create_cluster_environment_build(
+                CreateClusterEnvironmentBuild(
+                    cluster_environment_id=cluster_env.id, containerfile=containerfile,
+                )
+            ).result
+        except ExternalApiException as e:
+            if e.status == 400:
+                raise RuntimeError(
+                    "Invalid containerfile. Please check the syntax and try again.",
+                    e.body,
+                ) from None
 
         build_url = self.get_build_ui_url(
             cluster_env.id, build_op.cluster_environment_build_id
@@ -646,7 +717,9 @@ class AnyscaleClient(AnyscaleClientInterface):
 
         return build_op.cluster_environment_build_id
 
-    def get_cluster_env_build_id_from_image_uri(self, image_uri: ImageURI) -> str:
+    def get_cluster_env_build_id_from_image_uri(
+        self, image_uri: ImageURI, registry_login_secret: Optional[str] = None
+    ) -> str:
         if image_uri.is_cluster_env_image():
             identifier = image_uri.to_cluster_env_identifier()
             try:
@@ -677,6 +750,7 @@ class AnyscaleClient(AnyscaleClientInterface):
         if (
             build is not None
             and build.docker_image_name == image_uri_str
+            and build.registry_login_secret == registry_login_secret
             and build.status == ClusterEnvironmentBuildStatus.SUCCEEDED
         ):
             return build.id
@@ -687,6 +761,7 @@ class AnyscaleClient(AnyscaleClientInterface):
                 # For historical reasons, we have to use docker_image_name instead of image_uri; but it is just a URI to the image.
                 cluster_environment_id=cluster_env.id,
                 docker_image_name=image_uri_str,
+                registry_login_secret=registry_login_secret,
             )
         ).result
 
@@ -733,8 +808,18 @@ class AnyscaleClient(AnyscaleClientInterface):
 
         return service
 
+    def get_project(self, project_id: str) -> Optional[Project]:
+        return self._internal_api_client.get_project_api_v2_projects_project_id_get(
+            project_id
+        ).result
+
     def get_job(
-        self, name: Optional[str], job_id: Optional[str]
+        self,
+        *,
+        name: Optional[str],
+        job_id: Optional[str],
+        cloud: Optional[str],
+        project: Optional[str],
     ) -> Optional[ProductionJob]:
         if job_id is not None:
             try:
@@ -745,8 +830,8 @@ class AnyscaleClient(AnyscaleClientInterface):
                 raise e from None
         else:
             paging_token = None
-            cloud_id = self.get_cloud_id()
-            project_id = self.get_project_id(parent_cloud_id=cloud_id)
+            cloud_id = self.get_cloud_id(cloud_name=cloud)
+            project_id = self.get_project_id(parent_cloud_id=cloud_id, name=project)
             result: Optional[ProductionJob] = None
             while True:
                 resp = self._external_api_client.list_production_jobs(
@@ -853,3 +938,25 @@ class AnyscaleClient(AnyscaleClientInterface):
                 )
 
         return info.file_uri
+
+    def logs_for_job_run(self, job_run_id: str) -> str:
+        """Retrieves logs from the streaming job logs folder in S3/GCS"""
+        logs = ""
+        next_page_token = None
+        cluster_journal_events_start_line = 0
+
+        (logs, next_page_token, cluster_journal_events_start_line, _,) = asyncio.run(
+            _get_job_logs_from_storage_bucket_streaming(
+                self._internal_api_client,
+                cast(
+                    LogsLogger, self.logger
+                ),  # self.logger is a BlockLogger, cast to LogsLogger subclass
+                job_run_id=job_run_id,
+                remove_escape_chars=False,
+                next_page_token=next_page_token,
+                cluster_journal_events_start_line=cluster_journal_events_start_line,
+                no_cluster_journal_events=True,
+            )
+        )
+
+        return logs

{anyscale-0.24.36 → anyscale-0.24.38}/anyscale/_private/anyscale_client/common.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, Generator, List, Optional, Tuple
 
 from anyscale._private.models.image_uri import ImageURI
 from anyscale.client.openapi_client.models import (
@@ -8,12 +8,14 @@ from anyscale.client.openapi_client.models import (
     CreateInternalProductionJob,
     DecoratedComputeTemplate,
     InternalProductionJob,
+    Project,
 )
 from anyscale.client.openapi_client.models.production_job import ProductionJob
 from anyscale.sdk.anyscale_client.models import (
     ApplyServiceModel,
     Cluster,
     ClusterCompute,
+    ClusterEnvironment,
     Job as APIJobRun,
     ServiceModel,
 )
@@ -92,8 +94,17 @@ class AnyscaleClientInterface(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def get_project_id(self, parent_cloud_id: str) -> str:
-        """Get the default project ID for this user.
+    def get_project_id(
+        self, *, parent_cloud_id: Optional[str] = None, name: Optional[str] = None
+    ) -> str:
+        """Get the project ID.
+
+        If name is passed, projects with that name will be checked for a project with
+        a cloud matching parent_cloud_id. If a match is found the cloud id is returned,
+        otherwise a ValueError is raised.
+
+        If name is not passed, but parent_cloud_id is, the default project for that cloud
+        will be used.
 
         If running in a workspace, returns the workspace project ID.
         """
@@ -174,6 +185,23 @@ class AnyscaleClientInterface(ABC):
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def get_cluster_env_by_name(self, name: str) -> Optional[ClusterEnvironment]:
+        """Get a cluster environment by its name.
+
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def list_cluster_env_builds(
+        self, cluster_env_id: str
+    ) -> Generator[ClusterEnvironmentBuild, None, None]:
+        """List cluster environment builds for the provided cluster environment id.
+
+        Returns a list of cluster environment builds.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def get_cluster_env_build(self, build_id: str) -> Optional[ClusterEnvironmentBuild]:
         """Get the cluster env build.
@@ -193,7 +221,7 @@ class AnyscaleClientInterface(ABC):
 
     @abstractmethod
     def get_cluster_env_build_id_from_image_uri(
-        self, image_uri: ImageURI,
+        self, image_uri: ImageURI, registry_login_secret: Optional[str] = None
     ):
         """Get the cluster environment build ID for the cluster environment with provided image_uri.
 
@@ -226,11 +254,24 @@ class AnyscaleClientInterface(ABC):
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def get_project(self, project_id: str) -> Optional[Project]:
+        """Get a project by id.
+
+        Returns None if not found.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def get_job(
-        self, name: Optional[str], job_id: Optional[str]
+        self,
+        *,
+        name: Optional[str],
+        job_id: Optional[str],
+        cloud: Optional[str],
+        project: Optional[str],
     ) -> Optional[ProductionJob]:
-        """Get a job by name.
+        """Get a job by either name or id. Filter by cloud and project.
 
         Returns None if not found.
         """
@@ -289,7 +330,7 @@ class AnyscaleClientInterface(ABC):
         *,
         cloud_id: str,
         excludes: Optional[List[str]] = None,
-        overwrite_existing_file: bool = False
+        overwrite_existing_file: bool = False,
     ) -> str:
         """Upload the provided directory to cloud storage and return a URI for it.
 
@@ -302,3 +343,11 @@ class AnyscaleClientInterface(ABC):
         default if the target file URI already exists it will not be overwritten.
         """
         raise NotImplementedError
+
+    @abstractmethod
+    def logs_for_job_run(self, job_run_id: str) -> str:
+        """Returns the logs associated with a particular job run.
+
+        The job_run_id is the backend id for the job run.
+        """
+        raise NotImplementedError