anyscale 0.24.14__tar.gz → 0.24.16__tar.gz

{anyscale-0.24.14 → anyscale-0.24.16}/BUILD.bazel

@@ -56,7 +56,6 @@ py_library(
         requirement("tqdm"),
         requirement("humanize"),
         requirement("tzlocal"),
-        requirement("argon2-cffi"),
         requirement("google-cloud-compute"),
         requirement("google-cloud-resource-manager"),
         requirement("google-api-python-client"),

{anyscale-0.24.14 → anyscale-0.24.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anyscale
-Version: 0.24.14
+Version: 0.24.16
 Summary: Command Line Interface for Anyscale
 Author: Anyscale Inc.
 License: AS License

{anyscale-0.24.14 → anyscale-0.24.16}/anyscale/__init__.py

@@ -4,7 +4,7 @@ import os
 from sys import path
 from typing import Any, Dict, List
 
-from anyscale import snapshot_util, version
+from anyscale import version
 
 
 logger = logging.getLogger(__name__)
@@ -15,7 +15,7 @@ anyscale_dir = os.path.dirname(os.path.abspath(__file__))
 path.append(os.path.join(anyscale_dir, "client"))
 path.append(os.path.join(anyscale_dir, "sdk"))
 
-from anyscale import integrations
+from anyscale import compute_config, integrations, job, service
 from anyscale.cluster import get_job_submission_client_cluster_info
 from anyscale.cluster_compute import get_cluster_compute_from_name
 from anyscale.connect import ClientBuilder

{anyscale-0.24.14 → anyscale-0.24.16}/anyscale/_private/anyscale_client/anyscale_client.py

@@ -194,7 +194,7 @@ class AnyscaleClient(AnyscaleClientInterface):
 
     def get_build_ui_url(self, cluster_env_id, build_id: str) -> str:
         return get_endpoint(
-            f"/container-images/{cluster_env_id}/versions/{build_id}", host=self.host
+            f"v2/container-images/{cluster_env_id}/versions/{build_id}", host=self.host
         )
 
     def get_current_workspace_id(self) -> Optional[str]:
@@ -239,7 +239,6 @@ class AnyscaleClient(AnyscaleClientInterface):
             )
         try:
             fileobj = io.BytesIO()
-            print("downloading", bucket, object_key)
             self.s3_client.download_fileobj(bucket, object_key, fileobj)
             return fileobj.getvalue()
         except ClientError as e:
@@ -268,9 +267,7 @@ class AnyscaleClient(AnyscaleClientInterface):
         workspace_id = os.environ.get("ANYSCALE_EXPERIMENTAL_WORKSPACE_ID", "")
         workspace_artifact_path = (
             os.path.join(
-                system_storage_path,
-                ".anyscale/workspace_tracking_requirements",
-                workspace_id,
+                system_storage_path, "workspace_tracking_dependencies", workspace_id,
             )
             if workspace_id and system_storage_path
             else ""
@@ -286,7 +283,12 @@ class AnyscaleClient(AnyscaleClientInterface):
 
         dot_env = self._download_file_from_remote_storage(workspace_dot_env_path)
         if dot_env:
-            return parse_dot_env_file(dot_env)
+            parsed_dot_env = parse_dot_env_file(dot_env)
+            if parsed_dot_env:
+                self.logger.info(
+                    f"Using workspace runtime dependencies env vars: {parsed_dot_env}."
+                )
+            return parsed_dot_env
         return None
 
     def get_current_workspace_cluster(self) -> Optional[Cluster]:
@@ -514,7 +516,6 @@ class AnyscaleClient(AnyscaleClientInterface):
         self, cluster_env_name: str, base_build_id: str, requirements: List[str]
     ) -> str:
         if requirements:
-            self.logger.info(f"Building image with requirements: {requirements}.")
             base_build = self._external_api_client.get_cluster_environment_build(
                 base_build_id
             ).result
@@ -522,6 +523,7 @@ class AnyscaleClient(AnyscaleClientInterface):
                 base_build.status == ClusterEnvironmentBuildStatus.SUCCEEDED
                 and base_build.docker_image_name
             ):
+                self.logger.info(f"Using tracked python packages: {requirements}")
                 lines = [
                     "# syntax=docker/dockerfile:1",
                     f"FROM {base_build.docker_image_name}",
@@ -570,6 +572,7 @@ class AnyscaleClient(AnyscaleClientInterface):
                 build_id
             ).result
             if build.status == ClusterEnvironmentBuildStatus.SUCCEEDED:
+                self.logger.info("")
                 return
             elif build.status == ClusterEnvironmentBuildStatus.FAILED:
                 raise RuntimeError(f"Image build {build_id} failed.")
@@ -578,7 +581,8 @@ class AnyscaleClient(AnyscaleClientInterface):
 
             elapsed_secs += poll_interval_seconds
             self.logger.info(
-                f"Waiting for image build to complete. Elapsed time: {elapsed_secs} seconds",
+                f"Waiting for image build to complete. Elapsed time: {elapsed_secs} seconds.",
+                end="\r",
             )
             self._sleep(poll_interval_seconds)
         raise TimeoutError(

anyscale-0.24.16/anyscale/_private/docgen/README.md

@@ -0,0 +1,15 @@
+Tool to autogenerate documentation for the Anyscale SDK/CLI that can be embedded in a [Docusarus](https://docusaurus.io/) site.
+
+This is used by: https://github.com/anyscale/endpoint-docs.
+
+Usage:
+```bash
+Usage: python -m anyscale._private.docgen [OPTIONS] OUTPUT_DIR
+
+  Generate markdown docs for the Anyscale CLI & SDK.
+
+Options:
+  -r, --remove-existing  If set, all files in the 'output_dir' that were not
+                         generated will be removed.
+  --help                 Show this message and exit.
+```

anyscale-0.24.16/anyscale/_private/docgen/__main__.py

@@ -0,0 +1,114 @@
+"""Entrypoint to generate reference docs for the Anyscale CLI/SDK.
+
+Usage: python -m anyscale._private.docgen --help
+"""
+import os
+
+import click
+
+import anyscale
+from anyscale._private.docgen.generator import MarkdownGenerator, Module
+from anyscale.commands import job_commands, service_commands
+from anyscale.compute_config.models import (
+    ComputeConfig,
+    HeadNodeConfig,
+    MarketType,
+    WorkerNodeGroupConfig,
+)
+from anyscale.job.models import JobConfig
+from anyscale.service.models import (
+    RayGCSExternalStorageConfig,
+    ServiceConfig,
+    ServiceState,
+    ServiceStatus,
+    ServiceVersionState,
+    ServiceVersionStatus,
+)
+
+
+# Defines all modules to be documented.
+ALL_MODULES = [
+    Module(
+        title="Job",
+        filename="job-api.md",
+        cli_prefix="anyscale job",
+        cli_commands=[job_commands.submit],
+        sdk_prefix="anyscale.job",
+        sdk_commands=[anyscale.job.submit],
+        models=[JobConfig],
+    ),
+    Module(
+        title="Service",
+        filename="service-api.md",
+        cli_prefix="anyscale service",
+        cli_commands=[
+            service_commands.deploy,
+            service_commands.status,
+            service_commands.rollback,
+            service_commands.terminate,
+        ],
+        sdk_prefix="anyscale.service",
+        sdk_commands=[
+            anyscale.service.deploy,
+            anyscale.service.status,
+            anyscale.service.rollback,
+            anyscale.service.terminate,
+        ],
+        models=[
+            ServiceConfig,
+            RayGCSExternalStorageConfig,
+            ServiceStatus,
+            ServiceState,
+            ServiceVersionStatus,
+            ServiceVersionState,
+        ],
+    ),
+    Module(
+        title="Compute Config",
+        filename="compute-config-api.md",
+        cli_prefix="anyscale compute-config",
+        cli_commands=[],
+        sdk_prefix="anyscale.compute_config",
+        sdk_commands=[],
+        models=[ComputeConfig, HeadNodeConfig, WorkerNodeGroupConfig, MarketType],
+    ),
+]
+
+
+@click.command(help="Generate markdown docs for the Anyscale CLI & SDK.")
+@click.argument("output_dir")
+@click.option(
+    "-r",
+    "--remove-existing",
+    is_flag=True,
+    default=False,
+    help="If set, all files in the 'output_dir' that were not generated will be removed.",
+)
+def generate(
+    output_dir: str, *, remove_existing: bool = False,
+):
+    if not os.path.isdir(output_dir):
+        raise RuntimeError(f"output_dir '{output_dir}' does not exist.")
+
+    gen = MarkdownGenerator(ALL_MODULES)
+    gen.generate()
+
+    generated_files = set()
+    os.makedirs(output_dir, exist_ok=True)
+    for filename, content in gen.generate().items():
+        generated_files.add(filename)
+        full_path = os.path.join(output_dir, filename)
+        print(f"Writing output file {full_path}")
+        with open(full_path, "w") as f:
+            f.write(content)
+
+    if remove_existing:
+        to_remove = set(os.listdir(output_dir)) - generated_files
+        for path in to_remove:
+            full_path = os.path.join(output_dir, path)
+            print(f"Removing existing file {full_path}")
+            os.unlink(full_path)
+
+
+if __name__ == "__main__":
+    generate()

anyscale-0.24.16/anyscale/_private/docgen/generator.py

@@ -0,0 +1,377 @@
+import ast
+from dataclasses import dataclass, fields
+import inspect
+import typing
+from typing import Any, Callable, Dict, List, Optional, Type, Union
+
+import click
+import yaml
+
+from anyscale._private.models.model_base import ModelBaseType, ModelEnumType
+
+
+ModelType = Union[ModelBaseType, ModelEnumType]
+
+CLI_OPTIONS_TO_SKIP = {"help", "service_id", "project_id"}
+FILE_EXTENSION = ".md"
+
+CUSTOMER_HOSTED_ANCHOR = "#customer-hosted-only"
+CUSTOMER_HOSTED_HEADER = """\
+#### Customer-hosted cloud features {#customer-hosted-only}
+:::note
+Some features are only available on customer-hosted clouds. Reach out to preview-help@anyscale.com for info.
+:::"""
+
+CUSTOMER_HOSTED_QUALIFIER = (
+    "Only available on [customer-hosted clouds](#customer-hosted-only)."
+)
+
+
+@dataclass
+class Module:
+    title: str
+    filename: str
+    cli_prefix: str
+    cli_commands: List[click.Command]
+    sdk_prefix: str
+    sdk_commands: List[Callable]
+    models: List[ModelType]
+
+
+class MarkdownGenerator:
+    """Generates markdown reference documentations for a list of CLI modules."""
+
+    def __init__(self, modules: List[Module]):
+        self._modules = modules
+
+        # Used as an index to generate anchor links to models.
+        self._model_type_to_filename: Dict[ModelType, str] = {}
+        for m in modules:
+            if not m.filename.endswith(FILE_EXTENSION):
+                raise ValueError(f"All file names must be in '{FILE_EXTENSION}'.")
+
+            for model in m.models:
+                filename_no_extension = m.filename[: -len(FILE_EXTENSION)]
+                self._model_type_to_filename[model] = filename_no_extension
+
+    def generate(self) -> Dict[str, str]:
+        """Generate documentation for all of the input modules.
+
+        Returns a dictionary of filename to generated file contents.
+
+        Each module will contain three sections:
+            - Models
+            - CLI
+            - SDK
+
+        An index.md file will also be generated containing a table linking to all
+        top-level sections for each module.
+        """
+
+        output_files: Dict[str, str] = {}
+        index = "# Platform API\n\n"
+        for m in self._modules:
+            title_anchor = m.title.lower().replace(" ", "-")
+            filename_no_extension = m.filename[: -len(FILE_EXTENSION)]
+
+            output = "import Tabs from '@theme/Tabs';\n"
+            output += "import TabItem from '@theme/TabItem';\n\n"
+            output += f"# {m.title} API Reference\n\n"
+            output += CUSTOMER_HOSTED_HEADER + "\n\n"
+            index += f"- [{m.title} API Reference]({filename_no_extension})\n"
+            index += f"  - [{m.title} Models]({filename_no_extension}#{title_anchor}-models)\n"
+
+            output += f"## {m.title} Models\n"
+            for t in m.models:
+                output += "\n" + self._gen_markdown_for_model(t)
+
+            if len(m.cli_commands) > 0:
+                output += f"## {m.title} CLI\n"
+                index += (
+                    f"  - [{m.title} CLI]({filename_no_extension}#{title_anchor}-cli)\n"
+                )
+                for t in m.cli_commands:
+                    output += "\n" + self._gen_markdown_for_cli_command(
+                        t, cli_prefix=m.cli_prefix
+                    )
+
+            if len(m.sdk_commands) > 0:
+                output += f"## {m.title} SDK\n"
+                index += (
+                    f"  - [{m.title} SDK]({filename_no_extension}#{title_anchor}-sdk)\n"
+                )
+                for t in m.sdk_commands:
+                    output += "\n" + self._gen_markdown_for_sdk_command(
+                        t, sdk_prefix=m.sdk_prefix
+                    )
+
+            output_files[m.filename] = output
+
+        output_files["index.md"] = index
+        return output_files
+
+    def _get_anchor(self, t: ModelType):
+        """Get a markdown anchor (link) to the given type's docs."""
+        filename = self._model_type_to_filename[t]
+        return f"{filename}#{t.__name__.lower()}"
+
+    def _type_container_to_string(self, t: typing.Type) -> str:
+        """Return a str representation of a type hint."""
+        origin, args = typing.get_origin(t), typing.get_args(t)
+        assert origin is not None and args is not None
+
+        if origin is Union:
+            return " | ".join(self._model_type_to_string(arg) for arg in args)
+
+        if origin is dict:
+            arg_str = ", ".join([self._model_type_to_string(arg) for arg in args])
+            if args:
+                return f"Dict[{arg_str}]"
+            else:
+                return "Dict"
+
+        if origin is list:
+            arg_str = ", ".join([self._model_type_to_string(arg) for arg in args])
+            if arg_str:
+                return f"List[{arg_str}]"
+            else:
+                return "List"
+
+        raise NotImplementedError(f"Unhandled type: {t}")
+
+    def _model_type_to_string(self, t: Type):  # noqa: PLR0911
+        """Return a str representation of any Python type.
+
+        Any unrecognized types will be raise an error (handling must be explicitly added).
+        """
+        if t is Any:
+            return "Any"
+        if t is str:
+            return "str"
+        if t is bool:
+            return "bool"
+        if t is int:
+            return "int"
+        if t is float:
+            return "float"
+        if t is type(None):
+            return "None"
+        if typing.get_origin(t) is not None:
+            return self._type_container_to_string(t)
+        if isinstance(t, (ModelBaseType, ModelEnumType)):
+            return f"[{t.__name__}]({self._get_anchor(t)})"
+
+        # Avoid poor rendering of unhandled types.
+        raise NotImplementedError(
+            f"Unhandled type: {t}. Either this type should not be in our public APIs, or you must add handling for it to the doc generator."
+        )
+
+    def _gen_example_tabs(self, t: Union[Callable, ModelBaseType]) -> str:
+        """Generate a tab section that contains yaml, python, and/or CLI examples for the type.
+
+        The examples are pulled from magic attributes:
+            - __doc_yaml_example__ (required for models ending with "Config")
+            - __doc_py_example__ (required in all cases)
+            - __doc_cli_example__ (required for models)
+        """
+        yaml_example: Optional[str] = getattr(t, "__doc_yaml_example__", None)
+        py_example: Optional[str] = getattr(t, "__doc_py_example__", None)
+        cli_example: Optional[str] = getattr(t, "__doc_cli_example__", None)
+
+        if isinstance(t, ModelBaseType):
+            if not py_example:
+                raise ValueError(
+                    f"Model '{t.__name__}' is missing a '__doc_py_example__'."
+                )
+            if t.__name__.endswith("Config") and not yaml_example:
+                raise ValueError(
+                    f"Config model '{t.__name__}' is missing a '__doc_yaml_example__'."
+                )
+        else:
+            if not py_example:
+                raise ValueError(
+                    f"SDK command '{t.__name__}' is missing a '__doc_py_example__'."
+                )
+
+        md = "#### Examples\n\n"
+        md += "<Tabs>\n"
+        if yaml_example:
+            # Validate the YAML example's syntax.
+            try:
+                yaml.safe_load(yaml_example)
+            except Exception as e:  # noqa: BLE001
+                raise ValueError(
+                    f"'{t.__name__}.__doc_yaml_example__' is not valid YAML syntax"
+                ) from e
+
+            md += '<TabItem value="yamlconfig" label="YAML">\n'
+            md += f"```yaml\n{yaml_example}\n```\n"
+            md += "</TabItem>\n"
+        if py_example:
+            # Validate the Python example's syntax.
+            try:
+                ast.parse(py_example)
+            except Exception as e:  # noqa: BLE001
+                raise ValueError(
+                    f"'{t.__name__}.__doc_py_example__' is not valid Python syntax"
+                ) from e
+
+            md += '<TabItem value="pythonsdk" label="Python">\n'
+            md += f"```python\n{py_example}\n```\n"
+            md += "</TabItem>\n"
+        if cli_example:
+            md += '<TabItem value="cli" label="CLI">\n'
+            md += f"```bash\n{cli_example}\n```\n"
+            md += "</TabItem>\n"
+
+        md += "</Tabs>\n"
+
+        return md
+
+    def _gen_markdown_for_model(self, t: ModelType) -> str:
+        """Generate a section for a model type (config/status, or enum).
+
+        For config/status types, the sections will be:
+            - Fields (all fields must be documented via docstring metadata).
+            - Methods (standard methods shared across model types).
+            - Examples (every model must contain examples using the magic attributes).
+
+        For enums, the sections will be:
+            - Values (all values must be documented using the __docstrings__ attribute).
+        """
+        assert isinstance(t, (ModelBaseType, ModelEnumType))
+
+        md = f"### `{t.__name__}`"
+        assert isinstance(t.__doc__, str)
+        md += "\n\n" + t.__doc__ + "\n\n"
+
+        if isinstance(t, ModelBaseType):
+            md += "#### Fields\n\n"
+            for field in fields(t):
+                if field.name.startswith("_"):
+                    # Skip private fields.
+                    continue
+
+                docstring = field.metadata.get("docstring", None)
+                if not docstring:
+                    raise ValueError(
+                        f"Model '{t.__name__}' is missing a docstring for field '{field.name}'"
+                    )
+
+                md += f"- **`{field.name}` ({self._model_type_to_string(field.type)})**: {docstring}\n"
+
+                customer_hosted_only = field.metadata.get("customer_hosted_only", False)
+                if customer_hosted_only:
+                    md += f"  - {CUSTOMER_HOSTED_QUALIFIER}\n"
+            md += "\n\n"
+
+            md += "#### Python Methods\n\n"
+            md += "```python\n"
+            if t.__name__.endswith("Config"):
+                # Only include constructor docs for config models.
+                md += f"def __init__(self, **fields) -> {t.__name__}\n"
+                md += '    """Construct a model with the provided field values set."""\n\n'
+                md += f"def options(self, **fields) -> {t.__name__}\n"
+                md += '    """Return a copy of the model with the provided field values overwritten."""\n\n'
+            md += "def to_dict(self) -> Dict[str, Any]\n"
+            md += '    """Return a dictionary representation of the model."""\n'
+            md += "```\n"
+
+            md += self._gen_example_tabs(t)
+        elif isinstance(t, ModelEnumType):
+            md += "#### Values\n\n"
+            for value in t.__members__:
+                docstring = t.__docstrings__[value]
+                md += f" - **`{value}`**: {docstring}\n"
+            md += "\n"
+
+        return md
+
+    def _gen_markdown_for_cli_command(
+        self, c: click.Command, *, cli_prefix: str
+    ) -> str:
+        """Generate a markdown section for a CLI command.
+
+        The sections will be:
+            - Usage (signature + help str)
+            - Options (documentation is pulled from the help strings)
+
+        TODO(edoakes): add examples for CLI command usage.
+        """
+        ctx = click.Context(command=c)
+        usage_str = " ".join(c.collect_usage_pieces(ctx))
+        info_dict: Dict[str, Any] = c.to_info_dict(ctx)
+
+        md = f"### `{cli_prefix} {c.name}`\n\n"
+        md += "**Usage**\n\n"
+        md += f"`{cli_prefix} {c.name} {usage_str}`\n\n"
+        md += info_dict["help"] + "\n\n"
+
+        options = [
+            param
+            for param in info_dict["params"]
+            if param["param_type_name"] == "option"
+        ]
+        if options:
+            md += "**Options**\n\n"
+            for param in options:
+                if param["name"] in CLI_OPTIONS_TO_SKIP:
+                    continue
+
+                name = "/".join(param["opts"])
+                help_str = param.get("help", None)
+                assert (
+                    help_str
+                ), f"Missing help string for option '{name}' in command '{c.name}'"
+                md += f"- **`{name}`**: {help_str}\n"
+            md += "\n"
+
+        return md
+
+    def _gen_markdown_for_sdk_command(self, c: Callable, *, sdk_prefix: str) -> str:
+        """Generate a markdown section for an SDK command.
+
+        The sections will be:
+            - Arguments (docstrings pulled from __arg_docstrings__ magic attribute)
+            - Returns (if the return type annotation is not None)
+        """
+        md = f"### `{sdk_prefix}.{c.__name__}`\n\n"
+
+        if not c.__doc__:
+            raise ValueError(
+                f"SDK command '{sdk_prefix}.{c.__name__}' is missing a docstring."
+            )
+
+        md += c.__doc__ + "\n"
+
+        signature = inspect.signature(c)
+        if len(signature.parameters) > 0:
+            # TODO: add (optional) tag or `= None`.
+            md += "\n**Arguments**\n\n"
+            for name, param in signature.parameters.items():
+                # Skip private arguments.
+                if name.startswith("_"):
+                    continue
+
+                assert param.annotation is not inspect.Parameter.empty
+                type_str = self._model_type_to_string(param.annotation)
+                if param.default != inspect.Parameter.empty:
+                    type_str += f" = {str(param.default)}"
+
+                arg_docs = c.__arg_docstrings__.get(name, None)  # type: ignore
+                if not arg_docs:
+                    raise ValueError(
+                        f"SDK command '{sdk_prefix}.{c.__name__}' is missing a docstring for argument '{name}'"
+                    )
+
+                md += f"- **`{name}` ({type_str})**: {arg_docs}"
+                md += "\n"
+            md += "\n"
+
+        if signature.return_annotation != inspect.Signature.empty:
+            return_str = self._model_type_to_string(signature.return_annotation)
+            md += f"**Returns**: {return_str}\n\n"
+
+        md += self._gen_example_tabs(c)
+
+        return md

{anyscale-0.24.14 → anyscale-0.24.16}/anyscale/_private/models/image_uri.py

@@ -6,8 +6,9 @@ IMAGE_URI_PATTERN_RE = re.compile(
     r"^"
     # Optional registry host: hostname with optional port
     r"((?P<host>[a-zA-Z0-9.-]+)(?::(?P<port>[0-9]+))?/)?"
-    # Repository name: user_name/repository
-    r"(?P<repository>[a-zA-Z0-9-_]+/([a-zA-Z0-9-/_.])+)"
+    # Repository: user_name/repository
+    r"(?P<user>[a-zA-Z0-9-_]+/)?"  # user is optional.
+    r"(?P<repository>[a-zA-Z0-9-/_.]+)"
     # Optional Tag: version or string after ':'
     r"(:(?P<tag>[a-zA-Z0-9_.-]+))?"
     # Optional Digest: string after '@'

{anyscale-0.24.14 → anyscale-0.24.16}/anyscale/_private/models/model_base.py

@@ -27,8 +27,8 @@ class ModelEnumType(EnumMeta):
 
         missing = {v.value for v in cls} - set(cls.__docstrings__.keys())
         if missing:
-            raise TypeError(
-                f"ModelEnum '{cls.__name__}.__docstrings__' is missing docstrings for values: {missing}"
+            raise ValueError(
+                f"ModelEnum '{cls.__name__}.__docstrings__' is missing docstrings for values: {sorted(missing)}"
             )
 
         return cls
@@ -111,7 +111,7 @@ class ModelBase(metaclass=ModelBaseType):
         return cls(**d)
 
     @classmethod
-    def from_yaml(cls, path: str, **kwargs,) -> "ModelBase":
+    def from_yaml(cls, path: str, **kwargs) -> "ModelBase":
         with open(path) as f:
             args = yaml.safe_load(f) or {}
             if kwargs: