dagster-cloud-cli 1.12.16__py2.py3-none-any.whl

dagster_cloud_cli/commands/ci/checks.py

@@ -0,0 +1,135 @@
+import os
+import pathlib
+from dataclasses import dataclass, field
+from enum import Enum
+
+import pydantic
+
+from dagster_cloud_cli import gql, ui
+from dagster_cloud_cli.config.models import load_dagster_cloud_yaml
+from dagster_cloud_cli.config_utils import TOKEN_ENV_VAR_NAME
+
+
+def get_validation_errors(validation_error: pydantic.ValidationError) -> list[str]:
+    errors = []
+    for error in validation_error.errors():
+        if "type" in error:
+            location = ".".join([str(part) for part in error["loc"] if part != "__root__"])
+            if error["type"] == "value_error.missing":
+                errors.append(f"expected '{location}': missing required field")
+            elif error["type"] == "value_error.extra":
+                errors.append(f"unexpected '{location}': unknown field")
+            else:
+                errors.append(f"{error['type']} at '{location}': {error['msg']}")
+    return errors
+
+
+@dataclass
+class CheckResult:
+    errors: list[str] = field(default_factory=list)
+    messages: list[str] = field(default_factory=list)
+
+
+def check_dagster_cloud_yaml(yaml_path: pathlib.Path) -> CheckResult:
+    result = CheckResult()
+
+    if not yaml_path.exists():
+        result.errors.append(f"No such file {yaml_path}")
+        return result
+
+    yaml_text = yaml_path.read_text()
+    if not yaml_text.strip():
+        result.errors.append(f"Unexpected blank file {yaml_path}")
+        return result
+
+    try:
+        parsed = load_dagster_cloud_yaml(yaml_path.read_text())
+    except pydantic.ValidationError as err:
+        for error in get_validation_errors(err):
+            result.errors.append(error)
+        return result
+
+    for location in parsed.locations:
+        if location.build and location.build.directory:
+            build_path = yaml_path.parent / location.build.directory
+            if not build_path.is_dir():
+                result.errors.append(
+                    f"Build directory {build_path} not found for location"
+                    f" {location.location_name} at {build_path.absolute()}"
+                )
+    return result
+
+
+class Check(Enum):
+    error = "error"
+    warn = "warn"
+    skip = "skip"
+
+
+class Verdict(Enum):
+    failed = "failed"
+    warning = "warning"
+    skipped = "skipped"
+    passed = "passed"
+
+
+def handle_result(
+    result: CheckResult,
+    check: Check,
+    prefix_message: str,
+    success_message: str,
+    failure_message: str,
+) -> Verdict:
+    def full_msg(msg):
+        return prefix_message + msg
+
+    def passed(msg):
+        ui.print("✅" + full_msg(msg))
+
+    def warning(msg):
+        ui.print("🟡" + ui.yellow(full_msg(msg)))
+
+    def failed(msg):
+        ui.print("🚫" + ui.red(full_msg(msg)))
+
+    def print_indented(msgs):
+        lines = "\n".join(msgs).splitlines(keepends=False)
+        for line in lines:
+            ui.print("  | " + line)
+
+    if check == Check.skip:
+        return Verdict.skipped
+
+    if result.errors:
+        if check == Check.error:
+            failed(failure_message)
+            print_indented(result.messages)
+            print_indented(result.errors)
+            ui.print("\n")
+            return Verdict.failed
+        elif check == Check.warn:
+            warning(failure_message)
+            print_indented(result.messages)
+            print_indented(result.errors)
+            ui.print("\n")
+            return Verdict.warning
+
+    passed(success_message)
+    print_indented(result.messages)
+    ui.print("\n")
+    return Verdict.passed
+
+
+def check_connect_dagster_cloud(url) -> CheckResult:
+    if TOKEN_ENV_VAR_NAME not in os.environ:
+        return CheckResult([f"{TOKEN_ENV_VAR_NAME} not set"])
+    result = CheckResult()
+    result.messages.append(f"Connecting to {url} using {TOKEN_ENV_VAR_NAME}")
+    with gql.graphql_client_from_url(url, os.environ[TOKEN_ENV_VAR_NAME]) as client:
+        try:
+            gql.get_organization_settings(client)
+            result.messages.append("Connection successful")
+        except Exception as err:
+            result.errors.append(f"Failed to connect to {url}: {err}")
+
+    return result

dagster_cloud_cli/commands/ci/report.py

@@ -0,0 +1,49 @@
+from dagster_cloud_cli.commands.ci import state
+
+STATUS_IMAGES = {
+    state.LocationStatus.success: (
+        "https://raw.githubusercontent.com/dagster-io/dagster-cloud-action/main/assets/success.png"
+    ),
+    state.LocationStatus.pending: (
+        "https://raw.githubusercontent.com/dagster-io/dagster-cloud-action/main/assets/pending.png"
+    ),
+    state.LocationStatus.failed: (
+        "https://raw.githubusercontent.com/dagster-io/dagster-cloud-action/main/assets/failed.png"
+    ),
+}
+
+STATUS_MESSAGES = {
+    state.LocationStatus.success: "View in Cloud",
+    state.LocationStatus.pending: "Building...",
+    state.LocationStatus.failed: "Deploy failed",
+}
+
+
+def markdown_report(location_states: list[state.LocationState]):
+    markdown = []
+    markdown.append(
+        """
+| Location          | Status          | Link    | Updated         |
+| ----------------- | --------------- | ------- | --------------- | 
+"""
+    )
+    for location_state in location_states:
+        if not location_state.selected or not location_state.history:
+            continue
+        last_status = location_state.history[-1]
+        image_url = STATUS_IMAGES[last_status.status]
+        message = STATUS_MESSAGES[last_status.status]
+        if last_status.status == state.LocationStatus.success or not location_state.status_url:
+            link_url = f"{location_state.url}/{location_state.deployment_name}/locations"
+        else:
+            link_url = location_state.status_url
+        status_image = f'[<img src="{image_url}" width=25 height=25/>]({link_url})'
+        status_message = f"[{message}]({link_url})"
+
+        time_str = last_status.timestamp.strftime("%b %d, %Y at %I:%M %p (%Z)")
+        markdown.append(
+            f"| {location_state.location_name} | {status_image}  | {status_message}  |"
+            f" {time_str} |\n"
+        )
+
+    return "".join(markdown)

dagster_cloud_cli/commands/ci/state.py

@@ -0,0 +1,129 @@
+import datetime
+import json
+import os
+from abc import ABCMeta, abstractmethod
+from enum import Enum
+from typing import Literal
+
+from dagster_shared.serdes.objects.models import DefsStateInfo
+from pydantic import BaseModel, Extra, Field
+
+from dagster_cloud_cli.config import models
+
+
+class DockerBuildOutput(BaseModel, extra=Extra.forbid):
+    strategy: Literal["docker"] = "docker"
+    python_version: str | None = None
+    image: str
+
+
+class PexBuildOutput(BaseModel, extra=Extra.forbid):
+    strategy: Literal["python-executable"] = "python-executable"
+    python_version: str
+    image: str | None  # if None we determine the image from the python_version and agent version
+    pex_tag: str
+
+
+class BuildMetadata(BaseModel):
+    git_url: str | None
+    commit_hash: str | None
+    build_config: models.Build | None  # copied from dagster_cloud.yaml
+
+
+class LocationStatus(Enum):
+    success = "success"
+    pending = "pending"
+    failed = "failed"
+
+
+class StatusChange(BaseModel):
+    timestamp: datetime.datetime
+    status: LocationStatus
+    log: str
+
+
+class LocationState(BaseModel, extra=Extra.forbid):
+    # we intentionally don't save api_token here for security reasons
+    url: str
+    deployment_name: str
+    location_file: str
+    location_name: str
+    is_branch_deployment: bool
+    selected: bool = True
+    build: BuildMetadata
+    build_output: DockerBuildOutput | PexBuildOutput | None = Field(None, discriminator="strategy")
+    defs_state_info: DefsStateInfo | None = None
+    status_url: str | None  # link to cicd run url when building and dagster cloud url when done
+    history: list[StatusChange] = []
+    project_dir: str | None = None
+
+    def add_status_change(self, status: LocationStatus, log: str):
+        self.history.append(
+            StatusChange(
+                timestamp=datetime.datetime.now(datetime.timezone.utc),
+                status=status,
+                log=log,
+            )
+        )
+
+
+class Store(metaclass=ABCMeta):
+    @abstractmethod
+    def load(self, location_name: str) -> LocationState: ...
+
+    @abstractmethod
+    def save(self, location_state: LocationState): ...
+
+    @abstractmethod
+    def list_locations(self) -> list[LocationState]: ...
+
+    def list_selected_locations(self) -> list[LocationState]:
+        return [location for location in self.list_locations() if location.selected]
+
+
+class FileStore(Store):
+    def __init__(self, statedir: str):
+        self.statedir = os.path.abspath(statedir)
+        self.location_file_prefix = "location-"
+        if not os.path.isdir(self.statedir):
+            os.makedirs(self.statedir)
+
+    def __repr__(self):
+        return f"<FileStore(statedir={self.statedir!r})>"
+
+    def _get_filepath(self, location_name) -> str:
+        return os.path.join(self.statedir, f"{self.location_file_prefix}{location_name}.json")
+
+    def list_locations(self) -> list[LocationState]:
+        return [
+            self._location_from_file(os.path.join(self.statedir, filename))
+            for filename in os.listdir(self.statedir)
+            if filename.startswith(self.location_file_prefix)
+        ]
+
+    def load(self, location_name: str) -> LocationState:
+        filepath = self._get_filepath(location_name)
+        if not filepath:
+            raise KeyError(f"No saved state for {location_name} at {filepath}")
+        return self._location_from_file(self._get_filepath(location_name))
+
+    def _location_from_file(self, filepath: str) -> LocationState:
+        with open(filepath, encoding="utf-8") as f:
+            return LocationState.parse_obj(json.load(f))
+
+    def save(self, location_state: LocationState):
+        filepath = self._get_filepath(location_state.location_name)
+        with open(filepath, "w", encoding="utf-8") as f:
+            f.write(location_state.json())
+
+    def deselect(self, location_names: list[str]):
+        locations = [self.load(location_name) for location_name in location_names]
+        for location in locations:
+            location.selected = False
+            self.save(location)
+
+    def select(self, location_names: list[str]):
+        locations = [self.load(location_name) for location_name in location_names]
+        for location in locations:
+            location.selected = True
+            self.save(location)

dagster_cloud_cli/commands/ci/utils.py

@@ -0,0 +1,22 @@
+from contextlib import contextmanager
+from typing import Any
+
+from dagster_shared import check
+
+from dagster_cloud_cli import gql
+from dagster_cloud_cli.config_utils import get_user_token
+
+
+@contextmanager
+def client_from_env(url: str, deployment: str | None = None):
+    if deployment:
+        url = url + "/" + deployment
+    with gql.graphql_client_from_url(
+        url, check.not_none(get_user_token()), deployment_name=deployment
+    ) as client:
+        yield client
+
+
+def get_registry_info(url: str) -> dict[str, Any]:
+    with client_from_env(url) as client:
+        return gql.get_ecr_info(client)

dagster_cloud_cli/commands/config.py

@@ -0,0 +1,250 @@
+import json
+import webbrowser
+from enum import Enum
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from urllib import parse
+
+from dagster_shared.plus.config import DagsterPlusCliConfig
+from dagster_shared.plus.login_server import start_login_server
+from typer import Argument, Context, Option, Typer
+
+from dagster_cloud_cli import gql, ui
+from dagster_cloud_cli.config_utils import (
+    available_deployment_names,
+    dagster_cloud_options,
+    read_config,
+    write_config,
+)
+
+app = Typer(help="Configure the Dagster Cloud CLI.")
+
+
+@app.command()
+@dagster_cloud_options()
+def set_deployment(
+    ctx: Context,
+    deployment: str = Argument(..., autocompletion=available_deployment_names),
+):
+    """Set the default deployment for CLI commands."""
+    deployments = available_deployment_names(ctx=ctx)
+    if not deployment or deployment not in deployments:
+        raise ui.error(f"Deployment {ui.as_code(deployment)} not found")
+
+    config = read_config()
+    new_config = DagsterPlusCliConfig(
+        **{**config.__dict__, "default_deployment": deployment},
+    )
+    write_config(new_config)
+
+    ui.print(f"Default deployment changed to {ui.as_code(deployment)}")
+
+
+@app.command()
+def view(
+    show_token: bool = Option(
+        False,
+        "--show-token",
+        "-s",
+        help="Whether to display the user token in plaintext.",
+    ),
+):
+    """View the current CLI configuration."""
+    config = read_config()
+    config_to_display = {k: v for k, v in config.__dict__.items() if v is not None}
+    if not show_token and config.user_token:
+        config_to_display["user_token"] = ui.censor_token(config.user_token)
+    ui.print_yaml(config_to_display)
+
+
+app_configure = Typer()
+
+
+def create_token_callback_handler(nonce: str) -> type[BaseHTTPRequestHandler]:
+    class TokenCallbackHandler(BaseHTTPRequestHandler):
+        def _send_shared_headers(self):
+            self.send_header("Access-Control-Allow-Origin", "*")
+            self.send_header("Access-Control-Allow-Methods", "POST,OPTIONS")
+            self.send_header("Access-Control-Allow-Headers", "Content-Type")
+
+        def _error(self, code, message=None):
+            self.send_error(code, message)
+            self._send_shared_headers()
+            self.end_headers()
+
+        def do_OPTIONS(self):
+            self.send_response(200)
+            self._send_shared_headers()
+            self.end_headers()
+
+        def do_POST(self):
+            url = parse.urlparse(self.path)
+
+            if url.path != "/callback":
+                self._error(404, "Page not found")
+                return
+
+            content = self.rfile.read(int(self.headers["Content-Length"]))
+            body = json.loads(content)
+
+            if not body.get("nonce") or body.get("nonce") != nonce:
+                self._error(400, "Invalid nonce")
+                return
+            if not body.get("token"):
+                self._error(400, "No user token provided")
+                return
+            if not body.get("organization"):
+                self._error(400, "No organization provided")
+                return
+
+            organization = body.get("organization")
+            token = body.get("token")
+
+            self.send_response(200)
+            self._send_shared_headers()
+            response = b'{ "ok": true }'
+            self.send_header("Content-length", str(len(response)))
+            self.end_headers()
+
+            self.wfile.write(response)
+
+            # Pass back result values
+            assert isinstance(self.server, TokenServer)
+            self.server.set_result(organization=organization, token=token)
+            self.server.shutdown()
+
+        # Overridden so that the webserver doesn't log requests to the console
+
+        def log_request(self, code="-", size="-"):
+            return
+
+    return TokenCallbackHandler
+
+
+class TokenServer(HTTPServer):
+    organization: str | None = None
+    token: str | None = None
+
+    def __init__(self, host: tuple[str, int], nonce: str):
+        super().__init__(host, create_token_callback_handler(nonce))
+
+    def shutdown(self):
+        # Stop serving the token server
+        # https://stackoverflow.com/a/36017741
+        setattr(self, "_BaseServer__shutdown_request", True)
+
+    def set_result(self, organization: str, token: str):
+        self.organization = organization
+        self.token = token
+
+    def get_organization(self) -> str | None:
+        return self.organization
+
+    def get_token(self) -> str | None:
+        return self.token
+
+
+class SetupAuthMethod(Enum):
+    WEB = "web"
+    CLI = "cli"
+
+
+def _settings_method_input(api_token: str):
+    if api_token:
+        choices = [
+            ui.choice(SetupAuthMethod.CLI, "Authenticate using token or keep current settings"),
+            ui.choice(SetupAuthMethod.WEB, "Authenticate in browser"),
+        ]
+    else:
+        choices = [
+            ui.choice(SetupAuthMethod.WEB, "Authenticate in browser"),
+            ui.choice(SetupAuthMethod.CLI, "Authenticate using token"),
+        ]
+    return ui.list_input(
+        "How would you like to authenticate the CLI?",
+        choices,
+    )
+
+
+def _setup(organization: str, deployment: str, api_token: str):
+    setup_method = _settings_method_input(api_token)
+
+    if setup_method == SetupAuthMethod.WEB:
+        server, url = start_login_server()
+
+        try:
+            webbrowser.open(url, new=0, autoraise=True)
+            ui.print(
+                f"Opening browser...\nIf a window does not open automatically, visit {url} to"
+                " finish authorization"
+            )
+        except webbrowser.Error as e:
+            ui.warn(f"Error launching web browser: {e}\n\nTo finish authorization, visit {url}\n")
+
+        server.serve_forever()
+        new_org = server.get_organization()
+        new_api_token = server.get_token()
+        ui.print(f"Authorized for organization {ui.as_code(str(new_org))}\n")
+    else:
+        new_org = ui.input("Dagster Cloud organization:", default=organization or "") or None
+        if not api_token:
+            deployment_name = deployment if deployment else "prod"
+            ui.print(
+                "\nTo create a new user token or find an existing token, visit"
+                f" https://dagster.cloud/{new_org}/{deployment_name}/org-settings/tokens"
+            )
+        new_api_token = (
+            ui.password_input("Dagster Cloud user token:", default=api_token or "") or None
+        )
+
+    # Attempt to fetch deployment names from server, fallback to a text input upon failure
+    deployment_names = []
+    if new_org and new_api_token:
+        try:
+            with gql.graphql_client_from_url(gql.url_from_config(new_org), new_api_token) as client:
+                deployments = gql.fetch_full_deployments(client)
+            deployment_names = [deployment["deploymentName"] for deployment in deployments]
+        except:
+            ui.warn(
+                "Could not fetch deployment names from server - organization or user token may be"
+                " set incorrectly."
+            )
+
+    new_deployment: str | None = None
+    if deployment_names:
+        options = ["None"] + deployment_names
+        new_deployment = ui.list_input(
+            "Default deployment:",
+            choices=options,
+            default=deployment if deployment in options else "None",
+        )
+        if new_deployment == "None":
+            new_deployment = None
+    else:
+        new_deployment = ui.input("Default deployment:", default=deployment or "") or None
+
+    write_config(
+        DagsterPlusCliConfig(
+            organization=new_org,
+            default_deployment=new_deployment,
+            user_token=new_api_token,
+        )
+    )
+
+
+@app.command()
+@dagster_cloud_options(allow_empty=True)
+def setup(organization: str, deployment: str, api_token: str):
+    """Populate the CLI configuration."""
+    _setup(organization, deployment, api_token)
+
+
+app_configure = Typer(hidden=True)
+
+
+# Legacy, to support the old `dagster-cloud configure` path
+# New command is `dagster-cloud config setup`
+@app_configure.command(name="configure", hidden=True)
+@dagster_cloud_options(allow_empty=True)
+def configure_legacy(organization: str, deployment: str, api_token: str):
+    """Populate the CLI configuration."""
+    _setup(organization, deployment, api_token)

dagster_cloud_cli/commands/deployment/__init__.py

@@ -0,0 +1,108 @@
+from pathlib import Path
+from typing import Annotated
+
+import yaml
+from typer import Argument, Typer
+
+from dagster_cloud_cli import gql, ui
+from dagster_cloud_cli.config_utils import dagster_cloud_options
+from dagster_cloud_cli.core.artifacts import download_artifact, upload_artifact
+from dagster_cloud_cli.core.headers.auth import DagsterCloudInstanceScope
+from dagster_cloud_cli.utils import create_stub_app
+
+try:
+    from dagster_cloud_cli.commands.deployment.alert_policies.commands import (
+        app as alert_policies_app,
+    )
+except ImportError:
+    alert_policies_app = create_stub_app("dagster-cloud")
+
+app = Typer(help="Customize your Dagster Cloud deployment.")
+
+settings_app = Typer(help="Customize your deployment settings.")
+app.add_typer(settings_app, name="settings", no_args_is_help=True)
+app.add_typer(alert_policies_app, name="alert-policies", no_args_is_help=True)
+
+
+@settings_app.command(name="set-from-file")
+@dagster_cloud_options(allow_empty=True, requires_url=True)
+def set_from_filecommand(
+    api_token: str,
+    url: str,
+    deployment: str | None,
+    file_path: Path = Argument(..., readable=True, metavar="SETTINGS_FILE_PATH"),
+):
+    """Set the Dagster Cloud deployment settings from a YAML file."""
+    with open(file_path, encoding="utf8") as f:
+        settings = {"settings": yaml.safe_load(f) or {}}
+    with gql.graphql_client_from_url(url, api_token, deployment_name=deployment) as client:
+        gql.set_deployment_settings(client, settings)
+
+
+@settings_app.command(name="get")
+@dagster_cloud_options(allow_empty=True, requires_url=True)
+def get_command(
+    api_token: str,
+    url: str,
+    deployment: str | None,
+):
+    """Get the Dagster Cloud deployment settings."""
+    with gql.graphql_client_from_url(url, api_token, deployment_name=deployment) as client:
+        settings = gql.get_deployment_settings(client)
+    ui.print_yaml(settings)
+
+
+@app.command(name="upload-artifact")
+@dagster_cloud_options(allow_empty=True, requires_url=True)
+def upload_artifact_command(
+    key: Annotated[
+        str,
+        Argument(
+            help="The key for the artifact being uploaded.",
+        ),
+    ],
+    path: Annotated[
+        str,
+        Argument(
+            help="The path to the file to upload.",
+        ),
+    ],
+    api_token: str,
+    url: str,
+):
+    """Upload a deployment scoped artifact."""
+    upload_artifact(
+        url=url,
+        scope=DagsterCloudInstanceScope.DEPLOYMENT,
+        api_token=api_token,
+        key=key,
+        path=path,
+    )
+
+
+@app.command(name="download-artifact")
+@dagster_cloud_options(allow_empty=True, requires_url=True)
+def download_artifact_command(
+    key: Annotated[
+        str,
+        Argument(
+            help="The key for the artifact to download.",
+        ),
+    ],
+    path: Annotated[
+        str,
+        Argument(
+            help="Path to the file that the contents should be written to.",
+        ),
+    ],
+    api_token: str,
+    url: str,
+):
+    """Download a deployment scoped artifact."""
+    download_artifact(
+        url=url,
+        scope=DagsterCloudInstanceScope.DEPLOYMENT,
+        api_token=api_token,
+        key=key,
+        path=path,
+    )