dagster-cloud-cli 1.12.16__py2.py3-none-any.whl

dagster_cloud_cli/core/headers/auth.py

@@ -0,0 +1,16 @@
+from enum import Enum
+
+API_TOKEN_HEADER = "Dagster-Cloud-Api-Token"
+
+DEPLOYMENT_NAME_HEADER = "Dagster-Cloud-Deployment"
+
+DAGSTER_CLOUD_SCOPE_HEADER = "Dagster-Cloud-Scope"
+
+AUTHORIZATION_HEADER = "Authorization"
+BEARER_HEADER_SCHEME = "bearer"
+
+
+class DagsterCloudInstanceScope(Enum):
+    ORGANIZATION = "organization"
+    DEPLOYMENT = "deployment"
+    UNSCOPED = "unscoped"

dagster_cloud_cli/core/headers/impl.py

@@ -0,0 +1,31 @@
+import platform
+
+from dagster_cloud_cli.core.headers.auth import (
+    API_TOKEN_HEADER,
+    DAGSTER_CLOUD_SCOPE_HEADER,
+    DEPLOYMENT_NAME_HEADER,
+    DagsterCloudInstanceScope,
+)
+from dagster_cloud_cli.core.headers.versioning.constants import (
+    DAGSTER_CLOUD_VERSION_HEADER,
+    PYTHON_VERSION_HEADER,
+)
+from dagster_cloud_cli.version import __version__
+
+
+def get_dagster_cloud_api_headers(
+    agent_token: str,
+    scope: DagsterCloudInstanceScope,
+    deployment_name: str | None = None,
+    additional_headers: dict[str, str] | None = None,
+) -> dict[str, str]:
+    return {
+        **{
+            API_TOKEN_HEADER: agent_token,
+            PYTHON_VERSION_HEADER: platform.python_version(),
+            DAGSTER_CLOUD_VERSION_HEADER: __version__,
+            DAGSTER_CLOUD_SCOPE_HEADER: scope.value,
+        },
+        **({DEPLOYMENT_NAME_HEADER: deployment_name} if deployment_name else {}),
+        **(additional_headers if additional_headers else {}),
+    }

dagster_cloud_cli/core/headers/versioning/constants.py

@@ -0,0 +1,2 @@
+DAGSTER_CLOUD_VERSION_HEADER = "Dagster-Cloud-Version"
+PYTHON_VERSION_HEADER = "Python-Version"

dagster_cloud_cli/core/pex_builder/__init__.py

@@ -0,0 +1,8 @@
+from dagster_cloud_cli.core.pex_builder import (
+    deploy as deploy,
+    deps as deps,
+    parse_workspace as parse_workspace,
+    pex_registry as pex_registry,
+    source as source,
+    util as util,
+)

dagster_cloud_cli/core/pex_builder/code_location.py

@@ -0,0 +1,126 @@
+import logging
+import re
+import subprocess
+import time
+
+from dagster_cloud_cli import gql
+from dagster_cloud_cli.core.pex_builder import github_context
+from dagster_cloud_cli.core.pex_builder.github_context import get_git_commit_metadata
+from dagster_cloud_cli.types import SnapshotBaseDeploymentCondition
+
+
+def get_local_branch_name(project_dir: str) -> str:
+    return (
+        subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=project_dir)
+        .decode("utf-8")
+        .strip()
+    )
+
+
+def get_local_repo_name(project_dir: str) -> str:
+    repo_url = (
+        subprocess.check_output(["git", "config", "--get", "remote.origin.url"], cwd=project_dir)
+        .decode("utf-8")
+        .strip()
+    )
+
+    # Handle both SSH URLs (git@github.com:org/repo.git) and https URLs (https://github.com/org/repo.git)
+    delimiters = r"[:/]"
+
+    split_repo_url = re.split(delimiters, repo_url)
+    # Extract just the org/repo part from the git URL
+    return split_repo_url[-2] + "/" + split_repo_url[-1].replace(".git", "")
+
+
+def _get_local_commit_hash(project_dir: str) -> str:
+    return (
+        subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=project_dir)
+        .decode("utf-8")
+        .strip()
+    )
+
+
+def create_or_update_branch_deployment_from_local_git_context(
+    dagster_cloud_org_url: str,
+    dagster_cloud_api_token: str,
+    project_dir: str,
+    mark_closed: bool,
+    base_deployment_name: str | None,
+    snapshot_base_condition: SnapshotBaseDeploymentCondition | None,
+) -> str | None:
+    branch_name = get_local_branch_name(project_dir)
+
+    if not branch_name:
+        raise Exception("Could not determine branch name from local git context")
+
+    # Extract just the org/repo part from the git URL
+    repo_name = get_local_repo_name(project_dir)
+
+    if not repo_name:
+        raise Exception("Could not determine repo name from local git context")
+
+    commit_hash = _get_local_commit_hash(project_dir)
+
+    git_metadata = get_git_commit_metadata(project_dir)
+
+    try:
+        timestamp = float(git_metadata["timestamp"])
+    except Exception:
+        logging.warn("Could not determine commit timestamp from git metadata, using current time")
+        timestamp = time.time()
+
+    commit_msg = git_metadata["message"]
+    author_name = git_metadata["name"]
+    author_email = git_metadata["email"]
+
+    with gql.graphql_client_from_url(dagster_cloud_org_url, dagster_cloud_api_token) as client:
+        deployment_name = gql.create_or_update_branch_deployment(
+            client,
+            repo_name=repo_name,
+            branch_name=branch_name,
+            commit_hash=commit_hash,
+            timestamp=timestamp,
+            author_name=author_name,
+            author_email=author_email,
+            commit_message=commit_msg,
+            pull_request_status="CLOSED" if mark_closed else "OPEN",
+            base_deployment_name=base_deployment_name,
+            snapshot_base_condition=snapshot_base_condition,
+        )
+    return deployment_name
+
+
+def create_or_update_branch_deployment_from_github_context(
+    dagster_cloud_org_url: str,
+    dagster_cloud_api_token: str,
+    github_event: github_context.GithubEvent,
+    mark_closed: bool,
+    base_deployment_name: str | None,
+    snapshot_base_condition: SnapshotBaseDeploymentCondition | None,
+) -> str | None:
+    """Return the branch deployment associated with the github PR."""
+    event = github_event
+    logging.debug("Read github event GithubEvent(%r)", event.__dict__)
+    if not event.branch_name:
+        return None
+    else:
+        url = f"{dagster_cloud_org_url}"
+        with gql.graphql_client_from_url(url, dagster_cloud_api_token) as client:
+            deployment_name = gql.create_or_update_branch_deployment(
+                client,
+                repo_name=event.repo_name,
+                branch_name=event.branch_name,
+                commit_hash=event.github_sha,
+                timestamp=event.timestamp,
+                branch_url=event.branch_url,
+                pull_request_url=event.pull_request_url,
+                pull_request_status="CLOSED" if mark_closed else event.pull_request_status,
+                pull_request_number=event.pull_request_id,
+                author_name=event.author_name,
+                author_email=event.author_email,
+                commit_message=event.commit_msg,
+                author_avatar_url=github_event.get_github_avatar_url(),
+                base_deployment_name=base_deployment_name,
+                snapshot_base_condition=snapshot_base_condition,
+            )
+        return deployment_name

dagster_cloud_cli/core/pex_builder/deploy.py

@@ -0,0 +1,255 @@
+# end-to-end workflow for rebuilding and publishing code locations
+
+import logging
+import os
+import sys
+from dataclasses import dataclass
+
+import click
+from packaging import version
+
+from dagster_cloud_cli import gql, ui
+from dagster_cloud_cli.core.pex_builder import (
+    deps,
+    github_context,
+    parse_workspace,
+    pex_registry,
+    source,
+    util,
+)
+
+
+@dataclass
+class LocationBuild:
+    """Inputs and outputs for each code location."""
+
+    location: parse_workspace.Location
+
+    # local dependencies are packaged with the source.pex along with the main code location
+    local_packages: deps.LocalPackages
+
+    # all other dependencies are packaged in the deps.pex
+    deps_requirements: deps.DepsRequirements
+
+    # One of deps_pex_path or published_deps_pex should be set
+    deps_pex_path: str | None = None  # locally build deps.pex
+    published_deps_pex: str | None = None  # already published deps.pex
+    # dagster_version should be always set for both cases, pre published and locally built deps
+    dagster_version: str | None = None
+    source_pex_path: str | None = None
+    pex_tag: str | None = None  # composite tag used to identify the set of pex files
+
+    code_location_update_error: Exception | None = None
+
+
+@dataclass
+class DepsCacheTags:
+    deps_cache_from_tag: str | None
+    deps_cache_to_tag: str | None
+
+
+def build_locations(
+    dagster_cloud_url: str,
+    dagster_cloud_api_token: str,
+    locations: list[parse_workspace.Location],
+    output_directory: str,
+    upload_pex: bool,
+    deps_cache_tags: DepsCacheTags,
+    python_version: version.Version,
+    build_method: deps.BuildMethod = deps.BuildMethod.DOCKER_FALLBACK,
+) -> list[LocationBuild]:
+    location_builds = []
+    for location in locations:
+        local_packages, deps_requirements = deps.get_deps_requirements(
+            location.directory,
+            python_version=python_version,
+        )
+        location_builds.append(
+            LocationBuild(
+                location=location,
+                local_packages=local_packages,
+                deps_requirements=deps_requirements,
+            )
+        )
+
+    # dedup requirements so each is only built once
+    builds_for_requirements_hash: dict[str, list[LocationBuild]] = {}
+    for location_build in location_builds:
+        requirements_hash = location_build.deps_requirements.hash
+        builds_for_requirements_hash.setdefault(requirements_hash, []).append(location_build)
+
+    # build each deps pex once and assign to all related builds
+    for requirements_hash in builds_for_requirements_hash:
+        builds = builds_for_requirements_hash[requirements_hash]
+        deps_requirements = builds[0].deps_requirements
+
+        # if a --deps-cache-from is specified, don't build deps.pex files if it is already published
+        if upload_pex and deps_cache_tags.deps_cache_from_tag:
+            published_deps_pex_info = pex_registry.get_cached_deps_details(
+                dagster_cloud_url,
+                dagster_cloud_api_token,
+                deps_requirements.hash,
+                deps_cache_tags.deps_cache_from_tag,
+            )
+        else:
+            published_deps_pex_info = None
+
+        if published_deps_pex_info:
+            published_deps_pex = published_deps_pex_info["deps_pex_name"]
+            ui.print(
+                f"Found published deps.pex {published_deps_pex} for requirements_hash {deps_requirements.hash}, cache_tag {deps_cache_tags.deps_cache_from_tag}, "
+                "skipping rebuild.",
+            )
+
+            for location_build in builds:
+                location_build.published_deps_pex = published_deps_pex
+                location_build.dagster_version = published_deps_pex_info["dagster_version"]
+        else:
+            ui.print(
+                f"No published deps.pex found for requirements_hash {deps_requirements.hash}, cache_tag {deps_cache_tags.deps_cache_from_tag}, will rebuild.",
+            )
+            try:
+                deps_pex_path, dagster_version = deps.build_deps_from_requirements(
+                    deps_requirements,
+                    output_directory,
+                    build_method=build_method,
+                )
+            except deps.DepsBuildFailure as err:
+                logging.error("Failed to build dependencies: %s", err.stderr)
+                sys.exit(1)
+            for location_build in builds:
+                location_build.deps_pex_path = deps_pex_path
+                location_build.dagster_version = dagster_version
+
+    # build each source once
+    for location_build in location_builds:
+        location_build.source_pex_path = source.build_source_pex(
+            location_build.location.directory,
+            location_build.local_packages.local_package_paths,
+            output_directory,
+            python_version,
+        )
+
+    # compute pex tags
+    for location_build in location_builds:
+        deps_pex = (
+            location_build.deps_pex_path
+            if location_build.deps_pex_path
+            else location_build.published_deps_pex
+        )
+        if not deps_pex or not location_build.source_pex_path:
+            raise ValueError("No deps.pex or source.pex")
+
+        location_build.pex_tag = util.build_pex_tag([deps_pex, location_build.source_pex_path])
+    return location_builds
+
+
+def get_user_specified_base_image_for(
+    dagster_cloud_url: str, dagster_cloud_api_token: str, location_build: LocationBuild
+) -> str | None:
+    # Full path to base image is supplied
+    base_image = os.getenv("SERVERLESS_BASE_IMAGE")
+    if base_image:
+        return base_image
+
+    # Tag suffix for base image is supplied - uses custom uploaded image
+    base_image_tag = os.getenv("SERVERLESS_BASE_IMAGE_TAG")
+    if base_image_tag:
+        # Point to user's registry info with this tag suffix
+        with gql.graphql_client_from_url(dagster_cloud_url, dagster_cloud_api_token) as client:
+            registry_info = gql.get_ecr_info(client)
+            return f"{registry_info['registry_url']}:{base_image_tag}"
+
+    return None
+
+
+def notify(deployment_name: str | None, location_name: str, action: str):
+    if github_event:
+        github_context.update_pr_comment(
+            github_event,
+            action=action,
+            deployment_name=deployment_name,
+            location_name=location_name,
+        )
+
+
+github_event: github_context.GithubEvent | None = None
+
+
+def load_github_event(project_dir):
+    global github_event  # noqa: PLW0603
+    github_event = github_context.get_github_event(project_dir)
+
+
+@click.command()
+@click.argument("dagster_cloud_file", type=click.Path(exists=True))
+@click.argument("build_output_dir", type=click.Path(exists=False))
+@click.option(
+    "--upload-pex",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Upload PEX files to registry.",
+)
+@click.option(
+    "--deps-cache-from",
+    type=str,
+    required=False,
+    help=(
+        "Try to reuse a pre-existing deps pex file. A deps pex file is reused if it was "
+        "built with a --deps-cache-to value that matches this flag value, AND the requirements.txt "
+        "and setup.py were identical."
+    ),
+)
+@click.option(
+    "--deps-cache-to",
+    type=str,
+    required=False,
+    help=(
+        "Allow reusing the generated deps pex and associate with the given tag. "
+        "See --deps-cache-from for how to reuse deps pex files."
+    ),
+)
+@click.option(
+    "--update-code-location",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Update code location to use new PEX files.",
+)
+@click.option(
+    "--code-location-details",
+    callback=util.parse_kv,
+    help=(
+        "Syntax: --code-location-details deployment=NAME,commit_hash=HASH. "
+        "When not provided, details are inferred from the github action environment."
+    ),
+)
+@util.python_version_option()
+@click.option(
+    "--build-sdists/--no-build-sdists",
+    is_flag=True,
+    default=False,
+    help="Whether to build source only Python dependencies (sdists).",
+)
+def cli(
+    dagster_cloud_file,
+    build_output_dir,
+    upload_pex,
+    deps_cache_to,
+    deps_cache_from,
+    update_code_location,
+    code_location_details,
+    python_version,
+    build_sdists,
+):
+    logging.error(
+        "This entrypoint is obsolete. Please use `dagster-cloud serverless"
+        " deploy-python-executable`."
+    )
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    cli()