prefect-deployments-toolkit 0.0.1__py3-none-any.whl

prefect_deployments_toolkit/__main__.py

@@ -0,0 +1,145 @@
+"""Entry point: parse args and apply all deployments concurrently.
+
+Usage:
+python -m apply_deployments \
+    --deployment-names "hello_world3,hello_world_4" \
+    --enable-schedule false \
+    --tag dev \
+    --reference feature_branch1 \
+    --repo-name edp-flows \
+    --custom-image "" \
+    --deployments-dir prefect/deployments \
+    --dev-work-pool lapp-dev-work-pool-prefect3 \
+    --backend cli \
+    --enforce-unique-deployment-names false
+"""
+
+import argparse
+import logging
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from .deployment import DeploymentContext, apply_single_deployment
+from .log_buffer import buffered_deployment_log
+
+logger = logging.getLogger(__name__)
+
+MAX_WORKERS = 8
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Apply Prefect deployments.")
+    parser.add_argument("--deployment-names", required=True,
+                         help="Comma-separated list of deployment names.")
+    parser.add_argument("--enable-schedule", required=True,
+                         choices=["true", "false"],
+                         help="Whether to enable schedules after deploy.")
+    parser.add_argument("--tag", required=True,
+                         help="Image/environment tag (e.g. 'dev', 'v1.2.3').")
+    parser.add_argument("--reference", required=True,
+                         help="Git branch or tag reference (e.g. 'main', 'feature_branch1').")
+    parser.add_argument("--repo-name", required=True,
+                         help="Repository name used in job variable paths.")
+    parser.add_argument("--custom-image", default="",
+                         help="Full custom image reference, or empty string to use pool default.")
+    parser.add_argument("--deployments-dir", default="deployments",
+                         help="Path to the deployments directory.")
+    parser.add_argument("--dev-work-pool", default="",
+                         help="Dev work pool name. When set with --tag=dev, activates dev overrides.")
+    parser.add_argument("--max-workers", type=int, default=MAX_WORKERS,
+                         help=f"Max concurrent deployments (default: {MAX_WORKERS}).")
+    parser.add_argument("--backend", default="cli", choices=["cli", "rest"],
+                         help="Deployment backend: 'cli' (prefect CLI, default) or "
+                              "'rest' (direct Prefect Cloud REST API calls).")
+    parser.add_argument("--enforce-unique-deployment-names", default="false",
+                        choices=["true", "false"],
+                        help=(
+                            "If false (default), only WARN in logs when a deployment name is used by"
+                            "more than one flow — nothing is deleted. If true, delete every duplicate"
+                            "deployment record except the one matching the flow currently resolved from"
+                            "its entrypoint, enforcing globally unique deployment names."
+                        ))
+    return parser.parse_args()
+
+
+def _run_deployment(
+    deployment_name: str,
+    index: int,
+    total: int,
+    ctx: DeploymentContext,
+) -> str | None:
+    """Run a single deployment inside a buffered log context.
+
+    Returns the deployment_name if it failed, None on success.
+    """
+    separator = "#" * 60
+    with buffered_deployment_log(deployment_name):
+        logger.info("%s [%d/%d] %s %s", separator, index, total, deployment_name, separator)
+        try:
+            apply_single_deployment(deployment_name, ctx)
+            logger.info(">>> DONE [%d/%d] %s", index, total, deployment_name)
+            return None
+        except Exception as exc:  # noqa: BLE001
+            logger.error(">>> FAILED [%d/%d] %s: %s", index, total, deployment_name, exc)
+            return deployment_name
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(message)s",
+    )
+    args = _parse_args()
+    names = [n.strip() for n in args.deployment_names.split(",") if n.strip()]
+
+    if not names:
+        logger.error("No deployment names provided.")
+        sys.exit(1)
+
+    ctx = DeploymentContext(
+        deployment_names=names,
+        enable_schedule=args.enable_schedule == "true",
+        tag=args.tag,
+        reference=args.reference,
+        repo_name=args.repo_name,
+        custom_image=args.custom_image,
+        deployments_dir=Path(args.deployments_dir),
+        dev_prefect_work_pool_name=args.dev_work_pool,
+        backend=args.backend,
+        enforce_unique_deployment_names=args.enforce_unique_deployment_names == "true",
+    )
+
+    total = len(names)
+    workers = min(args.max_workers, total)
+    logger.info(
+        "Applying %d deployment(s) with up to %d concurrent workers (backend=%s).",
+        total, workers, ctx.backend,
+    )
+
+    failed: list[str] = []
+
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = {
+            executor.submit(_run_deployment, name, i, total, ctx): name
+            for i, name in enumerate(names, start=1)
+        }
+        for future in as_completed(futures):
+            result = future.result()  # re-raises only unexpected executor errors
+            if result is not None:
+                failed.append(result)
+
+    if failed:
+        logger.error(
+            "%d deployment(s) failed: %s",
+            len(failed),
+            ", ".join(failed),
+        )
+        sys.exit(1)
+
+    logger.info("All %d deployment(s) applied successfully.", total)
+
+
+if __name__ == "__main__":
+    main()
+ 

prefect_deployments_toolkit/deployment.py

@@ -0,0 +1,250 @@
+"""Core logic for applying a single Prefect deployment."""
+
+import logging
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from . import prefect_api, prefect_cli, prefect_rest, yaml_utils
+
+logger = logging.getLogger(__name__)
+
+DEV_PREFIX = "dev--"
+
+
+@dataclass
+class DeploymentContext:
+    """All runtime configuration for the apply run."""
+
+    deployment_names: list[str]
+    enable_schedule: bool
+    tag: str
+    reference: str
+    repo_name: str
+    custom_image: str
+    deployments_dir: Path
+    dev_prefect_work_pool_name: str = ""
+    backend: str = "cli"  # "cli" or "rest"
+    enforce_unique_deployment_names: bool = False
+
+    @property
+    def is_dev(self) -> bool:
+        return bool(self.dev_prefect_work_pool_name) and self.tag == "dev"
+
+    @property
+    def is_non_default_branch(self) -> bool:
+        return self.reference not in {"main", "master"}
+
+    @property
+    def client(self):
+        """Return the backend module (prefect_cli or prefect_rest) for this run."""
+        return prefect_rest if self.backend == "rest" else prefect_cli
+
+
+def _resolve_flow_name(deployment_name: str, ctx: DeploymentContext) -> tuple[str, str | None]:
+    """Return (flow_id, flow_name) for a deployment.
+
+    If more than one flow currently has a deployment with this name:
+      - default (enforce_unique_deployment_names=False): log a warning
+        listing every duplicate flow, and proceed using the FIRST match
+        returned by the API. Nothing is deleted.
+      - enforce mode (enforce_unique_deployment_names=True): proceed the
+        same way for now — the actual cleanup of duplicates happens AFTER
+        deploy, in apply_single_deployment, once we know the flow this
+        entrypoint currently resolves to.
+    """
+    flow_ids = prefect_api.get_flow_ids_for_deployment(deployment_name)
+
+    if len(flow_ids) <= 1:
+        flow_id = flow_ids[0] if flow_ids else ""
+        flow_name = prefect_api.get_flow_name(flow_id) if flow_id else None
+        return flow_id, flow_name
+
+    flow_names = [prefect_api.get_flow_name(fid) for fid in flow_ids]
+    logger.warning(
+        "Deployment name '%s' is currently used by %d different flows: %s. "
+        "Deployment names should be globally unique.",
+        deployment_name, len(flow_ids), flow_names,
+    )
+
+    if not ctx.enforce_unique_deployment_names:
+        logger.warning(
+            "enforce-unique-deployment-names is OFF — proceeding with flow '%s' "
+            "(first match) without deleting the others. Pass "
+            "--enforce-unique-deployment-names true to clean up duplicates.",
+            flow_names[0],
+        )
+    else:
+        logger.warning(
+            "enforce-unique-deployment-names is ON — duplicates not matching the "
+            "flow resolved from this deployment's entrypoint will be deleted after deploy."
+        )
+
+    return flow_ids[0], flow_names[0]
+
+
+def _cleanup_duplicate_deployments(
+    ctx: DeploymentContext,
+    deployment_name: str,
+    current_flow_id: str,
+    current_flow_name: str,
+) -> None:
+    """Delete every deployment named `deployment_name` under a flow OTHER than
+    current_flow_id. Only called when ctx.enforce_unique_deployment_names=True.
+    """
+    flow_ids = prefect_api.get_flow_ids_for_deployment(deployment_name)
+    stale_flow_ids = [fid for fid in flow_ids if fid != current_flow_id]
+
+    for stale_flow_id in stale_flow_ids:
+        stale_flow_name = prefect_api.get_flow_name(stale_flow_id)
+        logger.warning(
+            "Deleting duplicate deployment '%s' under flow '%s' (flow_id=%s) — "
+            "keeping the one under current flow '%s' (flow_id=%s).",
+            deployment_name, stale_flow_name, stale_flow_id,
+            current_flow_name, current_flow_id,
+        )
+        ctx.client.delete_deployment(f"{stale_flow_name}/{deployment_name}")
+
+
+def _build_tags(ctx: DeploymentContext, merged_file: Path, full_name: str) -> list[str]:
+    tags = [ctx.tag, ctx.reference]
+    tags += yaml_utils.get_deployment_tags(merged_file, full_name)
+    return tags
+
+
+def _build_job_variables(
+    ctx: DeploymentContext,
+    merged_file: Path,
+    full_name: str,
+) -> dict[str, str]:
+    job_vars: dict[str, str] = {"name": full_name}
+
+    if ctx.is_non_default_branch:
+        base = f"/opt/prefect/{ctx.repo_name}-{ctx.reference}"
+        job_vars["DBT_PROJECT_DIR"] = f"{base}/src/edp_flows/models"
+        job_vars["DBT_PROFILES_DIR"] = f"{base}/src/edp_flows/models"
+        job_vars["METRICS_EXPORTER_DIR"] = f"{base}/etc"
+
+    if ctx.custom_image:
+        job_vars["image"] = ctx.custom_image
+
+    # Merge job_variables from YAML; image from YAML only wins if not already set above
+    yaml_job_vars = yaml_utils.get_job_variables(merged_file, full_name)
+    for key, value in yaml_job_vars.items():
+        if key == "image" and "image" in job_vars:
+            continue
+        job_vars[key] = str(value)
+
+    return job_vars
+
+
+def _handle_schedules(
+    ctx: DeploymentContext,
+    merged_file: Path,
+    full_name: str,
+    flow_name: str,
+) -> None:
+    deployment_has_schedules = yaml_utils.has_schedules(merged_file, full_name)
+
+    if not deployment_has_schedules:
+        logger.info("No schedules in YAML — removing any existing schedules from Prefect Cloud...")
+        schedule_ids = prefect_api.get_schedule_ids(f"{flow_name}/{full_name}")
+        for sid in schedule_ids:
+            ctx.client.delete_schedule(f"{flow_name}/{full_name}", sid)
+        if not schedule_ids:
+            logger.info("No existing schedules to remove.")
+        return
+
+    if ctx.enable_schedule:
+        logger.info("Resuming all schedules for '%s'...", full_name)
+        schedule_ids = prefect_api.get_schedule_ids(f"{flow_name}/{full_name}")
+        for sid in schedule_ids:
+            ctx.client.resume_schedule(f"{flow_name}/{full_name}", sid)
+    else:
+        logger.info("Schedules exist in YAML but enable_schedule=false — will remain paused.")
+
+
+def remove_deployment(ctx: DeploymentContext, deployment_name: str, full_name: str, flow_name: str | None) -> None:
+    """Delete a deployment that no longer exists in the YAML files."""
+    logger.info("Deployment '%s' has been removed from YAML.", deployment_name)
+    if flow_name:
+        ctx.client.delete_deployment(f"{flow_name}/{full_name}")
+    else:
+        logger.info(
+            "Deployment '%s' does not exist on Prefect Cloud (no matching flow). Skipping.",
+            full_name,
+        )
+
+
+def apply_single_deployment(deployment_name: str, ctx: DeploymentContext) -> None:
+    """Apply (create/update) or remove a single deployment."""
+    yaml_file = yaml_utils.find_deployment_file(deployment_name, ctx.deployments_dir)
+    prefix = DEV_PREFIX if ctx.is_dev else ""
+    full_name = f"{prefix}{deployment_name}"
+
+    flow_id, flow_name = _resolve_flow_name(full_name, ctx)
+
+    if yaml_file is None:
+        remove_deployment(ctx, deployment_name, full_name, flow_name)
+        return
+
+    with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tmp:
+        merged_file = Path(tmp.name)
+
+    try:
+        yaml_utils.build_merged_prefect_file(
+            ctx.deployments_dir / "prefect_base.yaml",
+            yaml_file,
+            merged_file,
+        )
+
+        if ctx.is_dev:
+            yaml_utils.apply_dev_overrides(
+                merged_file, deployment_name, DEV_PREFIX, ctx.dev_prefect_work_pool_name
+            )
+
+        yaml_utils.validate_schedule(merged_file, full_name)
+
+        if ctx.is_non_default_branch:
+            yaml_utils.set_git_clone_branch(merged_file, ctx.reference)
+
+        if ctx.enable_schedule and yaml_utils.has_schedules(merged_file, full_name):
+            yaml_utils.set_schedules_active(merged_file, full_name, active=True)
+
+        tags = _build_tags(ctx, merged_file, full_name)
+        job_vars = _build_job_variables(ctx, merged_file, full_name)
+        logger.info("Tags: %s", tags)
+        logger.info("Job variables: %s", job_vars)
+        logger.info("Backend: %s", ctx.backend)
+
+        ctx.client.deploy(full_name, tags, job_vars, merged_file)
+        time.sleep(0.01)
+
+        # Detect flow rename: if flow_id changed post-deploy, clean up the old deployment
+        new_flow_ids = prefect_api.get_flow_ids_for_deployment(full_name)
+        new_flow_id = new_flow_ids[0] if new_flow_ids else ""
+        if flow_id and new_flow_id and new_flow_id != flow_id:
+            logger.info("Flow changed post-deploy — removing stale deployment under old flow.")
+            ctx.client.delete_deployment(f"{flow_name}/{full_name}")
+            flow_name = prefect_api.get_flow_name(new_flow_id)
+
+        if not flow_name:
+            logger.info("New deployment — retrieving flow name from Prefect Cloud post-deploy...")
+            post_ids = prefect_api.get_flow_ids_for_deployment(full_name)
+            new_flow_id = post_ids[0] if post_ids else ""
+            flow_name = prefect_api.get_flow_name(new_flow_id) if new_flow_id else None
+
+        # If duplicates existed under other flows and enforcement is on,
+        # clean up everything except the deployment matching the current flow.
+        if ctx.enforce_unique_deployment_names and flow_name and new_flow_id:
+            _cleanup_duplicate_deployments(ctx, full_name, new_flow_id, flow_name)
+
+        if flow_name:
+            _handle_schedules(ctx, merged_file, full_name, flow_name)
+        else:
+            logger.warning("Could not resolve flow name post-deploy — skipping schedule management.")
+
+    finally:
+        merged_file.unlink(missing_ok=True)
+        time.sleep(0.01)

prefect_deployments_toolkit/deployment_loader.py

@@ -0,0 +1,139 @@
+"""Load and normalize Prefect deployment configs from local filesystem or git refs."""
+
+import json
+import logging
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+# Keys excluded from deployment comparison — they don't affect runtime behaviour
+_IGNORED_KEYS = frozenset({"pull"})
+
+
+def normalize_deployment(deployment: dict) -> dict:
+    """Return deployment config with non-comparable keys removed."""
+    return {k: v for k, v in deployment.items() if k not in _IGNORED_KEYS}
+
+
+def get_deployments_from_source(
+    source: str,
+    source_name: str,
+    deployments_dir: str = "deployments",
+) -> dict[str, dict]:
+    """Load all deployments from a git ref or the local filesystem.
+
+    Parameters
+    ----------
+    source:
+        Git reference (e.g. "origin/main", "HEAD~1") or "local" for the
+        working-tree filesystem.
+    source_name:
+        Human-readable label used in log messages.
+    deployments_dir:
+        Path to the deployments directory.
+
+    Returns
+    -------
+    dict mapping deployment name → normalised config dict.
+    """
+    logger.info("Getting deployments in %s...", source_name)
+    deployments: dict[str, dict] = {}
+    is_local = source == "local"
+    base_file_path = f"{deployments_dir}/prefect_base.yaml"
+
+    try:
+        # --- load prefect_base.yaml ---
+        try:
+            if is_local:
+                base_path = Path(base_file_path)
+                base_content = base_path.read_text() if base_path.exists() else ""
+            else:
+                base_content = subprocess.check_output(
+                    ["git", "show", f"{source}:{base_file_path}"],  # noqa: S607
+                ).decode()
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            logger.warning("prefect_base.yaml not found in %s", source_name)
+            base_content = ""
+
+        # --- enumerate deployment YAML files ---
+        if is_local:
+            deployment_dir = Path(deployments_dir)
+            if not deployment_dir.exists():
+                logger.info("No %s folder found in %s", deployments_dir, source_name)
+                return deployments
+            yaml_files = [
+                str(f.relative_to("."))
+                for f in deployment_dir.rglob("*.yaml")
+                if f.name != "prefect_base.yaml"
+            ]
+        else:
+            raw = subprocess.check_output(
+                ["git", "ls-tree", "-r", "--name-only", source, f"{deployments_dir}/"],  # noqa: S607
+                stderr=subprocess.DEVNULL,
+            ).decode().strip().split("\n")
+            yaml_files = [
+                f for f in raw
+                if f
+                and f.endswith(".yaml")
+                and f.startswith(f"{deployments_dir}/")
+                and "prefect_base.yaml" not in f
+            ]
+
+        logger.info("Found %d YAML files in %s folder", len(yaml_files), deployments_dir)
+
+        # --- parse each file ---
+        for yaml_file in yaml_files:
+            try:
+                if is_local:
+                    deployment_content = Path(yaml_file).read_text()
+                else:
+                    deployment_content = subprocess.check_output(
+                        ["git", "show", f"{source}:{yaml_file}"],  # noqa: S607
+                    ).decode()
+
+                parsed = yaml.safe_load(base_content + "\n" + deployment_content)
+
+                if isinstance(parsed, dict) and "deployments" in parsed:
+                    deployment_list = parsed["deployments"]
+                elif isinstance(parsed, list):
+                    deployment_list = parsed
+                else:
+                    deployment_list = None
+
+                if not isinstance(deployment_list, list):
+                    continue
+
+                for d in deployment_list:
+                    if not isinstance(d, dict):
+                        logger.error(
+                            "Invalid deployment entry in %s: expected dict, got %s: %s",
+                            yaml_file, type(d).__name__, d,
+                        )
+                        sys.exit(1)
+                    name = d.get("name")
+                    if not isinstance(name, str) or not name:
+                        logger.error(
+                            "Deployment in %s is missing a valid 'name' field: %s",
+                            yaml_file, d,
+                        )
+                        sys.exit(1)
+                    deployments[name] = normalize_deployment(d)
+
+                logger.debug("Loaded %d deployments from %s", len(deployment_list), yaml_file)
+
+            except yaml.YAMLError as exc:
+                logger.error("Invalid YAML syntax in %s: %s", yaml_file, exc)
+                sys.exit(1)
+            except (subprocess.CalledProcessError, OSError) as exc:
+                logger.warning("Failed to load deployments from %s: %s", yaml_file, exc)
+                continue
+
+    except subprocess.CalledProcessError:
+        logger.info("No %s folder found in %s", deployments_dir, source_name)
+
+    logger.debug("Deployments in %s:\n%s", source_name, json.dumps(deployments, indent=2))
+    return deployments

prefect_deployments_toolkit/get_modified_deployments.py

@@ -0,0 +1,110 @@
+"""Retrieve all deployments added, modified, or removed in the current branch."""
+
+import argparse
+import logging
+import os
+import subprocess
+import sys
+
+from .deployment_loader import get_deployments_from_source
+
+logger = logging.getLogger(__name__)
+
+
+def get_base_branch_deployments(
+    base_branch: str,
+    deployments_dir: str = "deployments",
+) -> dict[str, dict]:
+    """Get all deployments defined in the base branch."""
+    try:
+        subprocess.run(
+            ["git", "fetch", "origin", base_branch],  # noqa: S607
+            check=True,
+            stderr=subprocess.DEVNULL,
+        )
+    except subprocess.CalledProcessError:
+        logger.warning("Failed to fetch base branch '%s'", base_branch)
+    return get_deployments_from_source(
+        f"origin/{base_branch}",
+        f"the base branch '{base_branch}'",
+        deployments_dir,
+    )
+
+
+def get_pr_branch_deployments(deployments_dir: str = "deployments") -> dict[str, dict]:
+    """Get all deployments defined in the current branch (local filesystem)."""
+    return get_deployments_from_source("local", "the current branch", deployments_dir)
+
+
+def get_previous_commit_deployments(deployments_dir: str = "deployments") -> dict[str, dict]:
+    """Get all deployments defined in the previous commit (HEAD~1)."""
+    result = subprocess.run(
+        ["git", "rev-parse", "--verify", "HEAD~1"],
+        capture_output=True,
+    )
+    if result.returncode != 0:
+        logger.warning("HEAD~1 does not exist (possibly initial commit). Returning empty.")
+        return {}
+    return get_deployments_from_source("HEAD~1", "the previous commit", deployments_dir)
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--modified-by",
+        choices=["pull_request_target", "pull_request", "push"],
+        help="GitHub Actions event type driving this run.",
+    )
+    parser.add_argument(
+        "--base-ref",
+        required=True,
+        help="Base branch name (e.g. master, main, rc_3.3.1).",
+    )
+    parser.add_argument(
+        "--deployments-dir",
+        default="deployments",
+        help="Path to the deployments directory (default: deployments).",
+    )
+    args = parser.parse_args()
+
+    is_commit_compare = args.modified_by in ("pull_request_target", "push")
+
+    previous = (
+        get_previous_commit_deployments(args.deployments_dir)
+        if is_commit_compare
+        else get_base_branch_deployments(args.base_ref, args.deployments_dir)
+    )
+    current = (
+        get_base_branch_deployments(args.base_ref, args.deployments_dir)
+        if is_commit_compare
+        else get_pr_branch_deployments(args.deployments_dir)
+    )
+
+    new_or_modified = [
+        name for name, cfg in current.items()
+        if name not in previous or cfg != previous[name]
+    ]
+    removed = [name for name in previous if name not in current]
+
+    if not new_or_modified and not removed:
+        logger.info("No new, modified, or removed deployments found.")
+        sys.exit(0)
+
+    all_changed = ",".join(new_or_modified + removed)
+    logger.info("Found the following new, modified, or removed deployments: '%s'.", all_changed)
+    logger.info("New or modified: '%s'.", ",".join(new_or_modified))
+    logger.info("Removed: '%s'.", ",".join(removed))
+    print(all_changed)
+
+    github_env = os.getenv("GITHUB_ENV")
+    if os.getenv("GITHUB_ACTION") and github_env:
+        with open(github_env, "a") as f:
+            f.write(f"DEPLOYMENT_NAMES={all_changed}\n")
+            f.write(f"NEW_OR_MODIFIED_DEPLOYMENT_NAMES={','.join(new_or_modified)}\n")
+            f.write(f"REMOVED_DEPLOYMENT_NAMES={','.join(removed)}\n")
+
+
+if __name__ == "__main__":
+    main()