gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/commands/checkout.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from .constants import GLUE_SET_FILE
+from .helpers import (
+    _apply_saved_params_to_config_path,
+    _clear_checked_out_jobs,
+    _examples_epilog,
+    _load_config_index,
+    _resolve_single_job_name,
+    _resolve_checkout_job_name,
+    _save_checked_out_jobs,
+    _set_checked_out_local_setup,
+)
+from ..cli import app
+
+
+@app.command(
+    "checkout",
+    epilog=_examples_epilog(
+        "gluekit checkout my-job",
+        "gluekit checkout --local",
+        "gluekit checkout --clear",
+    ),
+)
+def checkout_job(
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help="Glue job name to check out for later commands.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+    profile: Optional[str] = typer.Option(
+        None,
+        "--profile",
+        "-p",
+        help=(
+            "AWS CLI credential profile name to store as the active gluekit profile "
+            "scope for saved params."
+        ),
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show gluekit profile-scoped config updates without writing files.",
+    ),
+    clear: bool = typer.Option(
+        False,
+        "--clear",
+        help="Remove the active checkout selection without deleting saved params or local files.",
+    ),
+    local: bool = typer.Option(
+        False,
+        "--local",
+        help="Check out the single local development settings file in .gluekit/local.json.",
+    ),
+) -> None:
+    """Replace the active local checkout selection."""
+    if clear:
+        if job_name or local:
+            raise typer.BadParameter("Use either <job-name>, --local, or --clear.")
+        if dry_run:
+            typer.echo(f"Would clear active checkout selection in {GLUE_SET_FILE}.")
+            return
+        _clear_checked_out_jobs()
+        typer.echo(f"Cleared active checkout selection in {GLUE_SET_FILE}.")
+        return
+
+    if local:
+        if job_name:
+            raise typer.BadParameter("Use either <job-name> or --local, not both.")
+        if dry_run:
+            typer.echo(f"Would check out local settings in {GLUE_SET_FILE}.")
+            return
+        _set_checked_out_local_setup("local")
+        typer.echo(f"Checked out local settings in {GLUE_SET_FILE}.")
+        return
+
+    if not job_name and profile:
+        job_name = _resolve_single_job_name(None, "checkout --profile")
+
+    if not job_name:
+        raise typer.BadParameter("Provide <job-name>, use --local, or use --clear.")
+
+    resolved_job_name, source = _resolve_checkout_job_name(
+        job_name, config_dir=config_dir
+    )
+    if not dry_run:
+        _save_checked_out_jobs(
+            [resolved_job_name], resolved_job_name, source=source, profile=profile
+        )
+    action = "Would check out" if dry_run else "Checked out"
+    typer.echo(f"{action} 1 job(s) from {source} in {GLUE_SET_FILE}:")
+    typer.echo(f"- {resolved_job_name}")
+    if profile:
+        typer.echo(f"Active gluekit profile scope: {profile}")
+
+    config_index = _load_config_index(config_dir)
+    config_entry = config_index.get(resolved_job_name)
+    if config_entry:
+        _apply_saved_params_to_config_path(
+            config_path=config_entry["config_path"],
+            config_data=config_entry["config"],
+            job_name=resolved_job_name,
+            profile=profile,
+            dry_run=dry_run,
+        )

gluekit/commands/clone.py

@@ -0,0 +1,516 @@
+from __future__ import annotations
+
+import csv
+import copy
+import json
+import re
+import shutil
+import subprocess
+import tarfile
+import uuid
+import zipfile
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from email.parser import Parser
+from fnmatch import fnmatch
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import (
+    download_glue_job_files,
+    list_glue_jobs,
+    normalize_glue_config_data,
+    convert_script_to_notebook,
+    convert_notebook_to_script,
+    _resolve_notebook_path,
+    upload_glue_job_files_from_config,
+)
+from ..job_mgmt.magics import build_magic_cell_sources as _build_magic_cell_sources
+
+from .constants import *
+from .helpers import *
+from .edit import _parse_csv_list
+from ..cli import app, glue_config_app
+
+
+def _rewrite_clone_path_value(
+    value: str,
+    *,
+    source_job: str,
+    target_job: str,
+    fallback_suffixes: tuple[str, ...] = (),
+) -> str:
+    text = value.strip()
+    if not text:
+        return value
+
+    source_slug = slugify(source_job)
+    target_slug = slugify(target_job)
+    rewritten = text
+    replacements = (
+        (source_job, target_job),
+        (source_slug, target_slug),
+    )
+    for old_value, new_value in replacements:
+        if old_value and old_value in rewritten:
+            rewritten = rewritten.replace(old_value, new_value)
+            break
+    else:
+        for suffix in fallback_suffixes:
+            if not rewritten.endswith(suffix):
+                continue
+            head, _tail = (
+                rewritten.rsplit("/", 1) if "/" in rewritten else ("", rewritten)
+            )
+            candidate = f"{target_slug}{suffix}"
+            rewritten = f"{head}/{candidate}" if head else candidate
+            break
+
+    return rewritten
+
+
+def _should_rewrite_cloned_dependency(local_path: Path, source_job: str) -> bool:
+    normalized_parts = {part.lower() for part in local_path.parts}
+    source_slug = slugify(source_job).lower()
+    source_name = source_job.lower()
+    if source_slug in normalized_parts or source_name in normalized_parts:
+        return True
+
+    stem = local_path.stem.lower()
+    return source_slug in stem or source_name in stem
+
+
+def _prepare_clone_destination(path: Path, *, force: bool) -> None:
+    if not path.exists():
+        return
+    if not force:
+        raise typer.BadParameter(
+            f"Refusing to overwrite existing path without --force: {path}"
+        )
+    if path.is_dir():
+        shutil.rmtree(path)
+    else:
+        path.unlink()
+
+
+def _copy_cloned_artifact(source_path: Path, target_path: Path, *, force: bool) -> None:
+    _prepare_clone_destination(target_path, force=force)
+    target_path.parent.mkdir(parents=True, exist_ok=True)
+    if source_path.is_dir():
+        shutil.copytree(source_path, target_path)
+    else:
+        shutil.copy2(source_path, target_path)
+
+
+def _rewrite_extra_py_files(
+    existing_value: Optional[str],
+    s3_rewrites: dict[str, str],
+) -> Optional[str]:
+    if existing_value is None:
+        return None
+    updated_items = [
+        s3_rewrites.get(item, item) for item in _parse_csv_list(existing_value)
+    ]
+    return ",".join(_format_csv_item(item) for item in updated_items)
+
+
+def _rewrite_source_control_file_mappings(
+    entries: Any,
+    *,
+    field_name: str,
+    source_job: str,
+    target_job: str,
+    rewrite_local_paths: bool,
+    target_script_location: Any,
+    local_script_path_for_s3: Path,
+) -> dict[str, str]:
+    if entries is None:
+        return {}
+    if not isinstance(entries, list):
+        raise typer.BadParameter(f"SourceControlDetails.{field_name} must be a list.")
+
+    s3_rewrites: dict[str, str] = {}
+    for entry in entries:
+        if not isinstance(entry, dict):
+            raise typer.BadParameter(f"{field_name} entries must be objects.")
+        local_value = entry.get("LocalPath")
+        s3_value = entry.get("S3Path")
+        if not isinstance(local_value, str) or not isinstance(s3_value, str):
+            raise typer.BadParameter(
+                f"{field_name} entries must include LocalPath and S3Path."
+            )
+
+        source_local_path = Path(local_value)
+        target_local_path = source_local_path
+        if rewrite_local_paths and _should_rewrite_cloned_dependency(
+            source_local_path, source_job
+        ):
+            target_local_value = _rewrite_clone_path_value(
+                local_value,
+                source_job=source_job,
+                target_job=target_job,
+            )
+            target_local_path = Path(target_local_value)
+            entry["LocalPath"] = target_local_path.as_posix()
+
+        if not rewrite_local_paths:
+            target_s3_path = _rewrite_clone_path_value(
+                s3_value,
+                source_job=source_job,
+                target_job=target_job,
+            )
+        elif isinstance(target_script_location, str) and target_script_location.strip():
+            target_s3_path = _derive_s3_target(
+                target_local_path,
+                target_job,
+                target_script_location,
+                local_script_path_for_s3,
+            )
+            if target_local_path.is_dir() and not target_s3_path.endswith(".zip"):
+                target_s3_path = f"{target_s3_path}.zip"
+        else:
+            target_s3_path = _rewrite_clone_path_value(
+                s3_value,
+                source_job=source_job,
+                target_job=target_job,
+            )
+
+        entry["S3Path"] = target_s3_path
+        s3_rewrites[s3_value] = target_s3_path
+
+    return s3_rewrites
+
+
+def _rewrite_glue_job_config(
+    source_config: dict[str, Any],
+    *,
+    source_job: str,
+    target_job: str,
+    rewrite_local_paths: bool,
+) -> dict[str, Any]:
+    renamed_config = copy.deepcopy(source_config)
+    renamed_config["Name"] = target_job
+    renamed_config.pop("LastModifiedOn", None)
+
+    source_sc = source_config.get("SourceControlDetails", {}) or {}
+    if not isinstance(source_sc, dict):
+        source_sc = {}
+    target_sc = renamed_config.setdefault("SourceControlDetails", {})
+    if not isinstance(target_sc, dict):
+        raise typer.BadParameter("SourceControlDetails must be an object.")
+
+    source_command = source_config.get("Command", {}) or {}
+    if not isinstance(source_command, dict):
+        source_command = {}
+    target_command = renamed_config.setdefault("Command", {})
+    if not isinstance(target_command, dict):
+        raise typer.BadParameter("Command must be an object.")
+
+    source_local_script_path = Path(
+        source_sc.get("ScriptLocation")
+        or source_sc.get("LocalPath")
+        or f"glue/scripts/{slugify(source_job)}.py"
+    )
+    local_script_path_for_s3 = source_local_script_path
+
+    if rewrite_local_paths:
+        target_local_script_value = _rewrite_clone_path_value(
+            source_sc.get("ScriptLocation")
+            or source_sc.get("LocalPath")
+            or f"glue/scripts/{slugify(target_job)}.py",
+            source_job=source_job,
+            target_job=target_job,
+            fallback_suffixes=(".py",),
+        )
+        local_script_path_for_s3 = Path(target_local_script_value)
+
+        if isinstance(source_sc.get("ScriptLocation"), str):
+            target_sc["ScriptLocation"] = _rewrite_clone_path_value(
+                source_sc["ScriptLocation"],
+                source_job=source_job,
+                target_job=target_job,
+                fallback_suffixes=(".py",),
+            )
+        if isinstance(source_sc.get("LocalPath"), str):
+            target_sc["LocalPath"] = _rewrite_clone_path_value(
+                source_sc["LocalPath"],
+                source_job=source_job,
+                target_job=target_job,
+                fallback_suffixes=(".py",),
+            )
+        if isinstance(source_sc.get("NotebookLocation"), str):
+            target_sc["NotebookLocation"] = _rewrite_clone_path_value(
+                source_sc["NotebookLocation"],
+                source_job=source_job,
+                target_job=target_job,
+                fallback_suffixes=(".ipynb",),
+            )
+        if isinstance(source_sc.get("NotebookPath"), str):
+            target_sc["NotebookPath"] = _rewrite_clone_path_value(
+                source_sc["NotebookPath"],
+                source_job=source_job,
+                target_job=target_job,
+                fallback_suffixes=(".ipynb",),
+            )
+
+    if isinstance(source_command.get("ScriptLocation"), str):
+        target_command["ScriptLocation"] = _rewrite_clone_path_value(
+            source_command["ScriptLocation"],
+            source_job=source_job,
+            target_job=target_job,
+            fallback_suffixes=(".py",),
+        )
+
+    s3_rewrites: dict[str, str] = {}
+    target_script_location = target_command.get("ScriptLocation")
+    for field_name in ("AdditionalPythonFiles", "ExtraPyFiles"):
+        s3_rewrites.update(
+            _rewrite_source_control_file_mappings(
+                target_sc.get(field_name),
+                field_name=field_name,
+                source_job=source_job,
+                target_job=target_job,
+                rewrite_local_paths=rewrite_local_paths,
+                target_script_location=target_script_location,
+                local_script_path_for_s3=local_script_path_for_s3,
+            )
+        )
+
+    default_args = renamed_config.get("DefaultArguments")
+    if isinstance(default_args, dict) and s3_rewrites:
+        rewritten_extra = _rewrite_extra_py_files(
+            default_args.get("--extra-py-files"),
+            s3_rewrites,
+        )
+        if rewritten_extra is not None:
+            default_args["--extra-py-files"] = rewritten_extra
+
+    return renamed_config
+
+
+def _collect_source_control_file_local_renames(
+    source_config: dict[str, Any],
+    target_config: dict[str, Any],
+) -> list[tuple[Path, Path]]:
+    renames: list[tuple[Path, Path]] = []
+    source_sc = source_config.get("SourceControlDetails", {}) or {}
+    target_sc = target_config.get("SourceControlDetails", {}) or {}
+    for field_name in ("AdditionalPythonFiles", "ExtraPyFiles"):
+        source_entries = source_sc.get(field_name) or []
+        target_entries = target_sc.get(field_name) or []
+        if not isinstance(source_entries, list) or not isinstance(target_entries, list):
+            continue
+        for source_entry, target_entry in zip(source_entries, target_entries):
+            if not isinstance(source_entry, dict) or not isinstance(target_entry, dict):
+                continue
+            source_local = source_entry.get("LocalPath")
+            target_local = target_entry.get("LocalPath")
+            if not isinstance(source_local, str) or not isinstance(target_local, str):
+                continue
+            if source_local != target_local:
+                renames.append((Path(source_local), Path(target_local)))
+    return renames
+
+
+def _move_local_artifact(source_path: Path, target_path: Path) -> None:
+    if not source_path.exists() or source_path == target_path:
+        return
+    _prepare_clone_destination(target_path, force=True)
+    target_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.move(str(source_path), str(target_path))
+
+
+def _build_glue_clone_plan(
+    source_config: dict[str, Any],
+    *,
+    source_job: str,
+    target_job: str,
+) -> tuple[dict[str, Any], list[tuple[Path, Path, str]], list[str]]:
+    cloned_config = _rewrite_glue_job_config(
+        source_config,
+        source_job=source_job,
+        target_job=target_job,
+        rewrite_local_paths=True,
+    )
+
+    source_sc = source_config.get("SourceControlDetails", {}) or {}
+    if not isinstance(source_sc, dict):
+        source_sc = {}
+    target_sc = cloned_config.setdefault("SourceControlDetails", {})
+    if not isinstance(target_sc, dict):
+        raise typer.BadParameter("SourceControlDetails must be an object.")
+
+    source_command = source_config.get("Command", {}) or {}
+    if not isinstance(source_command, dict):
+        source_command = {}
+    target_command = cloned_config.setdefault("Command", {})
+    if not isinstance(target_command, dict):
+        raise typer.BadParameter("Command must be an object.")
+
+    source_local_script_path = Path(
+        source_sc.get("ScriptLocation")
+        or source_sc.get("LocalPath")
+        or f"glue/scripts/{slugify(source_job)}.py"
+    )
+    target_local_script_value = (
+        target_sc.get("ScriptLocation")
+        or target_sc.get("LocalPath")
+        or f"glue/scripts/{slugify(target_job)}.py"
+    )
+    target_local_script_path = Path(target_local_script_value)
+
+    source_notebook_path = Path(
+        source_sc.get("NotebookLocation")
+        or source_sc.get("NotebookPath")
+        or _resolve_notebook_path(source_local_script_path)
+    )
+    target_notebook_path = Path(
+        target_sc.get("NotebookLocation")
+        or target_sc.get("NotebookPath")
+        or _rewrite_clone_path_value(
+            _resolve_notebook_path(source_local_script_path),
+            source_job=source_job,
+            target_job=target_job,
+            fallback_suffixes=(".ipynb",),
+        )
+    )
+
+    artifact_copies: list[tuple[Path, Path, str]] = []
+    warnings: list[str] = []
+    if source_local_script_path.exists():
+        artifact_copies.append(
+            (source_local_script_path, target_local_script_path, "script")
+        )
+    else:
+        warnings.append(
+            f"Source script not found, skipping copy: {source_local_script_path}"
+        )
+
+    if source_notebook_path.exists():
+        artifact_copies.append((source_notebook_path, target_notebook_path, "notebook"))
+    elif source_sc.get("NotebookLocation") or source_sc.get("NotebookPath"):
+        warnings.append(
+            f"Source notebook not found, skipping copy: {source_notebook_path}"
+        )
+
+    for (
+        source_local_path,
+        target_local_path,
+    ) in _collect_source_control_file_local_renames(
+        source_config,
+        cloned_config,
+    ):
+        if source_local_path.exists():
+            artifact_copies.append((source_local_path, target_local_path, "dependency"))
+        else:
+            warnings.append(
+                f"Dependency source not found, keeping cloned config reference only: {source_local_path}"
+            )
+
+    return cloned_config, artifact_copies, warnings
+
+
+@app.command(
+    "clone",
+    no_args_is_help=True,
+    help="Clone local Glue job configurations and artifacts.",
+    epilog=_examples_epilog(
+        "gluekit clone job source-job target-job",
+        "gluekit clone source-job target-job",
+        "gluekit clone source-job target-job --dry-run",
+        "gluekit clone source-job target-job --force",
+    ),
+)
+def clone_job(
+    args: list[str] = typer.Argument(
+        ...,
+        help="Use either <source-job> <target-job> or job <source-job> <target-job>.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be cloned without writing files.",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Overwrite existing target config and cloned local artifacts.",
+    ),
+) -> None:
+    """Clone one local Glue job config and its local artifacts."""
+    if len(args) == 3:
+        clone_kind, source_job, target_job = args
+        if clone_kind != "job":
+            raise typer.BadParameter(
+                "Use 'gluekit clone <source-job> <target-job>' or 'gluekit clone job <source-job> <target-job>'."
+            )
+    elif len(args) == 2:
+        source_job, target_job = args
+    else:
+        raise typer.BadParameter(
+            "Use 'gluekit clone <source-job> <target-job>' or 'gluekit clone job <source-job> <target-job>'."
+        )
+
+    normalized_source = source_job.strip()
+    normalized_target = target_job.strip()
+    if not normalized_source or not normalized_target:
+        raise typer.BadParameter(
+            "Source and target job names must be non-empty strings."
+        )
+    if normalized_source == normalized_target:
+        raise typer.BadParameter("Source and target job names must differ.")
+
+    config_index = _load_config_index(config_dir)
+    source_entry = config_index.get(normalized_source)
+    if not source_entry:
+        raise typer.BadParameter(f'No config files matched "{normalized_source}".')
+    if normalized_target in config_index and not force:
+        raise typer.BadParameter(
+            f'Glue job "{normalized_target}" already exists. Use --force to overwrite it.'
+        )
+
+    target_config_path = config_dir / f"{slugify(normalized_target)}.json"
+    source_config_path: Path = source_entry["config_path"]
+    if target_config_path == source_config_path:
+        raise typer.BadParameter(
+            "Target job resolves to the same config path as the source job. Use a distinct target name."
+        )
+    if target_config_path.exists() and not force:
+        raise typer.BadParameter(
+            f"Refusing to overwrite existing config without --force: {target_config_path}"
+        )
+
+    cloned_config, artifact_copies, warnings = _build_glue_clone_plan(
+        source_entry["config"],
+        source_job=normalized_source,
+        target_job=normalized_target,
+    )
+
+    if dry_run:
+        typer.echo(f"Would clone config: {source_config_path} -> {target_config_path}")
+        for source_path, target_path, kind in artifact_copies:
+            typer.echo(f"Would copy {kind}: {source_path} -> {target_path}")
+        for message in warnings:
+            typer.echo(message)
+        return
+
+    config_dir.mkdir(parents=True, exist_ok=True)
+    for source_path, target_path, _kind in artifact_copies:
+        _copy_cloned_artifact(source_path, target_path, force=force)
+    target_config_path.write_text(json.dumps(cloned_config, indent=4))
+
+    typer.echo(f"Cloned Glue job '{normalized_source}' to '{normalized_target}'.")
+    typer.echo(f"Wrote config: {target_config_path}")
+    for source_path, target_path, kind in artifact_copies:
+        typer.echo(f"Copied {kind}: {source_path} -> {target_path}")
+    for message in warnings:
+        typer.echo(message)