gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/commands/pull.py

@@ -0,0 +1,374 @@
+from __future__ import annotations
+
+import copy
+import json
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import (
+    download_glue_job_files,
+    list_glue_jobs,
+    normalize_glue_config_data,
+)
+
+from .constants import PULL_COMPONENT_ALIASES, PULL_COMPONENTS
+from .helpers import (
+    _examples_epilog,
+    _get_checked_out_jobs,
+    _get_checked_out_profile,
+    _get_config_field_path,
+    _get_local_last_modified,
+    _get_saved_params_for_job,
+    _load_config_index,
+    _load_glue_set_store,
+    _parse_datetime,
+    _profile_param_expected_fields,
+    _set_saved_scope,
+    _validate_profile_mappings_align_with_config,
+    _write_glue_job_list_csv,
+)
+from .clone import (
+    _rewrite_glue_job_config,
+    _collect_source_control_file_local_renames,
+    _move_local_artifact,
+)
+from .edit import _normalize_component_filters
+from .init import detect_profile_s3_params
+from ..cli import app
+
+
+def _resolve_pull_local_path(
+    default_local_path: str, saved_params: dict[str, Any]
+) -> str:
+    for key in ("script_location", "local_script_path", "local_path"):
+        value = saved_params.get(key)
+        if isinstance(value, str) and value.strip():
+            return value
+    return default_local_path
+
+
+def _direct_profile_job_params(job_name: str, profile: Optional[str]) -> dict[str, Any]:
+    if not profile:
+        return {}
+    store = _load_glue_set_store()
+    profile_store = store.get("profiles", {}).get(profile, {})
+    if not isinstance(profile_store, dict):
+        return {}
+    params = profile_store.get("jobs", {}).get(job_name, {})
+    return dict(params) if isinstance(params, dict) else {}
+
+
+def _other_profile_job_params(
+    job_name: str, profile: Optional[str]
+) -> dict[str, dict[str, Any]]:
+    store = _load_glue_set_store()
+    profiles = store.get("profiles", {})
+    if not isinstance(profiles, dict):
+        return {}
+    result: dict[str, dict[str, Any]] = {}
+    for profile_name, profile_store in profiles.items():
+        if not isinstance(profile_name, str) or profile_name == profile:
+            continue
+        if not isinstance(profile_store, dict):
+            continue
+        params = profile_store.get("jobs", {}).get(job_name, {})
+        if isinstance(params, dict) and params:
+            result[profile_name] = dict(params)
+    return result
+
+
+def _detect_profile_params_from_pull(
+    *,
+    job_name: str,
+    profile: str,
+    baseline_config: dict[str, Any],
+    pulled_config: dict[str, Any],
+    other_profile_params: Mapping[str, dict[str, Any]],
+    dry_run: bool,
+) -> dict[str, Any]:
+    detected: dict[str, Any] = {}
+
+    role = pulled_config.get("Role")
+    if isinstance(role, str) and role.strip():
+        detected["Role"] = role.strip()
+
+    for params in other_profile_params.values():
+        for key in params:
+            for field_path, _expected in _profile_param_expected_fields(
+                key, params[key]
+            ):
+                value = _get_config_field_path(pulled_config, field_path)
+                if value is not None:
+                    detected[field_path] = value
+
+    if baseline_config:
+        s3_result = detect_profile_s3_params(
+            baseline_config=baseline_config,
+            target_config=pulled_config,
+            match_threshold=85,
+        )
+        detected.update(s3_result.params)
+
+    if not detected:
+        return {}
+
+    if dry_run:
+        typer.echo(f"Would save detected profile mappings for {profile}: {job_name}")
+    else:
+        _set_saved_scope(detected, job_name, False, profile=profile)
+        typer.echo(f"Saved detected profile mappings for {profile}: {job_name}")
+    for key, value in sorted(detected.items()):
+        typer.echo(f"- {key}: {value}")
+    return detected
+
+
+def _sync_profile_mappings_after_pull(
+    *,
+    job_name: str,
+    profile: Optional[str],
+    baseline_config: dict[str, Any],
+    pulled_config: dict[str, Any],
+    dry_run: bool,
+) -> None:
+    if not profile:
+        return
+
+    active_params = _direct_profile_job_params(job_name, profile)
+    if active_params:
+        _validate_profile_mappings_align_with_config(
+            job_name=job_name,
+            config_data=pulled_config,
+            profile=profile,
+            success_message="Profile mappings align with pulled config.",
+        )
+        return
+
+    other_params = _other_profile_job_params(job_name, profile)
+    if not other_params:
+        return
+
+    _detect_profile_params_from_pull(
+        job_name=job_name,
+        profile=profile,
+        baseline_config=baseline_config,
+        pulled_config=pulled_config,
+        other_profile_params=other_params,
+        dry_run=dry_run,
+    )
+
+
+@app.command(
+    "pull",
+    epilog=_examples_epilog(
+        "gluekit pull my-job --profile my-sso-profile",
+        "gluekit pull source-job target-job --profile my-sso-profile",
+        'gluekit pull "my-job-\\*" --force --include config,script --exclude notebook',
+        'gluekit pull "\\*" --dry-run --profile my-sso-profile --no-auto-login',
+    ),
+)
+def glue_pull(
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help='Glue job name or pattern to pull. Use "*" for all jobs.',
+    ),
+    target_job: Optional[str] = typer.Argument(
+        None,
+        help="Optional local job name to write pulled artifacts to.",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        "-f",
+        help="Download even if the local config is newer or equal to LastModifiedOn.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be downloaded without writing files.",
+    ),
+    include: Optional[list[str]] = typer.Option(
+        None,
+        "--include",
+        "-i",
+        help="Include only specific components (config, script, notebook, extra-py-files, extra-files).",
+    ),
+    exclude: Optional[list[str]] = typer.Option(
+        None,
+        "--exclude",
+        "-x",
+        help="Exclude specific components (config, script, notebook, extra-py-files, extra-files).",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+    write_job_list: bool = typer.Option(
+        True,
+        "--write-job-list/--no-write-job-list",
+        help="Write glue/glue_full_job_list.csv during the run.",
+    ),
+    profile: Optional[str] = typer.Option(
+        None,
+        "--profile",
+        "-p",
+        help="AWS CLI credential profile used for real AWS Glue/S3 API calls; not a gluekit local test profile.",
+    ),
+    auto_login: bool = typer.Option(
+        True,
+        "--auto-login/--no-auto-login",
+        help="For real AWS profiles, automatically run 'aws sso login' when credentials are missing or expired.",
+    ),
+) -> None:
+    """Download real AWS Glue job artifacts and config into the local workspace."""
+    active_profile = profile or _get_checked_out_profile()
+    requested_job_name = job_name or "*"
+    if target_job and requested_job_name == "*":
+        raise typer.BadParameter(
+            "glue pull <target-job> requires a single source job name."
+        )
+
+    components = _normalize_component_filters(
+        include,
+        exclude,
+        allowed=PULL_COMPONENTS,
+        aliases=PULL_COMPONENT_ALIASES,
+        context_label="glue pull",
+    )
+    jobs = list_glue_jobs(profile_name=active_profile, auto_login=auto_login)
+    if write_job_list:
+        Path("glue").mkdir(parents=True, exist_ok=True)
+        jobs_sorted = sorted(
+            jobs,
+            key=lambda job: (
+                _parse_datetime(job.get("LastModifiedOn"))
+                or datetime.min.replace(tzinfo=timezone.utc)
+            ),
+            reverse=True,
+        )
+        _write_glue_job_list_csv(jobs_sorted, Path("glue/glue_full_job_list.csv"))
+
+    checked_out_jobs = _get_checked_out_jobs()
+    if requested_job_name == "*" and checked_out_jobs:
+        selected_jobs = [job for job in jobs if job.get("Name") in checked_out_jobs]
+        typer.echo(f"Using active checkout selection ({len(selected_jobs)} job(s)).")
+    elif requested_job_name == "*":
+        selected_jobs = jobs
+    else:
+        selected_jobs = [
+            job for job in jobs if fnmatch(job.get("Name", ""), requested_job_name)
+        ]
+        if selected_jobs:
+            typer.echo(
+                f'Found {len(selected_jobs)} job(s) matching "{requested_job_name}".'
+            )
+        else:
+            raise typer.BadParameter(f'No Glue jobs matched "{requested_job_name}".')
+
+    if target_job and len(selected_jobs) != 1:
+        raise typer.BadParameter(
+            "glue pull <target-job> requires exactly one matching source job."
+        )
+
+    config_dir.mkdir(parents=True, exist_ok=True)
+    config_index = _load_config_index(config_dir)
+
+    for job in selected_jobs:
+        source_name = job.get("Name", "")
+        if not source_name:
+            continue
+
+        local_job_name = target_job or source_name
+
+        saved_params = _get_saved_params_for_job(local_job_name, profile=active_profile)
+        config_entry = config_index.get(local_job_name, {})
+        config_path: Path = config_entry.get(
+            "config_path",
+            config_dir / f"{slugify(local_job_name)}.json",
+        )
+        existing_config = config_entry.get("config", {})
+        baseline_config = copy.deepcopy(existing_config)
+        existing_sc = existing_config.get("SourceControlDetails", {})
+        local_path = _resolve_pull_local_path(
+            (
+                existing_sc.get("ScriptLocation")
+                or existing_sc.get("LocalPath")
+                or f"glue/scripts/{slugify(local_job_name)}.py"
+            ),
+            saved_params,
+        )
+
+        remove_nb_outputs = existing_sc.get("RemoveNbOutputs")
+        if remove_nb_outputs is None:
+            remove_nb_outputs = job.get("JobMode", "") == "NOTEBOOK"
+
+        local_last_modified = _get_local_last_modified(config_path, existing_config)
+        remote_last_modified = _parse_datetime(job.get("LastModifiedOn"))
+
+        should_download = (
+            force
+            or local_last_modified is None
+            or remote_last_modified is None
+            or remote_last_modified > local_last_modified
+        )
+
+        if should_download:
+            display_name = (
+                f"{source_name} -> {local_job_name}"
+                if local_job_name != source_name
+                else source_name
+            )
+            if dry_run:
+                typer.echo(f"Would pull: {display_name}")
+                continue
+            typer.echo(f"Pulling: {display_name}")
+            download_glue_job_files(
+                name=source_name,
+                config_path=config_path,
+                local_path=local_path,
+                remove_nb_outputs=remove_nb_outputs,
+                include_components=components,
+                existing_sc=existing_sc,
+                profile_name=active_profile,
+                auto_login=auto_login,
+            )
+
+            if local_job_name != source_name and config_path.exists():
+                pulled_config = normalize_glue_config_data(
+                    json.loads(config_path.read_text())
+                )
+                renamed_config = _rewrite_glue_job_config(
+                    pulled_config,
+                    source_job=source_name,
+                    target_job=local_job_name,
+                    rewrite_local_paths=True,
+                )
+                for (
+                    source_path,
+                    target_path,
+                ) in _collect_source_control_file_local_renames(
+                    pulled_config,
+                    renamed_config,
+                ):
+                    _move_local_artifact(source_path, target_path)
+                config_path.write_text(json.dumps(renamed_config, indent=4))
+        else:
+            typer.echo(f"Up to date: {local_job_name}")
+
+        if config_path.exists():
+            latest_config_data = normalize_glue_config_data(
+                json.loads(config_path.read_text())
+            )
+            _sync_profile_mappings_after_pull(
+                job_name=local_job_name,
+                profile=active_profile,
+                baseline_config=baseline_config,
+                pulled_config=latest_config_data,
+                dry_run=dry_run,
+            )

gluekit/commands/push.py

@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import Any, Optional
+
+import typer
+
+from ..job_mgmt.glue_jobs import upload_glue_job_files_from_config
+
+from .constants import PUSH_COMPONENT_ALIASES, PUSH_COMPONENTS
+from .helpers import (
+    _examples_epilog,
+    _find_workspace_root,
+    _get_checked_out_jobs,
+    _get_checked_out_profile,
+    _load_config_index,
+    _raise_missing_local_config,
+    _validate_profile_mappings_align_with_config,
+)
+from .clone import _rewrite_glue_job_config
+from .edit import _normalize_component_filters
+from ..cli import app
+
+
+@app.command(
+    "push",
+    epilog=_examples_epilog(
+        "gluekit push my-job --profile my-sso-profile",
+        "gluekit push source-job target-job --profile my-sso-profile",
+        'gluekit push "my-job-\\*" --include script,job-config --exclude notebook',
+        "gluekit push my-job --build --build-tool auto --dry-run",
+    ),
+)
+def glue_push(
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help='Glue job name or pattern to push. Use "*" for all configs.',
+    ),
+    target_job: Optional[str] = typer.Argument(
+        None,
+        help="Optional remote Glue job name to push to.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be uploaded without writing files.",
+    ),
+    include: Optional[list[str]] = typer.Option(
+        None,
+        "--include",
+        "-i",
+        help=(
+            "Include only specific components (script, notebook, additional-python-modules, "
+            "extra-files, job-config)."
+        ),
+    ),
+    exclude: Optional[list[str]] = typer.Option(
+        None,
+        "--exclude",
+        "-x",
+        help=(
+            "Exclude specific components (script, notebook, additional-python-modules, "
+            "extra-files, job-config)."
+        ),
+    ),
+    update_config: bool = typer.Option(
+        True,
+        "--update-config/--no-update-config",
+        help="Update Glue job configuration after uploading files.",
+    ),
+    build: bool = typer.Option(
+        False,
+        "--build",
+        help="Build local package artifacts in the workspace root before push.",
+    ),
+    build_tool: str = typer.Option(
+        "auto",
+        "--build-tool",
+        help="Build frontend for --build: auto, uv, or build.",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Print local build command(s) before execution.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+    profile: Optional[str] = typer.Option(
+        None,
+        "--profile",
+        "-p",
+        help="AWS CLI credential profile used for real AWS Glue/S3 API calls; not a gluekit local test profile.",
+    ),
+    auto_login: bool = typer.Option(
+        True,
+        "--auto-login/--no-auto-login",
+        help="For real AWS profiles, automatically run 'aws sso login' when credentials are missing or expired.",
+    ),
+) -> None:
+    """Upload local artifacts to real AWS Glue/S3 and optionally update the job config."""
+    active_profile = profile or _get_checked_out_profile()
+    requested_job_name = job_name or "*"
+    if target_job and requested_job_name == "*":
+        raise typer.BadParameter(
+            "glue push <target-job> requires a single local source job."
+        )
+
+    components = _normalize_component_filters(
+        include,
+        exclude,
+        allowed=PUSH_COMPONENTS,
+        aliases=PUSH_COMPONENT_ALIASES,
+        context_label="glue push",
+    )
+    if build:
+        from .build import run_project_build
+
+        workspace_root = _find_workspace_root()
+        run_project_build(
+            build_tool, cwd=workspace_root, dry_run=dry_run, verbose=verbose
+        )
+
+    config_index = _load_config_index(config_dir)
+    if not config_index:
+        raise typer.BadParameter(f"No config files found in {config_dir}")
+
+    checked_out_jobs = _get_checked_out_jobs()
+    if requested_job_name == "*" and checked_out_jobs:
+        selected_entries = [
+            (name, entry)
+            for name, entry in config_index.items()
+            if name in checked_out_jobs
+        ]
+        if not selected_entries:
+            missing_job_names = ", ".join(sorted(checked_out_jobs))
+            raise typer.BadParameter(
+                f"No local config files were found for the active checkout selection: {missing_job_names}. "
+                "Create or clone the local config before running glue push."
+            )
+        typer.echo(
+            f"Using active checkout selection ({len(selected_entries)} config(s))."
+        )
+    elif requested_job_name == "*":
+        selected_entries = list(config_index.items())
+    else:
+        selected_entries = [
+            (name, entry)
+            for name, entry in config_index.items()
+            if fnmatch(name, requested_job_name)
+        ]
+        if selected_entries:
+            typer.echo(
+                f'Found {len(selected_entries)} config(s) matching "{requested_job_name}".'
+            )
+        else:
+            _raise_missing_local_config(requested_job_name, config_dir, "glue push")
+
+    if target_job and len(selected_entries) != 1:
+        raise typer.BadParameter(
+            "glue push <target-job> requires exactly one matching local config."
+        )
+
+    for name, entry in selected_entries:
+        config_path: Path = entry["config_path"]
+        config_data = entry.get("config", {})
+        if not config_data:
+            typer.echo(f"Skipping empty config for {name}")
+            continue
+
+        _validate_profile_mappings_align_with_config(
+            job_name=name,
+            config_data=config_data,
+            profile=active_profile,
+            success_message="Profile mappings align with config...now pushing config",
+        )
+        _sync_package_wheel_mapping_before_push(
+            job_name=name,
+            config_path=config_path,
+            config_data=config_data,
+            profile=active_profile,
+            build=build,
+            dry_run=dry_run,
+        )
+
+        push_config = config_data
+        display_name = name
+        if target_job and target_job != name:
+            push_config = _rewrite_glue_job_config(
+                config_data,
+                source_job=name,
+                target_job=target_job,
+                rewrite_local_paths=False,
+            )
+            display_name = f"{name} -> {target_job}"
+
+        typer.echo(f"Pushing: {display_name}")
+        upload_glue_job_files_from_config(
+            push_config,
+            dry_run=dry_run,
+            update_job_config=update_config and "job-config" in components,
+            include_components=components,
+            profile_name=active_profile,
+            auto_login=auto_login,
+        )
+
+
+def _sync_package_wheel_mapping_before_push(
+    *,
+    job_name: str,
+    config_path: Path,
+    config_data: dict[str, Any],
+    profile: Optional[str],
+    build: bool,
+    dry_run: bool,
+) -> None:
+    if not build:
+        return
+
+    from .build import (
+        _apply_package_wheel_to_config,
+        _find_workspace_root,
+        _resolve_built_package_wheel_path,
+    )
+
+    if dry_run and not build:
+        return
+
+    workspace_root = _find_workspace_root()
+    try:
+        package_wheel_path = _resolve_built_package_wheel_path(
+            workspace_root=workspace_root,
+            out_dir=None,
+        )
+    except typer.BadParameter:
+        if build and not dry_run:
+            raise
+        return
+
+    _apply_package_wheel_to_config(
+        job_name=job_name,
+        config_path=config_path,
+        config_data=config_data,
+        package_wheel_path=package_wheel_path,
+        profile=profile,
+        dry_run=dry_run,
+    )