gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/commands/remove.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import csv
+import copy
+import json
+import re
+import shutil
+import subprocess
+import tarfile
+import uuid
+import zipfile
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from email.parser import Parser
+from fnmatch import fnmatch
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import (
+    download_glue_job_files,
+    list_glue_jobs,
+    normalize_glue_config_data,
+    convert_script_to_notebook,
+    convert_notebook_to_script,
+    _resolve_notebook_path,
+    upload_glue_job_files_from_config,
+)
+from ..job_mgmt.magics import build_magic_cell_sources as _build_magic_cell_sources
+
+from .constants import *
+from .helpers import *
+from ..cli import app, glue_config_app
+
+
+def _remove_local_path(path: Path) -> bool:
+    if not path.exists():
+        return False
+    if path.is_dir():
+        shutil.rmtree(path)
+    else:
+        path.unlink()
+    return True
+
+
+@app.command(
+    "remove",
+    epilog=_examples_epilog(
+        "gluekit remove my-job",
+        "gluekit remove my-job --remove-additional-python-files",
+        'gluekit remove "my-job-\\*" --remove-extra-files --dry-run',
+    ),
+)
+def glue_remove(
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help='Local Glue job name or pattern to remove. Use "*" for all local configs.',
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be removed without deleting files.",
+    ),
+    remove_additional_python_files: bool = typer.Option(
+        False,
+        "--remove-additional-python-files",
+        help="Also remove local files or directories referenced by SourceControlDetails.AdditionalPythonFiles.",
+    ),
+    remove_extra_files: bool = typer.Option(
+        False,
+        "--remove-extra-files",
+        help="Also remove local files or directories referenced by SourceControlDetails.AdditionalFiles and ExtraFiles.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+) -> None:
+    """Remove local Glue job configs and local artifacts."""
+    config_index = _load_config_index(config_dir)
+    if not config_index:
+        raise typer.BadParameter(f"No config files found in {config_dir}")
+
+    checked_out_jobs = _get_checked_out_jobs()
+    if job_name is None:
+        if not checked_out_jobs:
+            raise typer.BadParameter(
+                "Provide <job-name> or run 'gluekit checkout <job-name>' before glue remove."
+            )
+        selected_entries = [
+            (name, entry)
+            for name, entry in config_index.items()
+            if name in checked_out_jobs
+        ]
+        if not selected_entries:
+            missing_job_names = ", ".join(sorted(checked_out_jobs))
+            raise typer.BadParameter(
+                f"No local config files were found for the active checkout selection: {missing_job_names}."
+            )
+        typer.echo(
+            f"Using active checkout selection ({len(selected_entries)} config(s))."
+        )
+    elif job_name == "*" and checked_out_jobs:
+        selected_entries = [
+            (name, entry)
+            for name, entry in config_index.items()
+            if name in checked_out_jobs
+        ]
+        if not selected_entries:
+            missing_job_names = ", ".join(sorted(checked_out_jobs))
+            raise typer.BadParameter(
+                f"No local config files were found for the active checkout selection: {missing_job_names}."
+            )
+        typer.echo(
+            f"Using active checkout selection ({len(selected_entries)} config(s))."
+        )
+    elif job_name == "*":
+        selected_entries = list(config_index.items())
+    else:
+        selected_entries = [
+            (name, entry)
+            for name, entry in config_index.items()
+            if fnmatch(name, job_name)
+        ]
+        if selected_entries:
+            typer.echo(
+                f'Found {len(selected_entries)} config(s) matching "{job_name}".'
+            )
+        else:
+            _raise_missing_local_config(job_name, config_dir, "glue remove")
+
+    for name, entry in selected_entries:
+        config_path: Path = entry["config_path"]
+        config_data = entry.get("config", {})
+        artifact_paths = _collect_config_local_artifact_paths(
+            config_data,
+            job_name=name,
+            include_additional_python_files=remove_additional_python_files,
+            include_extra_files=remove_extra_files,
+        )
+
+        seen_paths = {config_path.resolve(): ("config", config_path)}
+        for kind, path in artifact_paths:
+            resolved_path = path.resolve()
+            if resolved_path not in seen_paths:
+                seen_paths[resolved_path] = (kind, path)
+
+        if dry_run:
+            for kind, path in seen_paths.values():
+                typer.echo(f"Would remove {kind}: {path}")
+            continue
+
+        for kind, path in seen_paths.values():
+            if _remove_local_path(path):
+                typer.echo(f"Removed {kind}: {path}")
+            else:
+                typer.echo(f"Skipped missing {kind}: {path}")

gluekit/commands/run.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from ..cli import app
+from ..local import run_glue_script
+from ..local.local_fixtures import LOCAL_RUN_CONFIG_FILE
+from .helpers import _get_checked_out_local_setup
+from .helpers import _examples_epilog
+
+
+def _parse_key_value(raw: str, *, option_name: str) -> tuple[str, str]:
+    key, separator, value = raw.partition("=")
+    if not separator:
+        raise typer.BadParameter(
+            f"{option_name} entries must be KEY=VALUE. Received: {raw!r}"
+        )
+    key = key.strip()
+    if not key:
+        raise typer.BadParameter(
+            f"{option_name} entries must include a non-empty key. Received: {raw!r}"
+        )
+    return key, value
+
+
+@app.command(
+    "run",
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+    epilog=_examples_epilog(
+        "gluekit run ../ppsc-survey/glue/scripts/job_a.py --create-bucket my-input-bucket",
+        "gluekit run glue/scripts/job-a.py --ssm-parameter /app/env=dev --glue-arg report_date=2026-06-18",
+        "gluekit run glue/scripts/job-a.py -- --JOB_NAME custom-local-job --report_date 2026-06-18",
+    ),
+)
+def glue_run(
+    ctx: typer.Context,
+    script_path: Path = typer.Argument(
+        ...,
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+        resolve_path=True,
+        help="Path to the Glue Python script to execute locally.",
+    ),
+    glue_arg: Optional[list[str]] = typer.Option(
+        None,
+        "--glue-arg",
+        help="Glue script argument in KEY=VALUE form; passed through as --KEY VALUE.",
+    ),
+    job_name: str = typer.Option(
+        "local-glue-job",
+        "--job-name",
+        help="Default JOB_NAME used when not provided in script args.",
+    ),
+    create_bucket: Optional[list[str]] = typer.Option(
+        None,
+        "--create-bucket",
+        help="Create a mocked S3 bucket only inside the local run. Repeat as needed.",
+    ),
+    ssm_parameter: Optional[list[str]] = typer.Option(
+        None,
+        "--ssm-parameter",
+        help="Seed a mocked SSM parameter only inside the local run as NAME=VALUE. Repeat as needed.",
+    ),
+    aws_region: Optional[str] = typer.Option(
+        None,
+        "--aws-region",
+        help="Mock AWS region used for local boto3 clients. Defaults to us-east-1.",
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        exists=False,
+        file_okay=True,
+        dir_okay=False,
+        resolve_path=True,
+        help="Local fixture config file for mocked S3/SSM. Defaults to .gluekit/local.json when local settings are checked out.",
+    ),
+) -> None:
+    """Run a Glue script locally with emulated Glue libraries and mocked AWS services."""
+    # Translate CLI --glue-arg KEY=VALUE list into a dict for run_glue_script.
+    glue_args: dict[str, str] = {}
+    for item in glue_arg or []:
+        key, value = _parse_key_value(item, option_name="--glue-arg")
+        glue_args[key] = value
+
+    # Translate --ssm-parameter NAME=VALUE list into a dict.
+    ssm_parameters: dict[str, str] = {}
+    for entry in ssm_parameter or []:
+        name, value = _parse_key_value(entry, option_name="--ssm-parameter")
+        ssm_parameters[name] = value
+
+    # Validate bucket names up-front so the error is a clean typer message.
+    buckets: list[str] = []
+    for bucket_name in create_bucket or []:
+        normalized = bucket_name.strip()
+        if not normalized:
+            raise typer.BadParameter(
+                "--create-bucket entries must be non-empty bucket names."
+            )
+        buckets.append(normalized)
+
+    local_setup = _get_checked_out_local_setup()
+    default_config_file: Path | None = None
+    if config_file is not None:
+        default_config_file = config_file
+    elif isinstance(local_setup, dict):
+        default_config_file = LOCAL_RUN_CONFIG_FILE
+
+    try:
+        run_glue_script(
+            script_path,
+            script_args=list(ctx.args),
+            glue_args=glue_args or None,
+            job_name=job_name,
+            create_buckets=buckets or None,
+            ssm_parameters=ssm_parameters or None,
+            aws_region=aws_region,
+            config_file=default_config_file,
+        )
+    except (ImportError, ValueError) as exc:
+        raise typer.BadParameter(str(exc)) from exc

gluekit/commands/status.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+
+from ..cli import app
+from ..job_mgmt.glue_jobs import resolve_glue_artifact_mappings
+from .helpers import (
+    _examples_epilog,
+    _get_checked_out_jobs,
+    _get_checked_out_profile,
+    _load_config_index,
+)
+
+
+def _relative_display_path(path: Path) -> str:
+    try:
+        return path.resolve().relative_to(Path.cwd().resolve()).as_posix()
+    except ValueError:
+        return path.as_posix()
+
+
+def _print_available_jobs(config_index: dict[str, dict]) -> None:
+    if not config_index:
+        typer.echo("- (none)")
+        return
+    for job_name in sorted(config_index):
+        config_path = config_index[job_name]["config_path"]
+        typer.echo(f"- {job_name}: {_relative_display_path(config_path)}")
+
+
+def _print_pretty_config(config_data: dict) -> None:
+    typer.echo("Checked out config:")
+    typer.echo(json.dumps(config_data, indent=4))
+
+
+@app.command(
+    "status",
+    epilog=_examples_epilog(
+        "gluekit status",
+        "gluekit status --config-dir glue/configs",
+    ),
+)
+def glue_status(
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+) -> None:
+    """Show the active local checkout and its local-to-S3 artifact mappings."""
+    config_index = _load_config_index(config_dir)
+    checked_out_jobs = _get_checked_out_jobs()
+    checked_out_profile = _get_checked_out_profile()
+
+    if not checked_out_jobs:
+        typer.echo("No jobs checked out, available jobs are:")
+        _print_available_jobs(config_index)
+        return
+
+    missing_jobs = [
+        job_name for job_name in checked_out_jobs if job_name not in config_index
+    ]
+    if missing_jobs:
+        for job_name in missing_jobs:
+            typer.echo(f"Checked out job: {job_name}")
+            typer.echo(f"Missing local config in: {_relative_display_path(config_dir)}")
+        typer.echo("Available jobs are:")
+        _print_available_jobs(config_index)
+        raise typer.Exit(code=1)
+
+    for index, job_name in enumerate(checked_out_jobs):
+        if index:
+            typer.echo()
+        entry = config_index[job_name]
+        config_path = entry["config_path"]
+        resolution = resolve_glue_artifact_mappings(entry["config"])
+
+        typer.echo(f"Checked out job: {job_name}")
+        typer.echo(f"Profile: {checked_out_profile or '(none)'}")
+        typer.echo(f"Config: {_relative_display_path(config_path)}")
+        _print_pretty_config(entry["config"])
+        typer.echo("Artifacts:")
+        for mapping in resolution["mappings"]:
+            local_path = Path(mapping["local_path"])
+            local_status = "present" if local_path.exists() else "missing"
+            typer.echo(
+                f"- {mapping['artifact_type']}: "
+                f"{mapping['local_path']} -> {mapping['s3_path']} ({local_status})"
+            )
+        if not resolution["mappings"]:
+            typer.echo("- (none)")
+
+        for warning in resolution["warnings"]:
+            typer.echo(f"Warning: {warning}", err=True)

gluekit/commands/sync.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import csv
+import copy
+import json
+import re
+import shutil
+import subprocess
+import tarfile
+import uuid
+import zipfile
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from email.parser import Parser
+from fnmatch import fnmatch
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import (
+    download_glue_job_files,
+    list_glue_jobs,
+    normalize_glue_config_data,
+    convert_script_to_notebook,
+    convert_notebook_to_script,
+    _resolve_notebook_path,
+    upload_glue_job_files_from_config,
+)
+from ..job_mgmt.magics import build_magic_cell_sources as _build_magic_cell_sources
+
+from .constants import *
+from .helpers import *
+from .convert import _update_script_config_cell, _update_notebook_config_cell
+from ..cli import app, glue_config_app
+
+
+@app.command(
+    "sync",
+    epilog=_examples_epilog(
+        "gluekit sync my-job",
+        "gluekit sync my-job --dry-run",
+    ),
+)
+def glue_sync(
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help="Glue job name to sync. Defaults to the active checkout selection.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be updated without writing files.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+) -> None:
+    """Sync generated Glue config metadata cells in local script/notebook files."""
+    job_name = _resolve_single_job_name(job_name, "glue sync")
+    config_index = _load_config_index(config_dir)
+    config_entry = config_index.get(job_name)
+    if not config_entry:
+        _raise_missing_local_config(job_name, config_dir, "glue sync")
+
+    config_data = config_entry.get("config", {})
+    sc = config_data.get("SourceControlDetails", {})
+    script_path = Path(
+        sc.get("ScriptLocation")
+        or sc.get("LocalPath")
+        or f"glue/scripts/{slugify(job_name)}.py"
+    )
+    notebook_path_value = sc.get("NotebookLocation") or sc.get("NotebookPath")
+    if notebook_path_value:
+        notebook_path = Path(notebook_path_value)
+    else:
+        notebook_path = Path(_resolve_notebook_path(script_path))
+
+    updated_any = _update_script_config_cell(script_path, config_data, dry_run)
+
+    if config_data.get("JobMode") == "NOTEBOOK":
+        updated_any = (
+            _update_notebook_config_cell(
+                notebook_path,
+                config_data,
+                {"--additional-python-modules", "--extra-py-files"},
+                dry_run,
+            )
+            or updated_any
+        )
+
+    if not updated_any:
+        typer.echo("No sync updates needed.")

gluekit/commands/update.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import csv
+import copy
+import json
+import re
+import shutil
+import subprocess
+import tarfile
+import uuid
+import zipfile
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from email.parser import Parser
+from fnmatch import fnmatch
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import (
+    download_glue_job_files,
+    list_glue_jobs,
+    normalize_glue_config_data,
+    convert_script_to_notebook,
+    convert_notebook_to_script,
+    _resolve_notebook_path,
+    upload_glue_job_files_from_config,
+)
+from ..job_mgmt.magics import build_magic_cell_sources as _build_magic_cell_sources
+
+from .constants import *
+from .helpers import *
+from .edit import _apply_edit_mutations
+from ..cli import app, glue_config_app
+from .edit import _apply_edit_mutations
+
+
+def _resolve_glue_update_target(
+    args: list[str],
+    *,
+    config_dir: Path,
+) -> tuple[str, str, str]:
+    if len(args) == 3:
+        config_index = _load_config_index(config_dir)
+        if args[0] in config_index:
+            return args[0], args[1], args[2]
+
+    if len(args) == 2:
+        job_name = _resolve_single_job_name(None, "glue update")
+        return job_name, args[0], args[1]
+
+    raise typer.BadParameter(
+        "Use 'gluekit update <job-name> <property> <value>' or, with one checked-out job, 'gluekit update <property> <value>'."
+    )
+
+
+@app.command(
+    "update",
+    help="Compatibility command for generic property updates. Prefer edit for explicit config changes.",
+    epilog=_examples_epilog(
+        "gluekit update my-job Description 'Updated job description'",
+        "gluekit update my-job command.ScriptLocation s3://my-bucket/scripts/my-job.py",
+        "gluekit update default_arguments.--TempDir s3://my-bucket/tmp/",
+    ),
+)
+def glue_update(
+    args: list[str] = typer.Argument(
+        ...,
+        help="Either <job-name> <property> <value> or, with the active checkout selection, just <property> <value>.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be updated without writing files.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+) -> None:
+    """Compatibility command for generic property updates."""
+    job_name, property_name, raw_value = _resolve_glue_update_target(
+        args, config_dir=config_dir
+    )
+    if not property_name.strip():
+        raise typer.BadParameter("Property name must be a non-empty string.")
+
+    config_index = _load_config_index(config_dir)
+    config_entry = config_index.get(job_name)
+    if not config_entry:
+        _raise_missing_local_config(job_name, config_dir, "glue update")
+
+    config_path: Path = config_entry["config_path"]
+    config_data = config_entry["config"]
+    _emit_compatibility_notice("update", "for named field flags and list operations")
+    changes = _apply_saved_params_to_config(
+        config_data,
+        {property_name: _coerce_set_value(raw_value)},
+    )
+    _write_config_changes(config_path, config_data, changes, dry_run=dry_run)