gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+import sys
+
+from .job_mgmt import glue_jobs as glue_jobs
+
+sys.modules.setdefault(f"{__name__}.glue_jobs", glue_jobs)

gluekit/cli.py

@@ -0,0 +1,64 @@
+# ruff: noqa: E402,F401,F403
+"""Gluekit CLI entry point for local-first Glue job workflows.
+
+The current command surface centers on a single active checkout context:
+- `gluekit checkout <job-name>` selects the default local target.
+- `gluekit edit` is the primary command for explicit local config mutations.
+- `gluekit add` and `gluekit update` remain as compatibility shims.
+- `gluekit run` executes Glue scripts locally with runtime emulators.
+- `gluekit sync`, `gluekit convert`, and `gluekit remove` can also default to it.
+- `gluekit pull` and `gluekit push` still support explicit job names and `*`.
+"""
+
+from __future__ import annotations
+
+import typer
+
+app = typer.Typer(no_args_is_help=True, rich_markup_mode="markdown")
+glue_config_app = typer.Typer(
+    no_args_is_help=True,
+    rich_markup_mode="markdown",
+    help="Show and set local reusable Glue config parameters.",
+)
+app.add_typer(glue_config_app, name="config")
+glue_local_app = typer.Typer(
+    no_args_is_help=True,
+    rich_markup_mode="markdown",
+    help="Manage local-only Glue development setups and mocked AWS fixtures.",
+)
+app.add_typer(glue_local_app, name="local")
+
+# Import all commands to register them
+from .commands import (
+    build,
+    checkout,
+    clone,
+    config_commands,
+    convert,
+    edit,
+    init,
+    list,
+    local_commands,
+    pull,
+    push,
+    run,
+    remove,
+    status,
+    sync,
+    add,
+    update,
+)
+
+__all__ = ["app", "glue_config_app", "glue_local_app"]
+
+
+from .commands.constants import *
+from .commands.helpers import *
+from .commands.config_commands import (
+    _coerce_set_value,
+    _parse_set_args,
+    _to_csv_if_list,
+    _set_if_changed,
+    _write_config_changes,
+)
+from .commands.build import run_project_build, _build_zip_artifacts_from_sdists

gluekit/commands/__init__.py

@@ -0,0 +1 @@
+

gluekit/commands/add.py

@@ -0,0 +1,455 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import _resolve_notebook_path
+from .helpers import (
+    _derive_s3_target,
+    _emit_compatibility_notice,
+    _examples_epilog,
+    _get_checked_out_jobs,
+    _load_config_index,
+    _looks_like_remote_module_spec,
+    _raise_missing_local_config,
+    _resolve_single_job_name,
+    _routes_to_additional_python_modules,
+    _write_config_changes,
+)
+from .edit import (
+    _append_csv_items,
+    _parse_csv_list,
+    _remove_csv_items,
+)
+from .convert import _update_notebook_config_cell
+from ..cli import app
+
+
+def _resolve_glue_add_target(
+    args: list[str],
+    *,
+    job_name: Optional[str],
+) -> tuple[str, list[str]]:
+    # Keep `gluekit add` context-first: it uses the active checked-out job by
+    # default, and `--job-name` is a one-command override similar to `gh --repo`.
+    if not args:
+        raise typer.BadParameter("Provide one or more paths or PyPI modules.")
+
+    if job_name:
+        return job_name, args
+
+    resolved_job_name = _resolve_single_job_name(None, "glue add")
+    return resolved_job_name, args
+
+
+def _resolve_package_wheel_path(dist_dir: Path = Path("dist")) -> Path:
+    if not dist_dir.exists():
+        raise typer.BadParameter(f"Package wheel directory not found: {dist_dir}")
+    if not dist_dir.is_dir():
+        raise typer.BadParameter(f"Package wheel path is not a directory: {dist_dir}")
+
+    wheel_paths = sorted(dist_dir.glob("*.whl"))
+    if not wheel_paths:
+        raise typer.BadParameter(f"No package wheel found matching {dist_dir}/*.whl")
+    if len(wheel_paths) > 1:
+        rendered = ", ".join(path.as_posix() for path in wheel_paths)
+        raise typer.BadParameter(
+            f"Multiple package wheels found matching {dist_dir}/*.whl: {rendered}. "
+            "Pass the intended wheel path explicitly."
+        )
+    return wheel_paths[0]
+
+
+def _is_package_wheel_local_path(value: str) -> bool:
+    path = Path(value)
+    return (
+        len(path.parts) >= 2
+        and path.parts[0] == "dist"
+        and path.suffix.lower() == ".whl"
+    )
+
+
+def _remove_tracked_package_wheels(config_data: dict[str, Any]) -> list[str]:
+    default_args = config_data.setdefault("DefaultArguments", {})
+    sc = config_data.setdefault("SourceControlDetails", {})
+    additional_files = sc.get("AdditionalPythonFiles")
+    if additional_files is None:
+        return []
+    if not isinstance(additional_files, list):
+        raise typer.BadParameter(
+            "SourceControlDetails.AdditionalPythonFiles must be a list."
+        )
+
+    remaining_entries: list[dict[str, str]] = []
+    removed_s3_paths: list[str] = []
+    changes: list[str] = []
+
+    for entry in additional_files:
+        if not isinstance(entry, dict):
+            raise typer.BadParameter("AdditionalPythonFiles entries must be objects.")
+        local_path = entry.get("LocalPath")
+        s3_path = entry.get("S3Path")
+        if not local_path or not s3_path:
+            raise typer.BadParameter(
+                "AdditionalPythonFiles entries must include LocalPath and S3Path."
+            )
+        if _is_package_wheel_local_path(local_path):
+            removed_s3_paths.append(s3_path)
+            changes.append(f"Removed package wheel mapping: {local_path}")
+            continue
+        remaining_entries.append(entry)
+
+    if len(remaining_entries) != len(additional_files):
+        if remaining_entries:
+            sc["AdditionalPythonFiles"] = remaining_entries
+        else:
+            sc.pop("AdditionalPythonFiles", None)
+
+    updated_modules, removed_modules = _remove_csv_items(
+        default_args.get("--additional-python-modules"),
+        removed_s3_paths,
+    )
+    if removed_modules:
+        if updated_modules:
+            default_args["--additional-python-modules"] = updated_modules
+        else:
+            default_args.pop("--additional-python-modules", None)
+        changes.append(
+            f"Removed --additional-python-modules entries: {', '.join(removed_modules)}"
+        )
+
+    updated_extra_py_files, removed_extra_py_files = _remove_csv_items(
+        default_args.get("--extra-py-files"),
+        removed_s3_paths,
+    )
+    if removed_extra_py_files:
+        if updated_extra_py_files:
+            default_args["--extra-py-files"] = updated_extra_py_files
+        else:
+            default_args.pop("--extra-py-files", None)
+        changes.append(
+            f"Removed --extra-py-files entries: {', '.join(removed_extra_py_files)}"
+        )
+
+    return changes
+
+
+def _resolve_glue_add_package_wheel_target(
+    args: list[str],
+    *,
+    job_name: Optional[str],
+    config_index: dict[str, dict[str, Any]],
+) -> str:
+    if job_name:
+        if args:
+            raise typer.BadParameter(
+                "Do not pass positional args with --job-name and --package-whl."
+            )
+        return job_name
+
+    if len(args) == 1 and args[0] in config_index:
+        return args[0]
+    if args:
+        raise typer.BadParameter(
+            "With --package-whl, pass at most one positional Glue job name."
+        )
+
+    return _resolve_single_job_name(None, "glue add --package-whl")
+
+
+def _sync_existing_notebook_config(
+    *,
+    config_data: dict[str, Any],
+    job_name: str,
+    dry_run: bool,
+) -> bool:
+    sc = config_data.get("SourceControlDetails", {})
+    if not isinstance(sc, dict):
+        return False
+
+    script_path = Path(
+        sc.get("ScriptLocation")
+        or sc.get("LocalPath")
+        or f"glue/scripts/{slugify(job_name)}.py"
+    )
+    notebook_value = sc.get("NotebookLocation") or sc.get("NotebookPath")
+    notebook_path = (
+        Path(notebook_value)
+        if isinstance(notebook_value, str)
+        else Path(_resolve_notebook_path(script_path))
+    )
+    if not notebook_path.exists():
+        return False
+
+    return _update_notebook_config_cell(
+        notebook_path,
+        config_data,
+        {"--additional-python-modules"},
+        dry_run,
+    )
+
+
+def _apply_add_mutations(
+    *,
+    config_data: dict[str, Any],
+    job_name: str,
+    items: list[str],
+    as_path: bool,
+    as_pypi: bool,
+) -> list[str]:
+    default_args = config_data.setdefault("DefaultArguments", {})
+    sc = config_data.setdefault("SourceControlDetails", {})
+
+    command = config_data.get("Command", {})
+    script_location = command.get("ScriptLocation")
+    if not script_location:
+        raise typer.BadParameter("Missing Command.ScriptLocation in config.")
+    local_script_path = Path(
+        sc.get("ScriptLocation")
+        or sc.get("LocalPath")
+        or f"glue/scripts/{slugify(job_name)}.py"
+    )
+
+    modules: list[str] = []
+    paths: list[Path] = []
+
+    for item in items:
+        if as_path:
+            kind = "path"
+        elif as_pypi:
+            kind = "pypi"
+        elif _looks_like_remote_module_spec(item):
+            kind = "pypi"
+        else:
+            looks_like_path = (
+                "/" in item
+                or "\\" in item
+                or item.startswith(".")
+                or Path(item).suffix in {".py", ".zip"}
+                or Path(item).exists()
+            )
+            kind = "path" if looks_like_path else "pypi"
+
+        if kind == "path":
+            path = Path(item)
+            if path.is_absolute():
+                raise typer.BadParameter(f"Local path must be relative: {path}")
+            if not path.exists():
+                raise typer.BadParameter(f"Local path not found: {path}")
+            paths.append(path)
+        else:
+            modules.append(item.strip())
+
+    updates: list[str] = []
+
+    if modules:
+        updated = _append_csv_items(
+            default_args.get("--additional-python-modules"), modules
+        )
+        if updated != default_args.get("--additional-python-modules", ""):
+            default_args["--additional-python-modules"] = updated
+            updates.append(f"Added python modules: {', '.join(modules)}")
+
+    if paths:
+        additional_files = sc.get("AdditionalPythonFiles")
+        if additional_files is None:
+            additional_files = []
+        if not isinstance(additional_files, list):
+            raise typer.BadParameter(
+                "SourceControlDetails.AdditionalPythonFiles must be a list."
+            )
+
+        local_to_entry: dict[str, dict[str, str]] = {}
+        s3_to_local: dict[str, str] = {}
+        for entry in additional_files:
+            if not isinstance(entry, dict):
+                raise typer.BadParameter(
+                    "AdditionalPythonFiles entries must be objects."
+                )
+            local = entry.get("LocalPath")
+            s3_path = entry.get("S3Path")
+            if not local or not s3_path:
+                raise typer.BadParameter(
+                    "AdditionalPythonFiles entries must include LocalPath and S3Path."
+                )
+            if local in local_to_entry:
+                raise typer.BadParameter(
+                    f"Duplicate LocalPath in AdditionalPythonFiles: {local}"
+                )
+            if s3_path in s3_to_local and s3_to_local[s3_path] != local:
+                raise typer.BadParameter(
+                    f"S3Path {s3_path} already mapped to {s3_to_local[s3_path]}"
+                )
+            local_to_entry[local] = entry
+            s3_to_local[s3_path] = local
+
+        existing_extra = _parse_csv_list(default_args.get("--extra-py-files"))
+        existing_modules = _parse_csv_list(
+            default_args.get("--additional-python-modules")
+        )
+        extra_list = list(existing_extra)
+        extra_set = set(existing_extra)
+        module_list = list(existing_modules)
+        module_set = set(existing_modules)
+
+        for path in paths:
+            s3_path = _derive_s3_target(
+                path, job_name, script_location, local_script_path
+            )
+            if path.is_dir() and not s3_path.endswith(".zip"):
+                s3_path = f"{s3_path}.zip"
+            use_additional_modules = _routes_to_additional_python_modules(path)
+
+            local_key = path.as_posix()
+            if s3_path in s3_to_local and s3_to_local[s3_path] != local_key:
+                raise typer.BadParameter(
+                    f"S3Path {s3_path} already mapped to {s3_to_local[s3_path]}"
+                )
+
+            existing_entry = local_to_entry.get(local_key)
+            if existing_entry:
+                old_s3 = existing_entry.get("S3Path")
+                if old_s3 != s3_path:
+                    existing_entry["S3Path"] = s3_path
+                    s3_to_local.pop(old_s3, None)
+                    s3_to_local[s3_path] = local_key
+                    if old_s3 in extra_set:
+                        extra_set.remove(old_s3)
+                        if old_s3 in extra_list:
+                            extra_list.remove(old_s3)
+                    updates.append(
+                        f"Updated python file mapping: {local_key} -> {s3_path}"
+                    )
+            else:
+                new_entry = {"LocalPath": local_key, "S3Path": s3_path}
+                additional_files.append(new_entry)
+                local_to_entry[local_key] = new_entry
+                s3_to_local[s3_path] = local_key
+                updates.append(f"Added python file mapping: {local_key} -> {s3_path}")
+
+            if use_additional_modules:
+                if s3_path in extra_set:
+                    extra_set.remove(s3_path)
+                    if s3_path in extra_list:
+                        extra_list.remove(s3_path)
+                    updates.append(f"Removed --extra-py-files entry: {s3_path}")
+                if s3_path not in module_set:
+                    module_set.add(s3_path)
+                    module_list.append(s3_path)
+                    updates.append(
+                        f"Added --additional-python-modules entry: {s3_path}"
+                    )
+            else:
+                if s3_path not in extra_set:
+                    extra_set.add(s3_path)
+                    extra_list.append(s3_path)
+                    updates.append(f"Added --extra-py-files entry: {s3_path}")
+
+        if updates:
+            sc["AdditionalPythonFiles"] = additional_files
+            if extra_list:
+                default_args["--extra-py-files"] = ",".join(extra_list)
+            else:
+                default_args.pop("--extra-py-files", None)
+            if module_list:
+                default_args["--additional-python-modules"] = ",".join(module_list)
+            elif not modules:
+                default_args.pop("--additional-python-modules", None)
+
+    return updates
+
+
+@app.command(
+    "add",
+    help="Compatibility command for inferred artifact and module additions. Prefer edit for explicit config changes.",
+    epilog=_examples_epilog(
+        "gluekit checkout my-job",
+        "gluekit add dist/my-package.whl",
+        "gluekit add --job-name other-job requests==2.32.3 --as-pypi",
+    ),
+)
+def glue_add(
+    args: Optional[list[str]] = typer.Argument(
+        None,
+        help="Paths or module specifiers to add to the selected Glue job config.",
+    ),
+    job_name: Optional[str] = typer.Option(
+        None,
+        "--job-name",
+        help="Override the active checked-out job for this command only.",
+    ),
+    as_path: bool = typer.Option(
+        False,
+        "--as-path",
+        help="Treat all items as local paths.",
+    ),
+    as_pypi: bool = typer.Option(
+        False,
+        "--as-pypi",
+        help="Treat all items as PyPI modules.",
+    ),
+    package_whl: bool = typer.Option(
+        False,
+        "--package-whl",
+        help="Use the package wheel from dist/*.whl.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be updated without writing files.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+) -> None:
+    """Compatibility command for inferred artifact and module additions."""
+    if as_path and as_pypi:
+        raise typer.BadParameter("Use only one of --as-path or --as-pypi.")
+
+    config_index = _load_config_index(config_dir)
+    if package_whl:
+        job_name = _resolve_glue_add_package_wheel_target(
+            args or [],
+            job_name=job_name,
+            config_index=config_index,
+        )
+        items = [_resolve_package_wheel_path().as_posix()]
+        as_path = True
+    else:
+        job_name, items = _resolve_glue_add_target(args or [], job_name=job_name)
+        if not items:
+            raise typer.BadParameter("Provide one or more paths or PyPI modules.")
+
+    config_entry = config_index.get(job_name)
+    if not config_entry:
+        if job_name in _get_checked_out_jobs():
+            _raise_missing_local_config(job_name, config_dir, "glue add")
+        raise typer.BadParameter(
+            f'No config files matched "{job_name}" in {config_dir}.'
+        )
+
+    config_path: Path = config_entry["config_path"]
+    config_data = config_entry["config"]
+    _emit_compatibility_notice("add", "when you want explicit field-level mutations")
+    removal_updates = []
+    if package_whl:
+        removal_updates = _remove_tracked_package_wheels(config_data)
+    updates = removal_updates + _apply_add_mutations(
+        config_data=config_data,
+        job_name=job_name,
+        items=items,
+        as_path=as_path,
+        as_pypi=as_pypi,
+    )
+    _write_config_changes(config_path, config_data, updates, dry_run=dry_run)
+    if updates:
+        _sync_existing_notebook_config(
+            config_data=config_data,
+            job_name=job_name,
+            dry_run=dry_run,
+        )