gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/commands/config_commands.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import csv
+import copy
+import json
+import re
+import shutil
+import subprocess
+import tarfile
+import uuid
+import zipfile
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from email.parser import Parser
+from fnmatch import fnmatch
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import (
+    download_glue_job_files,
+    list_glue_jobs,
+    normalize_glue_config_data,
+    convert_script_to_notebook,
+    convert_notebook_to_script,
+    _resolve_notebook_path,
+    upload_glue_job_files_from_config,
+)
+from ..job_mgmt.magics import build_magic_cell_sources as _build_magic_cell_sources
+
+from .constants import *
+from .helpers import *
+from ..cli import app, glue_config_app
+
+
+def _parse_set_args(args: list[str]) -> dict[str, Any]:
+    parsed: dict[str, Any] = {}
+    idx = 0
+    while idx < len(args):
+        token = args[idx]
+        if not token.startswith("--"):
+            raise typer.BadParameter(
+                f"Invalid token '{token}'. Use --key value or --key=value."
+            )
+        key_token = token[2:]
+        if not key_token:
+            raise typer.BadParameter("Invalid empty parameter name.")
+
+        if "=" in key_token:
+            key, raw_value = key_token.split("=", 1)
+            if not key:
+                raise typer.BadParameter("Invalid empty parameter name.")
+            parsed[key] = _coerce_set_value(raw_value)
+            idx += 1
+            continue
+
+        key = key_token
+        idx += 1
+        if idx >= len(args):
+            raise typer.BadParameter(f"Missing value for --{key}.")
+        raw_value = args[idx]
+        if raw_value.startswith("--"):
+            raise typer.BadParameter(f"Missing value for --{key}.")
+        parsed[key] = _coerce_set_value(raw_value)
+        idx += 1
+
+    if not parsed:
+        raise typer.BadParameter("Provide one or more --key value pairs.")
+    return parsed
+
+
+@app.command(
+    "set",
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+    epilog=_examples_epilog(
+        "gluekit set my-job --script_location glue/scripts/my-job.py --extra_files s3://my-bucket/shared/config.json",
+        "gluekit pull my-job --profile my-sso-profile",
+        'gluekit set --global --additional_python_modules "pydantic==2.11.7"',
+        'gluekit push "my-job-\\*" --profile my-sso-profile',
+        "gluekit config set my-job --default_arguments.--TempDir s3://my-bucket/tmp/",
+        "gluekit config get my-job",
+        "gluekit config get",
+        "gluekit set my-job --extra_py_files s3://my-bucket/dist/gluekit-0.0.1-py3-none-any.whl",
+        "gluekit push my-job --build --build-tool auto --profile my-sso-profile",
+    ),
+)
+def glue_set(
+    ctx: typer.Context,
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help="Glue job name to save params for.",
+    ),
+    global_scope: bool = typer.Option(
+        False,
+        "--global",
+        help="Save params as global defaults for all jobs.",
+    ),
+    profile: Optional[str] = typer.Option(
+        None,
+        "--profile",
+        "-p",
+        help=(
+            "Save params under this gluekit profile scope, usually matching an "
+            "AWS CLI profile name; does not contact AWS."
+        ),
+    ),
+) -> None:
+    """Set reusable local key/value parameters for a Glue job."""
+    parsed = _parse_set_args(list(ctx.args))
+    target = _set_saved_scope(parsed, job_name, global_scope, profile=profile)
+    typer.echo(f"Saved {len(parsed)} parameter(s) for '{target}' in {GLUE_SET_FILE}.")
+
+
+@glue_config_app.command(
+    "set",
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+    epilog=_examples_epilog(
+        "gluekit config set my-job --default_arguments.--TempDir s3://my-bucket/tmp/",
+        'gluekit config set --global --additional_python_modules "pydantic==2.11.7"',
+    ),
+)
+def glue_config_set(
+    ctx: typer.Context,
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help="Glue job name to save params for.",
+    ),
+    global_scope: bool = typer.Option(
+        False,
+        "--global",
+        help="Save params as global defaults for all jobs.",
+    ),
+    profile: Optional[str] = typer.Option(
+        None,
+        "--profile",
+        "-p",
+        help=(
+            "Save params under this gluekit profile scope, usually matching an "
+            "AWS CLI profile name; does not contact AWS."
+        ),
+    ),
+) -> None:
+    """Set reusable local Glue config params by job or globally."""
+    parsed = _parse_set_args(list(ctx.args))
+    target = _set_saved_scope(parsed, job_name, global_scope, profile=profile)
+    typer.echo(f"Saved {len(parsed)} parameter(s) for '{target}' in {GLUE_SET_FILE}.")
+
+
+@glue_config_app.command(
+    "get",
+    epilog=_examples_epilog(
+        "gluekit config get my-job",
+        "gluekit config get",
+    ),
+)
+def glue_config_get(
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help="Optional Glue job name to view merged params (global + job).",
+    ),
+    profile: Optional[str] = typer.Option(
+        None,
+        "--profile",
+        "-p",
+        help=(
+            "Include params from this gluekit profile scope, usually matching an "
+            "AWS CLI profile name; does not contact AWS."
+        ),
+    ),
+) -> None:
+    """Show stored local Glue config params."""
+    store = _load_glue_set_store()
+    if not job_name:
+        typer.echo(json.dumps(store, indent=4))
+        return
+    merged = _get_saved_params_for_job(job_name, profile=profile)
+    typer.echo(json.dumps(merged, indent=4))

gluekit/commands/constants.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+PULL_COMPONENTS = {
+    "config",
+    "script",
+    "notebook",
+    "additional-python-modules",
+    "extra-files",
+}
+
+PUSH_COMPONENTS = {
+    "script",
+    "notebook",
+    "additional-python-modules",
+    "extra-files",
+    "job-config",
+}
+
+PULL_COMPONENT_ALIASES = {
+    "extra-py-files": "additional-python-modules",
+    "extra_py_files": "additional-python-modules",
+    "extra-py": "additional-python-modules",
+    "additional-python-files": "additional-python-modules",
+    "additional-python": "additional-python-modules",
+    "additional-py-files": "additional-python-modules",
+    "extra_files": "extra-files",
+    "additional-files": "extra-files",
+    "extra": "extra-files",
+}
+
+PUSH_COMPONENT_ALIASES = {
+    **PULL_COMPONENT_ALIASES,
+    "config": "job-config",
+    "job_config": "job-config",
+    "update-config": "job-config",
+    "update_config": "job-config",
+}
+
+GLUE_SET_FILE = Path(".gluekit/glue_set.json")
+
+__all__ = [
+    "PULL_COMPONENTS",
+    "PUSH_COMPONENTS",
+    "PULL_COMPONENT_ALIASES",
+    "PUSH_COMPONENT_ALIASES",
+    "GLUE_SET_FILE",
+]

gluekit/commands/convert.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+import csv
+import copy
+import json
+import re
+import shutil
+import subprocess
+import tarfile
+import uuid
+import zipfile
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from email.parser import Parser
+from fnmatch import fnmatch
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Optional
+
+import typer
+from slugify import slugify
+
+from ..job_mgmt.glue_jobs import (
+    download_glue_job_files,
+    list_glue_jobs,
+    normalize_glue_config_data,
+    convert_script_to_notebook,
+    convert_notebook_to_script,
+    _resolve_notebook_path,
+    upload_glue_job_files_from_config,
+)
+from ..job_mgmt.magics import build_magic_cell_sources as _build_magic_cell_sources
+
+from .constants import *
+from .helpers import *
+from ..cli import app, glue_config_app
+
+
+def _notebook_source_to_lines(source: Any) -> list[str]:
+    if source is None:
+        return []
+    if isinstance(source, str):
+        return source.splitlines(keepends=True)
+    lines: list[str] = []
+    for part in source:
+        if part.endswith("\n"):
+            lines.append(part)
+        else:
+            lines.append(f"{part}\n")
+    return lines
+
+
+def _build_script_magic_cell_lines(config_data: dict[str, Any]) -> list[str]:
+    lines = ["#%% [markdown]\n", "# AWS Glue configs\n"]
+
+    for source_lines in _build_magic_cell_sources(config_data):
+        for line in source_lines:
+            stripped = line.rstrip("\n")
+            lines.append(f"# {stripped}\n" if stripped else "#\n")
+
+    lines.append("\n")
+    lines.append("#%%\n")
+    return lines
+
+
+def _update_script_config_cell(
+    script_path: Path,
+    config_data: dict[str, Any],
+    dry_run: bool,
+) -> bool:
+    if not script_path.exists():
+        typer.echo(f"Script not found: {script_path}")
+        return False
+
+    script_text = script_path.read_text()
+    lines = script_text.splitlines(keepends=True)
+    magic_lines = _build_script_magic_cell_lines(config_data)
+
+    marker_re = re.compile(r"^#\s*%%")
+
+    def is_glue_magic_line(raw_line: str) -> bool:
+        candidate = raw_line.lstrip()
+        if candidate.startswith("#"):
+            candidate = candidate[1:].lstrip()
+        return (
+            candidate.startswith("%")
+            or "%%configure" in candidate
+            or "AWS Glue configs" in candidate
+        )
+
+    def split_into_cells(all_lines: list[str]) -> list[list[str]]:
+        starts = [idx for idx, line in enumerate(all_lines) if marker_re.match(line)]
+        if not starts:
+            return [all_lines]
+        if starts[0] != 0:
+            starts = [0] + starts
+        boundaries = starts[1:] + [len(all_lines)]
+        return [all_lines[start:end] for start, end in zip(starts, boundaries)]
+
+    kept: list[str] = []
+    for cell_lines in split_into_cells(lines):
+        if any(is_glue_magic_line(line) for line in cell_lines):
+            first_code_idx = None
+            for idx, line in enumerate(cell_lines):
+                stripped = line.strip()
+                if not stripped:
+                    continue
+                if stripped.startswith("#"):
+                    continue
+                if marker_re.match(line):
+                    continue
+                first_code_idx = idx
+                break
+            if first_code_idx is not None:
+                kept.extend(cell_lines[first_code_idx:])
+            continue
+        kept.extend(cell_lines)
+
+    new_lines = magic_lines + kept
+
+    if new_lines == lines:
+        return False
+
+    if dry_run:
+        typer.echo(f"Would update script config cell: {script_path}")
+        return True
+
+    script_path.write_text("".join(new_lines))
+    typer.echo(f"Updated script config cell: {script_path}")
+    return True
+
+
+def _extract_config_from_cell(lines: list[str]) -> tuple[int, dict[str, Any]]:
+    for idx, line in enumerate(lines):
+        if "%%configure" in line:
+            raw = "".join(lines[idx + 1 :]).strip()
+            if not raw:
+                return idx, {}
+            start = raw.find("{")
+            end = raw.rfind("}")
+            if start == -1 or end == -1 or end < start:
+                raise ValueError("Unable to locate JSON block after %%configure.")
+            payload = raw[start : end + 1]
+            return idx, json.loads(payload)
+    raise ValueError("No %%configure cell found.")
+
+
+def _new_notebook_code_cell(source_lines: list[str]) -> dict[str, Any]:
+    return {
+        "cell_type": "code",
+        "id": str(uuid.uuid4()),
+        "metadata": {
+            "tags": [],
+            "trusted": True,
+            "vscode": {"languageId": "python_glue_session"},
+        },
+        "source": source_lines,
+        "execution_count": None,
+        "outputs": [],
+    }
+
+
+def _new_notebook_markdown_cell(source_lines: list[str]) -> dict[str, Any]:
+    return {
+        "cell_type": "markdown",
+        "id": str(uuid.uuid4()),
+        "metadata": {},
+        "source": source_lines,
+    }
+
+
+def _is_notebook_magic_cell(cell: dict[str, Any]) -> bool:
+    if cell.get("cell_type") != "code":
+        return False
+    lines = _notebook_source_to_lines(cell.get("source"))
+    for line in lines:
+        if line.lstrip().startswith("%"):
+            return True
+    return False
+
+
+def _is_generated_glue_heading_cell(cell: dict[str, Any]) -> bool:
+    if cell.get("cell_type") != "markdown":
+        return False
+    lines = _notebook_source_to_lines(cell.get("source"))
+    text = "".join(lines).strip()
+    return text in {"# AWS Glue configs", "# AWS Glue Script"}
+
+
+def _build_notebook_magic_cells(config_data: dict[str, Any]) -> list[dict[str, Any]]:
+    cells: list[dict[str, Any]] = [
+        _new_notebook_markdown_cell(["# AWS Glue configs\n"])
+    ]
+    cells.extend(
+        _new_notebook_code_cell(source_lines)
+        for source_lines in _build_magic_cell_sources(config_data)
+    )
+
+    cells.append(_new_notebook_markdown_cell(["# AWS Glue Script\n"]))
+    return cells
+
+
+def _update_notebook_config_cell(
+    notebook_path: Path,
+    config_data: dict[str, Any],
+    keys_to_update: set[str],
+    dry_run: bool,
+) -> bool:
+    if not notebook_path.exists():
+        typer.echo(f"Notebook not found: {notebook_path}")
+        return False
+
+    del keys_to_update
+
+    notebook = json.loads(notebook_path.read_text())
+    existing_cells = notebook.get("cells", [])
+    cleaned_cells = [
+        cell
+        for cell in existing_cells
+        if not _is_notebook_magic_cell(cell)
+        and not _is_generated_glue_heading_cell(cell)
+    ]
+
+    new_magic_cells = _build_notebook_magic_cells(config_data)
+    notebook["cells"] = new_magic_cells + cleaned_cells
+
+    if dry_run:
+        typer.echo(f"Would update notebook config cell: {notebook_path}")
+        return True
+
+    notebook_path.write_text(json.dumps(notebook, indent=4))
+    typer.echo(f"Updated notebook config cell: {notebook_path}")
+    return True
+
+
+@app.command(
+    "convert",
+    epilog=_examples_epilog(
+        "gluekit convert my-job --from script --to notebook --use-config",
+        "gluekit convert my-job --from notebook --to script --script-only",
+    ),
+)
+def glue_convert(
+    job_name: Optional[str] = typer.Argument(
+        None,
+        help="Glue job name to convert. Defaults to the active checkout selection.",
+    ),
+    from_format: str = typer.Option(
+        "script",
+        "--from",
+        help="Source format (script or notebook).",
+    ),
+    to_format: str = typer.Option(
+        "notebook",
+        "--to",
+        help="Target format (script or notebook).",
+    ),
+    use_config: bool = typer.Option(
+        True,
+        "--use-config/--script-only",
+        help="Use config metadata (magics and paths) when available.",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be converted without writing files.",
+    ),
+    config_dir: Path = typer.Option(
+        Path("glue/configs"),
+        "--config-dir",
+        help="Directory containing Glue job config files.",
+    ),
+) -> None:
+    """Convert Glue jobs between script and notebook formats."""
+    job_name = _resolve_single_job_name(job_name, "glue convert")
+    from_format = from_format.lower()
+    to_format = to_format.lower()
+
+    if from_format == to_format:
+        raise typer.BadParameter("--from and --to must be different values.")
+
+    if from_format not in {"script", "notebook"} or to_format not in {
+        "script",
+        "notebook",
+    }:
+        raise typer.BadParameter("--from/--to must be either script or notebook.")
+
+    config_data = None
+    if use_config:
+        config_index = _load_config_index(config_dir)
+        config_entry = config_index.get(job_name)
+        if config_entry:
+            config_data = config_entry.get("config", {})
+            sc = config_data.get("SourceControlDetails", {})
+            script_path = Path(
+                sc.get("ScriptLocation")
+                or sc.get("LocalPath")
+                or f"glue/scripts/{slugify(job_name)}.py"
+            )
+            notebook_path = sc.get("NotebookPath") or sc.get("NotebookLocation")
+            if notebook_path:
+                notebook_path = Path(notebook_path)
+            else:
+                notebook_path = Path(_resolve_notebook_path(script_path))
+        else:
+            script_path = Path(f"glue/scripts/{slugify(job_name)}.py")
+            notebook_path = Path(f"glue/notebooks/{slugify(job_name)}.ipynb")
+    else:
+        script_path = Path(f"glue/scripts/{slugify(job_name)}.py")
+        notebook_path = Path(f"glue/notebooks/{slugify(job_name)}.ipynb")
+
+    if from_format == "script" and not script_path.exists():
+        raise typer.BadParameter(f"Script not found: {script_path}")
+    if from_format == "notebook" and not notebook_path.exists():
+        raise typer.BadParameter(f"Notebook not found: {notebook_path}")
+
+    if dry_run:
+        if from_format == "notebook":
+            typer.echo(f"Would convert: {notebook_path} -> {script_path}")
+        else:
+            typer.echo(f"Would convert: {script_path} -> {notebook_path}")
+        return
+
+    if from_format == "script" and to_format == "notebook":
+        converted_path = convert_script_to_notebook(
+            script_path,
+            notebook_path,
+            config_data=config_data,
+            include_magics=use_config,
+        )
+        typer.echo(f"Converted: {script_path} -> {converted_path}")
+    elif from_format == "notebook" and to_format == "script":
+        converted_path = convert_notebook_to_script(notebook_path, script_path)
+        typer.echo(f"Converted: {notebook_path} -> {converted_path}")
+    else:
+        raise typer.BadParameter("Only script <-> notebook conversions are supported.")