gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/job_mgmt/glue_jobs.py

@@ -0,0 +1,1323 @@
+"""
+Glue job sync tooling for real AWS Glue resources.
+
+This module pulls AWS Glue job configs and scripts into the repo so they can be
+tracked alongside migration artifacts. The main flow:
+
+- Fetch Glue jobs via boto3 using an AWS CLI credential profile when provided.
+- Download the job config JSON, the script (.py), and (when present) the
+  notebook (.ipynb) from the job's ScriptLocation bucket.
+- If the job is a NOTEBOOK, remove outputs to keep diffs clean.
+
+These helpers are different from the local fixture commands in
+``gluekit.commands.local_commands``: they use real AWS Glue and S3 APIs, and
+``profile_name`` means an AWS CLI/boto3 credential profile.
+
+Preferred usage:
+- Use the Typer CLI for a git-like flow:
+  gluekit pull <job_name> or gluekit pull "*"
+
+Outputs:
+- glue/glue_full_job_list.csv: complete job list for auditing.
+- glue/configs/<slug>.json: job config with SourceControlDetails metadata.
+- glue/scripts/<slug>.py and glue/notebooks/<slug>.ipynb (when available).
+"""
+
+import csv
+import json
+import re
+import subprocess
+import tempfile
+import uuid
+import zipfile
+from copy import deepcopy
+from datetime import datetime
+from pathlib import Path
+
+import boto3
+from botocore.exceptions import BotoCoreError, ClientError
+from slugify import slugify
+
+from dotenv import load_dotenv
+from .magics import build_magic_cell_sources
+from .schema import DEPRECATED_JOB_FIELDS, get_updateable_job_fields
+
+load_dotenv()
+
+
+def normalize_glue_config_data(config_data):
+    if not isinstance(config_data, dict):
+        return config_data
+
+    job_data = config_data.get("Job")
+    if not isinstance(job_data, dict):
+        return config_data
+
+    normalized = deepcopy(job_data)
+    for key, value in config_data.items():
+        if key == "Job":
+            continue
+        if key == "SourceControlDetails" and isinstance(value, dict):
+            existing = normalized.get("SourceControlDetails")
+            if isinstance(existing, dict):
+                merged = deepcopy(value)
+                merged.update(existing)
+                normalized["SourceControlDetails"] = merged
+            else:
+                normalized["SourceControlDetails"] = deepcopy(value)
+            continue
+        normalized.setdefault(key, value)
+
+    return normalized
+
+
+def _new_boto3_session(profile_name=None):
+    if profile_name:
+        return boto3.Session(profile_name=profile_name)
+    return boto3.Session()
+
+
+def ensure_aws_login(profile_name=None, auto_login=True):
+    """
+    Ensure real AWS credentials are valid for an AWS CLI/boto3 profile.
+
+    If credentials are invalid and auto_login is enabled, trigger
+    ``aws sso login`` and return a fresh boto3 session.
+    """
+    session = _new_boto3_session(profile_name)
+    sts = session.client("sts")
+    try:
+        sts.get_caller_identity()
+        return session
+    except (BotoCoreError, ClientError):
+        if not auto_login:
+            raise
+
+        profile_label = profile_name or "default"
+        print(
+            f"AWS auth missing/expired for profile '{profile_label}'. Running aws sso login..."
+        )
+        login_cmd = ["aws", "sso", "login"]
+        if profile_name:
+            login_cmd.extend(["--profile", profile_name])
+        try:
+            subprocess.run(login_cmd, check=True)
+        except FileNotFoundError as exc:
+            raise RuntimeError(
+                "AWS CLI is not installed or not on PATH. It is required for SSO login."
+            ) from exc
+        except subprocess.CalledProcessError as exc:
+            raise RuntimeError("aws sso login failed or was canceled.") from exc
+        return _new_boto3_session(profile_name)
+
+
+def custom_serializer(obj):
+    if isinstance(obj, datetime):
+        return obj.isoformat()  # Converts datetime to an ISO 8601 string
+    raise TypeError(f"Type {type(obj)} not serializable")
+
+
+def list_glue_jobs(profile_name=None, auto_login=True):
+    session = ensure_aws_login(profile_name=profile_name, auto_login=auto_login)
+    glue = session.client("glue")
+    jobs = []
+    next_token = None
+    while True:
+        response = (
+            glue.get_jobs(NextToken=next_token) if next_token else glue.get_jobs()
+        )
+        jobs.extend(response["Jobs"])
+        next_token = response.get("NextToken")
+        if not next_token:
+            break
+    return jobs
+
+
+def get_glue_job_config(name, profile_name=None, auto_login=True):
+    """Fetch one Glue job config without writing local artifacts."""
+    session = ensure_aws_login(profile_name=profile_name, auto_login=auto_login)
+    glue = session.client("glue")
+    return normalize_glue_config_data(glue.get_job(JobName=name).get("Job"))
+
+
+def parse_s3_bucket_and_prefix(s3_path):
+    s3_path = s3_path.replace("s3://", "")
+    bucket_name, prefix = s3_path.split("/", 1)
+    return bucket_name, prefix
+
+
+def _strip_deprecated_fields(job_data):
+    for field in DEPRECATED_JOB_FIELDS:
+        job_data.pop(field, None)
+    return job_data
+
+
+def download_glue_job_files(
+    name,
+    config_path,
+    local_path=None,
+    remove_nb_outputs=True,
+    include_components=None,
+    existing_sc=None,
+    profile_name=None,
+    auto_login=True,
+):
+
+    session = ensure_aws_login(profile_name=profile_name, auto_login=auto_login)
+    glue = session.client("glue")
+    job = normalize_glue_config_data(glue.get_job(JobName=name).get("Job"))
+    if not job:
+        print(f"Job {name} not found")
+        return None
+    s3 = session.client("s3")
+
+    bucket_name, script_location = parse_s3_bucket_and_prefix(
+        job["Command"]["ScriptLocation"]
+    )
+    # Define paths
+    config_file_name = str(config_path)
+    if local_path:
+        script_file_name = local_path
+    else:
+        script_file_name = f"glue/scripts/{slugify(job['Name'])}.py"
+
+    script_path = Path(script_file_name)
+    if "scripts" in script_path.parts:
+        notebook_file_name = str(
+            Path(
+                *(
+                    "notebooks" if part == "scripts" else part
+                    for part in script_path.parts
+                )
+            ).with_suffix(".ipynb")
+        )
+    else:
+        notebook_file_name = str(
+            Path("glue/notebooks") / script_path.with_suffix(".ipynb").name
+        )
+
+    # Get notebook
+    if "scripts/" in script_location and script_location.endswith(".py"):
+        notebook_location = script_location.replace("scripts/", "notebooks/").replace(
+            ".py", ".ipynb"
+        )
+    else:
+        notebook_location = script_location.replace(".py", ".ipynb")
+
+    print(f"Notebook location: {notebook_location}")
+
+    notebook_dict = None
+    if "notebook" in include_components:
+        response = s3.list_objects_v2(Bucket=bucket_name, Prefix=notebook_location)
+        if "Contents" in response:
+            response = s3.get_object(Bucket=bucket_name, Key=notebook_location)
+            notebook_content = response["Body"].read().decode("utf-8")
+            notebook_dict = json.loads(notebook_content)
+            if remove_nb_outputs:
+                notebook_dict = clear_notebook_outputs(notebook_dict)
+                print(f"Cleared outputs in notebook {notebook_file_name}")
+        else:
+            print(
+                f"No notebook found at {notebook_location}, skipping notebook download."
+            )
+
+    job.setdefault("SourceControlDetails", {})
+    if existing_sc:
+        for key, value in existing_sc.items():
+            job["SourceControlDetails"].setdefault(key, value)
+    job["SourceControlDetails"]["NotebookLocation"] = notebook_file_name
+    job["SourceControlDetails"]["ScriptLocation"] = script_file_name
+    job["SourceControlDetails"].pop("LocalPath", None)
+    job["SourceControlDetails"]["RemoveNbOutputs"] = remove_nb_outputs
+    _strip_deprecated_fields(job)
+
+    # Write files
+    if "config" in include_components:
+        Path(config_file_name).write_text(
+            json.dumps(job, indent=4, default=custom_serializer)
+        )
+        print(f"Configuration file created: {config_file_name}")
+
+    if "script" in include_components:
+        script_path.parent.mkdir(parents=True, exist_ok=True)
+        s3.download_file(bucket_name, script_location, script_file_name)
+        print(f"Script downloaded: {script_file_name}")
+
+    if notebook_dict and "notebook" in include_components:
+        notebook_path = Path(notebook_file_name)
+        notebook_path.parent.mkdir(parents=True, exist_ok=True)
+        notebook_path.write_text(
+            json.dumps(notebook_dict, indent=4, default=custom_serializer)
+        )
+        print(f"Notebook downloaded: {notebook_file_name}")
+
+    if "script" in include_components and "notebook" in include_components:
+        _ensure_local_script_from_notebook(notebook_file_name, script_file_name)
+        _ensure_local_notebook_from_script(
+            script_file_name,
+            notebook_file_name,
+            config_data=job,
+            include_magics=True,
+        )
+
+    if include_components.intersection({"extra-py-files", "extra-files"}):
+        sc_for_extras = job.get("SourceControlDetails", {})
+        additional_mappings, additional_errors = _collect_additional_file_mappings(
+            sc_for_extras
+        )
+        for error in additional_errors:
+            print(f"Additional files error: {error}")
+
+        include_sources: set[str] = set()
+        if "extra-py-files" in include_components:
+            include_sources.update({"AdditionalPythonFiles", "ExtraPyFiles"})
+        if "extra-files" in include_components:
+            include_sources.update({"AdditionalFiles", "ExtraFiles"})
+
+        additional_mappings = [
+            mapping
+            for mapping in additional_mappings
+            if mapping["source"] in include_sources
+        ]
+
+        if not additional_mappings:
+            print("No additional files configured in SourceControlDetails; skipping.")
+        for mapping in additional_mappings:
+            local = Path(mapping["local_path"])
+            s3_path = mapping["s3_path"]
+            if not s3_path.startswith("s3://"):
+                print(f"Additional file has invalid S3 path: {s3_path}")
+                continue
+            bucket, key = parse_s3_bucket_and_prefix(s3_path)
+            is_zip = s3_path.endswith(".zip")
+            treat_as_dir = is_zip and (
+                local.exists() and local.is_dir() or local.suffix != ".zip"
+            )
+            if treat_as_dir:
+                local.mkdir(parents=True, exist_ok=True)
+                with tempfile.TemporaryDirectory() as temp_dir:
+                    zip_path = Path(temp_dir) / "additional_files.zip"
+                    s3.download_file(bucket, key, str(zip_path))
+                    with zipfile.ZipFile(zip_path, "r") as archive:
+                        archive.extractall(local)
+                print(f"Extracted additional archive: {s3_path} -> {local}")
+            else:
+                local.parent.mkdir(parents=True, exist_ok=True)
+                s3.download_file(bucket, key, str(local))
+                print(f"Downloaded additional file: {s3_path} -> {local}")
+
+    print("S3 sync complete.")
+
+
+def _resolve_notebook_path(script_path):
+    if "scripts" in script_path.parts:
+        return str(
+            Path(
+                *(
+                    "notebooks" if part == "scripts" else part
+                    for part in script_path.parts
+                )
+            ).with_suffix(".ipynb")
+        )
+    return str(Path("glue/notebooks") / script_path.with_suffix(".ipynb").name)
+
+
+def _script_location_to_notebook_key(script_key):
+    if "scripts/" in script_key and script_key.endswith(".py"):
+        return script_key.replace("scripts/", "notebooks/").replace(".py", ".ipynb")
+    return script_key.replace(".py", ".ipynb")
+
+
+def _load_config_data(config):
+    if isinstance(config, (str, Path)):
+        config_path = Path(config)
+        try:
+            config_data = normalize_glue_config_data(
+                json.loads(config_path.read_text())
+            )
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"Invalid JSON in {config_path}") from exc
+        return config_data, config_path
+    if isinstance(config, dict):
+        return normalize_glue_config_data(config), None
+    raise TypeError("config must be a dict or a path-like string")
+
+
+def _collect_additional_file_mappings(sc):
+    mappings: list[dict[str, str]] = []
+    errors: list[str] = []
+    seen: set[tuple[str, str]] = set()
+    sources = [
+        ("AdditionalPythonFiles", sc.get("AdditionalPythonFiles")),
+        ("ExtraPyFiles", sc.get("ExtraPyFiles")),
+        ("AdditionalFiles", sc.get("AdditionalFiles")),
+        ("ExtraFiles", sc.get("ExtraFiles")),
+    ]
+
+    for label, items in sources:
+        if not items:
+            continue
+        if isinstance(items, dict):
+            items = [items]
+        if not isinstance(items, list):
+            errors.append(f"{label} must be a list or dict.")
+            continue
+        for item in items:
+            local_path = None
+            s3_path = None
+            if isinstance(item, dict):
+                local_path = (
+                    item.get("LocalPath")
+                    or item.get("local_path")
+                    or item.get("localPath")
+                )
+                s3_path = (
+                    item.get("S3Path") or item.get("s3_path") or item.get("s3Path")
+                )
+            elif isinstance(item, (list, tuple)) and len(item) == 2:
+                local_path, s3_path = item
+            else:
+                errors.append(f"{label} contains an unsupported entry: {item!r}")
+                continue
+
+            if not local_path or not s3_path:
+                errors.append(f"{label} entry missing LocalPath/S3Path: {item!r}")
+                continue
+
+            key = (str(local_path), str(s3_path))
+            if key in seen:
+                continue
+            seen.add(key)
+            mappings.append(
+                {
+                    "local_path": str(local_path),
+                    "s3_path": str(s3_path),
+                    "source": label,
+                }
+            )
+
+    return mappings, errors
+
+
+def _parse_csv_list(value):
+    if not value:
+        return []
+    return [item.strip() for item in next(csv.reader([value])) if item.strip()]
+
+
+def _infer_local_dependency_path(s3_path, script_location, local_script_path):
+    if not isinstance(s3_path, str) or not s3_path.startswith("s3://"):
+        return None
+
+    try:
+        _bucket, key = parse_s3_bucket_and_prefix(s3_path)
+    except ValueError:
+        return None
+
+    key_path = Path(key)
+    candidates: list[Path] = [key_path]
+    if key_path.parts and key_path.parts[0] != "glue":
+        candidates.append(Path("glue") / key_path)
+
+    if isinstance(script_location, str) and script_location.startswith("s3://"):
+        try:
+            _script_bucket, script_key = parse_s3_bucket_and_prefix(script_location)
+        except ValueError:
+            script_key = None
+        if script_key is not None:
+            script_dir = Path(script_key).parent
+            key_parts = key_path.parts
+            script_dir_parts = script_dir.parts
+            if (
+                script_dir_parts
+                and key_parts[: len(script_dir_parts)] == script_dir_parts
+            ):
+                rel = key_path.relative_to(script_dir)
+                candidates.append(local_script_path.parent / rel)
+
+    seen: set[str] = set()
+    for candidate in candidates:
+        candidate_key = candidate.as_posix()
+        if candidate_key in seen:
+            continue
+        seen.add(candidate_key)
+        if candidate.exists():
+            return candidate
+
+    return None
+
+
+def _collect_configured_dependency_mappings(config_data, explicit_mappings):
+    default_args = config_data.get("DefaultArguments", {})
+    if not isinstance(default_args, dict):
+        return [], []
+
+    command = config_data.get("Command", {})
+    if not isinstance(command, dict):
+        command = {}
+    sc = config_data.get("SourceControlDetails", {})
+    if not isinstance(sc, dict):
+        sc = {}
+
+    job_name = config_data.get("Name") or "unknown"
+    local_script_path = Path(
+        sc.get("ScriptLocation")
+        or sc.get("LocalPath")
+        or f"glue/scripts/{slugify(job_name)}.py"
+    )
+    script_location = command.get("ScriptLocation")
+
+    explicit_s3_paths = {mapping["s3_path"] for mapping in explicit_mappings}
+    seen: set[tuple[str, str]] = set()
+    mappings: list[dict[str, str]] = []
+    errors: list[str] = []
+    dependency_args = (
+        ("DefaultArguments.--additional-python-modules", "--additional-python-modules"),
+        ("DefaultArguments.--extra-py-files", "--extra-py-files"),
+    )
+
+    for source_label, arg_name in dependency_args:
+        for item in _parse_csv_list(default_args.get(arg_name)):
+            if not item.startswith("s3://"):
+                continue
+            if item in explicit_s3_paths:
+                continue
+
+            local_path = _infer_local_dependency_path(
+                item, script_location, local_script_path
+            )
+            if local_path is None:
+                errors.append(
+                    f"{arg_name} entry has no matching local path for upload: {item}"
+                )
+                continue
+
+            key = (local_path.as_posix(), item)
+            if key in seen:
+                continue
+            seen.add(key)
+            mappings.append(
+                {
+                    "local_path": local_path.as_posix(),
+                    "s3_path": item,
+                    "source": source_label,
+                }
+            )
+
+    return mappings, errors
+
+
+def _zip_directory(source_dir, zip_path):
+    source_dir = Path(source_dir)
+    zip_path = Path(zip_path)
+    zip_path.parent.mkdir(parents=True, exist_ok=True)
+
+    base_dir = source_dir.parent
+    with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+        for path in source_dir.rglob("*"):
+            if path.is_dir():
+                continue
+            archive.write(path, path.relative_to(base_dir).as_posix())
+
+
+def _build_job_update(config_data):
+    allowed_keys = {
+        "Description",
+        "LogUri",
+        "Role",
+        "ExecutionProperty",
+        "Command",
+        "DefaultArguments",
+        "NonOverridableArguments",
+        "Connections",
+        "MaxRetries",
+        "Timeout",
+        "MaxCapacity",
+        "WorkerType",
+        "NumberOfWorkers",
+        "SecurityConfiguration",
+        "NotificationProperty",
+        "GlueVersion",
+        "CodeGenConfigurationNodes",
+        "ExecutionClass",
+        "JobMode",
+        "ExecutionClass",
+        "JobRunQueuingEnabled",
+    }
+    return {
+        key: config_data[key]
+        for key in allowed_keys
+        if key in config_data and key not in DEPRECATED_JOB_FIELDS
+    }
+
+
+def _get_job_update_allowed_fields(profile_name=None):
+    del profile_name
+    return get_updateable_job_fields(prefer_live=True)
+
+
+def _filter_job_update_fields(job_update, allowed_fields):
+    if not allowed_fields:
+        return job_update, []
+    filtered = {
+        key: value for key, value in job_update.items() if key in allowed_fields
+    }
+    dropped = [key for key in job_update if key not in allowed_fields]
+    return filtered, dropped
+
+
+def update_glue_job_from_config(
+    config_data, dry_run=False, profile_name=None, auto_login=True
+):
+    """
+    Uses the Glue service model (via botocore) when available to validate
+    JobUpdate fields and report anything dropped during dry runs.
+
+    TODO: Consider fetching the current Glue job definition and merging with
+    local config values before update, so unchanged fields stay in sync.
+    """
+    name = config_data.get("Name")
+    if not name:
+        print("Missing Name in config data, skipping Glue job update.")
+        return False
+
+    deprecated_in_config = [key for key in DEPRECATED_JOB_FIELDS if key in config_data]
+    if deprecated_in_config:
+        print(
+            f"Skipping deprecated fields for {name}: {', '.join(sorted(deprecated_in_config))}"
+        )
+
+    job_update = _build_job_update(config_data)
+    allowed_fields = _get_job_update_allowed_fields(profile_name=profile_name)
+    job_update, dropped = _filter_job_update_fields(job_update, allowed_fields)
+    if dropped:
+        print(
+            f"Dropping unsupported JobUpdate fields for {name}: {', '.join(sorted(dropped))}"
+        )
+    if dry_run and not allowed_fields:
+        print("Model-based validation unavailable; using local allowlist only.")
+    if not job_update:
+        print(f"No updatable fields found for {name}, skipping Glue job update.")
+        return False
+
+    if dry_run:
+        keys = ", ".join(sorted(job_update.keys()))
+        print(f"Would update Glue job {name} with: {keys}")
+        return True
+
+    session = ensure_aws_login(profile_name=profile_name, auto_login=auto_login)
+    glue = session.client("glue")
+    glue.update_job(JobName=name, JobUpdate=job_update)
+    print(f"Updated Glue job configuration: {name}")
+    return True
+
+
+def _job_supports_notebook_artifacts(config_data):
+    job_mode = str(config_data.get("JobMode") or "").strip().upper()
+    if not job_mode:
+        return True
+    return job_mode == "NOTEBOOK"
+
+
+def resolve_glue_artifact_mappings(config_data, include_components=None):
+    """Resolve local-to-S3 artifact mappings without contacting AWS."""
+    config_data = normalize_glue_config_data(config_data)
+    include_components = set(
+        include_components
+        or {
+            "script",
+            "notebook",
+            "additional-python-modules",
+            "extra-files",
+        }
+    )
+    warnings: list[str] = []
+    dependency_warnings: list[str] = []
+
+    command = config_data.get("Command", {})
+    if not isinstance(command, dict):
+        warnings.append("Command must be an object.")
+        command = {}
+
+    sc = config_data.get("SourceControlDetails", {})
+    if not isinstance(sc, dict):
+        warnings.append("SourceControlDetails must be an object.")
+        sc = {}
+
+    name = config_data.get("Name") or "unknown"
+    script_path = Path(
+        sc.get("ScriptLocation")
+        or sc.get("LocalPath")
+        or f"glue/scripts/{slugify(name)}.py"
+    )
+    script_s3_path = command.get("ScriptLocation")
+    notebook_enabled = (
+        "notebook" in include_components
+        and _job_supports_notebook_artifacts(config_data)
+    )
+    notebook_path = None
+    notebook_s3_path = None
+    mappings: list[dict[str, str]] = []
+
+    if "script" in include_components:
+        if isinstance(script_s3_path, str) and script_s3_path.startswith("s3://"):
+            mappings.append(
+                {
+                    "artifact_type": "script",
+                    "local_path": script_path.as_posix(),
+                    "s3_path": script_s3_path,
+                    "source": "Command.ScriptLocation",
+                }
+            )
+        else:
+            warnings.append("Command.ScriptLocation must be a valid S3 path.")
+
+    if notebook_enabled:
+        notebook_value = sc.get("NotebookPath") or sc.get("NotebookLocation")
+        notebook_path = (
+            Path(notebook_value)
+            if notebook_value
+            else Path(_resolve_notebook_path(script_path))
+        )
+        if isinstance(script_s3_path, str) and script_s3_path.startswith("s3://"):
+            try:
+                bucket_name, script_key = parse_s3_bucket_and_prefix(script_s3_path)
+            except ValueError:
+                warnings.append(
+                    f"Cannot derive notebook S3 path from Command.ScriptLocation: {script_s3_path}"
+                )
+            else:
+                notebook_key = _script_location_to_notebook_key(script_key)
+                notebook_s3_path = f"s3://{bucket_name}/{notebook_key}"
+                mappings.append(
+                    {
+                        "artifact_type": "notebook",
+                        "local_path": notebook_path.as_posix(),
+                        "s3_path": notebook_s3_path,
+                        "source": "Command.ScriptLocation",
+                    }
+                )
+
+    include_sources: set[str] = set()
+    dependency_components = {"additional-python-modules", "extra-py-files"}
+    if include_components.intersection(dependency_components):
+        include_sources.update({"AdditionalPythonFiles", "ExtraPyFiles"})
+    if "extra-files" in include_components:
+        include_sources.update({"AdditionalFiles", "ExtraFiles"})
+
+    additional_mappings: list[dict[str, str]] = []
+    if include_sources:
+        additional_mappings, additional_errors = _collect_additional_file_mappings(sc)
+        dependency_warnings.extend(additional_errors)
+        warnings.extend(additional_errors)
+        additional_mappings = [
+            mapping
+            for mapping in additional_mappings
+            if mapping["source"] in include_sources
+        ]
+
+    if include_components.intersection(dependency_components):
+        configured_mappings, configured_errors = (
+            _collect_configured_dependency_mappings(
+                config_data,
+                additional_mappings,
+            )
+        )
+        additional_mappings.extend(configured_mappings)
+        dependency_warnings.extend(configured_errors)
+        warnings.extend(configured_errors)
+
+    artifact_types = {
+        "AdditionalPythonFiles": "additional-python-file",
+        "ExtraPyFiles": "extra-py-file",
+        "AdditionalFiles": "extra-file",
+        "ExtraFiles": "extra-file",
+        "DefaultArguments.--additional-python-modules": "additional-python-module",
+        "DefaultArguments.--extra-py-files": "extra-py-file",
+    }
+    for mapping in additional_mappings:
+        mappings.append(
+            {
+                **mapping,
+                "artifact_type": artifact_types.get(mapping["source"], "dependency"),
+            }
+        )
+
+    deduplicated: list[dict[str, str]] = []
+    seen: set[tuple[str, str]] = set()
+    for mapping in mappings:
+        key = (mapping["local_path"], mapping["s3_path"])
+        if key in seen:
+            continue
+        seen.add(key)
+        deduplicated.append(mapping)
+
+    return {
+        "mappings": deduplicated,
+        "warnings": warnings,
+        "dependency_warnings": dependency_warnings,
+        "script_path": script_path,
+        "script_s3_path": script_s3_path,
+        "notebook_enabled": notebook_enabled,
+        "notebook_path": notebook_path,
+        "notebook_s3_path": notebook_s3_path,
+    }
+
+
+def upload_glue_job_files_from_config(
+    config,
+    dry_run=False,
+    update_job_config=True,
+    include_components=None,
+    profile_name=None,
+    auto_login=True,
+):
+    config_data, config_path = _load_config_data(config)
+    include_components = set(
+        include_components
+        or {
+            "script",
+            "notebook",
+            "additional-python-modules",
+            "extra-files",
+            "job-config",
+        }
+    )
+    artifact_resolution = resolve_glue_artifact_mappings(
+        config_data,
+        include_components=include_components,
+    )
+    notebook_enabled = artifact_resolution["notebook_enabled"]
+    script_location = artifact_resolution["script_s3_path"]
+    needs_script_location = "script" in include_components or notebook_enabled
+    if needs_script_location and not script_location:
+        print(f"Missing Command.ScriptLocation in {config_path or 'config data'}")
+        return False
+
+    if script_location:
+        bucket_name, script_key = parse_s3_bucket_and_prefix(script_location)
+        notebook_key = _script_location_to_notebook_key(script_key)
+    else:
+        bucket_name = None
+        script_key = None
+        notebook_key = None
+    script_path = artifact_resolution["script_path"]
+    notebook_path = artifact_resolution["notebook_path"]
+
+    script_available = script_path.exists()
+    notebook_available = bool(notebook_path and Path(notebook_path).exists())
+
+    if "script" in include_components and notebook_path:
+        script_available = (
+            _ensure_local_script_from_notebook(
+                notebook_path,
+                script_path,
+                dry_run=dry_run,
+            )
+            or script_available
+        )
+    if notebook_enabled and notebook_path:
+        notebook_available = (
+            _ensure_local_notebook_from_script(
+                script_path,
+                notebook_path,
+                config_data=config_data,
+                include_magics=True,
+                dry_run=dry_run,
+            )
+            or notebook_available
+        )
+
+    if "script" in include_components and not script_available:
+        print(f"Local script not found: {script_path}")
+        return False
+
+    additional_mappings = [
+        mapping
+        for mapping in artifact_resolution["mappings"]
+        if mapping["artifact_type"] not in {"script", "notebook"}
+    ]
+    additional_errors = artifact_resolution["dependency_warnings"]
+
+    if dry_run:
+        if "script" in include_components:
+            print(
+                f"Would upload script: {script_path} -> s3://{bucket_name}/{script_key}"
+            )
+        if notebook_enabled:
+            if notebook_available:
+                print(
+                    f"Would upload notebook: {notebook_path} -> s3://{bucket_name}/{notebook_key}"
+                )
+            else:
+                print(f"Notebook not found locally: {notebook_path}")
+        for error in additional_errors:
+            print(f"Additional files error: {error}")
+        for mapping in additional_mappings:
+            local = Path(mapping["local_path"])
+            s3_path = mapping["s3_path"]
+            if not local.exists():
+                print(f"Additional file not found locally: {local}")
+                continue
+            if not s3_path.startswith("s3://"):
+                print(f"Additional file has invalid S3 path: {s3_path}")
+                continue
+            if local.is_dir():
+                if not s3_path.endswith(".zip"):
+                    print(f"Additional directory must target a .zip S3 path: {s3_path}")
+                    continue
+                print(f"Would zip directory: {local} -> {s3_path}")
+                continue
+            bucket, key = parse_s3_bucket_and_prefix(s3_path)
+            print(f"Would upload additional file: {local} -> s3://{bucket}/{key}")
+        if update_job_config and "job-config" in include_components:
+            update_glue_job_from_config(
+                config_data,
+                dry_run=True,
+                profile_name=profile_name,
+                auto_login=auto_login,
+            )
+        return True
+
+    ok = True
+    file_components = include_components.intersection(
+        {
+            "script",
+            "notebook",
+            "additional-python-modules",
+            "extra-py-files",
+            "extra-files",
+        }
+    )
+    needs_aws = bool(
+        file_components or (update_job_config and "job-config" in include_components)
+    )
+    session = (
+        ensure_aws_login(profile_name=profile_name, auto_login=auto_login)
+        if needs_aws
+        else None
+    )
+    s3 = session.client("s3") if file_components and session else None
+
+    if "script" in include_components:
+        s3.upload_file(str(script_path), bucket_name, script_key)
+        print(f"Uploaded script: {script_path} -> s3://{bucket_name}/{script_key}")
+
+    if notebook_enabled:
+        if notebook_available:
+            s3.upload_file(str(notebook_path), bucket_name, notebook_key)
+            print(
+                f"Uploaded notebook: {notebook_path} -> s3://{bucket_name}/{notebook_key}"
+            )
+        else:
+            print(f"Notebook not found locally: {notebook_path}")
+
+    for error in additional_errors:
+        print(f"Additional files error: {error}")
+        ok = False
+
+    for mapping in additional_mappings:
+        local = Path(mapping["local_path"])
+        s3_path = mapping["s3_path"]
+        if not local.exists():
+            print(f"Additional file not found locally: {local}")
+            ok = False
+            continue
+        if not s3_path.startswith("s3://"):
+            print(f"Additional file has invalid S3 path: {s3_path}")
+            ok = False
+            continue
+        if local.is_dir():
+            if not s3_path.endswith(".zip"):
+                print(f"Additional directory must target a .zip S3 path: {s3_path}")
+                ok = False
+                continue
+            bucket, key = parse_s3_bucket_and_prefix(s3_path)
+            with tempfile.TemporaryDirectory() as temp_dir:
+                zip_path = Path(temp_dir) / f"{local.name}.zip"
+                _zip_directory(local, zip_path)
+                s3.upload_file(str(zip_path), bucket, key)
+            print(f"Uploaded additional directory: {local} -> s3://{bucket}/{key}")
+            continue
+        bucket, key = parse_s3_bucket_and_prefix(s3_path)
+        s3.upload_file(str(local), bucket, key)
+        print(f"Uploaded additional file: {local} -> s3://{bucket}/{key}")
+
+    if update_job_config and "job-config" in include_components:
+        updated = update_glue_job_from_config(
+            config_data,
+            dry_run=False,
+            profile_name=profile_name,
+            auto_login=auto_login,
+        )
+        ok = ok and updated
+
+    return ok
+
+
+def clear_notebook_outputs(notebook):
+    """
+    Clears the outputs of all code cells in a Jupyter notebook.
+
+    This function takes a Jupyter notebook (either as a dictionary, JSON string or a file path),
+    clears the outputs of all code cells
+    Args:
+        notebook (str or dict): The Jupyter notebook to be processed. This can be either:
+        - A string representing the path to the notebook file.
+        - A JSON string representing the notebook content.
+        - A dictionary representing the notebook content.
+    Raises:
+        ValueError: If the provided string is neither a valid path nor a valid JSON string.
+    Returns:
+        None
+    Example:
+        # This can also be the JSON/dictionary version of the notebook content
+        clear_notebook_outputs(notebook_dict, 'path/to/output_notebook.ipynb')
+    """
+
+    if isinstance(notebook, (str, Path)):
+        try:
+            # Check if the string is a path-like string
+            if Path(notebook).exists():
+                notebook = json.loads(Path(notebook).read_text(encoding="utf-8"))
+            else:
+                # Try to parse the string as JSON
+                notebook = json.loads(notebook)
+        except json.JSONDecodeError:
+            raise ValueError(
+                "The provided string is neither a valid path nor a valid JSON string."
+            )
+
+    for cell in notebook.get("cells", []):
+        if cell.get("cell_type") == "code":
+            cell["outputs"] = []
+
+    return notebook
+
+
+def _new_code_cell(source_lines):
+    return {
+        "cell_type": "code",
+        "id": str(uuid.uuid4()),
+        "metadata": {
+            "tags": [],
+            "trusted": True,
+            "vscode": {"languageId": "python_glue_session"},
+        },
+        "source": source_lines,
+        "execution_count": None,
+        "outputs": [],
+    }
+
+
+def _new_markdown_cell(source_lines):
+    return {
+        "cell_type": "markdown",
+        "id": str(uuid.uuid4()),
+        "metadata": {},
+        "source": source_lines,
+    }
+
+
+def _parse_percent_cell_marker(raw_line):
+    match = re.match(r"^\s*#\s*%%(.*)$", raw_line)
+    if not match:
+        return None
+
+    marker_text = match.group(1).strip()
+    return {
+        "marker_text": marker_text,
+        "is_markdown": "markdown" in marker_text.casefold(),
+    }
+
+
+def _canonical_code_cell_marker():
+    return "#%%\n"
+
+
+def _code_cell_has_payload(source_lines):
+    for line in source_lines:
+        if _parse_percent_cell_marker(line):
+            continue
+        if line.strip():
+            return True
+    return False
+
+
+def _code_cell_starts_with_percent_marker(source_lines):
+    for line in source_lines:
+        if not line.strip():
+            continue
+        return _parse_percent_cell_marker(line) is not None
+    return False
+
+
+def _ensure_local_script_from_notebook(notebook_path, script_path, *, dry_run=False):
+    notebook_path = Path(notebook_path)
+    script_path = Path(script_path)
+    if script_path.exists() or not notebook_path.exists():
+        return False
+    if dry_run:
+        print(f"Would generate script from notebook: {notebook_path} -> {script_path}")
+        return True
+    convert_notebook_to_script(notebook_path, script_path)
+    print(f"Generated script from notebook: {notebook_path} -> {script_path}")
+    return True
+
+
+def _ensure_local_notebook_from_script(
+    script_path,
+    notebook_path,
+    *,
+    config_data=None,
+    include_magics=True,
+    dry_run=False,
+):
+    script_path = Path(script_path)
+    notebook_path = Path(notebook_path)
+    if notebook_path.exists() or not script_path.exists():
+        return False
+    if dry_run:
+        print(f"Would generate notebook from script: {script_path} -> {notebook_path}")
+        return True
+    convert_script_to_notebook(
+        script_path,
+        notebook_path,
+        config_data=config_data,
+        include_magics=include_magics,
+    )
+    print(f"Generated notebook from script: {script_path} -> {notebook_path}")
+    return True
+
+
+def _script_to_cells(script_text):
+    cells = []
+    current_lines = []
+    current_type = "code"
+
+    def flush():
+        if not current_lines:
+            return
+        if current_type == "code":
+            if not _code_cell_has_payload(current_lines):
+                return
+        elif not any(line.strip() for line in current_lines):
+            return
+        source = current_lines
+        if current_type == "code":
+            cell = _new_code_cell(source)
+        else:
+            cell = _new_markdown_cell(source)
+        cells.append(cell)
+
+    for line in script_text.splitlines(keepends=True):
+        marker = _parse_percent_cell_marker(line)
+        if marker:
+            if current_type == "code" and not _code_cell_has_payload(current_lines):
+                if marker["is_markdown"]:
+                    current_lines = []
+                else:
+                    current_lines = [
+                        existing_line
+                        for existing_line in current_lines
+                        if _parse_percent_cell_marker(existing_line)
+                    ]
+            else:
+                flush()
+                current_lines = []
+            current_type = "markdown" if marker["is_markdown"] else "code"
+            if current_type == "code":
+                current_lines.append(_canonical_code_cell_marker())
+            continue
+        current_lines.append(line)
+
+    flush()
+    return cells
+
+
+def _strip_generated_script_config_cells(script_text):
+    def is_glue_magic_line(raw_line):
+        if _parse_percent_cell_marker(raw_line):
+            return False
+        candidate = raw_line.lstrip()
+        if candidate.startswith("#"):
+            candidate = candidate[1:].lstrip()
+        return (
+            candidate.startswith("%")
+            or "%%configure" in candidate
+            or "AWS Glue configs" in candidate
+        )
+
+    def split_into_cells(all_lines):
+        starts = [
+            idx
+            for idx, line in enumerate(all_lines)
+            if _parse_percent_cell_marker(line)
+        ]
+        if not starts:
+            return [all_lines]
+        if starts[0] != 0:
+            starts = [0] + starts
+        boundaries = starts[1:] + [len(all_lines)]
+        return [all_lines[start:end] for start, end in zip(starts, boundaries)]
+
+    kept = []
+    for cell_lines in split_into_cells(script_text.splitlines(keepends=True)):
+        if any(is_glue_magic_line(line) for line in cell_lines):
+            first_code_idx = None
+            for idx, line in enumerate(cell_lines):
+                stripped = line.strip()
+                if not stripped:
+                    continue
+                if _parse_percent_cell_marker(line):
+                    continue
+                if stripped.startswith("#"):
+                    continue
+                first_code_idx = idx
+                break
+            if first_code_idx is not None:
+                kept.extend(cell_lines[first_code_idx:])
+            continue
+        kept.extend(cell_lines)
+
+    return "".join(kept)
+
+
+def _build_magic_cells(config_data):
+    if not config_data:
+        return []
+
+    cells = [_new_markdown_cell(["# AWS Glue configs\n"])]
+    cells.extend(
+        _new_code_cell(source_lines)
+        for source_lines in build_magic_cell_sources(config_data)
+    )
+
+    cells.append(_new_markdown_cell(["# AWS Glue Script\n"]))
+
+    return cells
+
+
+def _cell_source_to_lines(source):
+    if source is None:
+        return []
+    if isinstance(source, str):
+        return source.splitlines(keepends=True)
+    lines: list[str] = []
+    for part in source:
+        if part.endswith("\n"):
+            lines.append(part)
+        else:
+            lines.append(f"{part}\n")
+    return lines
+
+
+def _format_markdown_lines(source_lines):
+    formatted: list[str] = []
+    for line in source_lines:
+        stripped = line.rstrip("\n")
+        if stripped.strip():
+            formatted.append(f"# {stripped}\n")
+        else:
+            formatted.append("#\n")
+    return formatted
+
+
+def _is_magic_cell(source_lines):
+    for line in source_lines:
+        stripped = line.lstrip()
+        if not stripped:
+            continue
+        if stripped.startswith("%"):
+            return True
+    return False
+
+
+def _default_notebook_metadata():
+    return {
+        "kernelspec": {
+            "name": "glue_pyspark",
+            "display_name": "Glue PySpark",
+            "language": "python",
+        },
+        "language_info": {
+            "name": "Python_Glue_Session",
+            "mimetype": "text/x-python",
+            "codemirror_mode": {"name": "python", "version": 3},
+            "pygments_lexer": "python3",
+            "file_extension": ".py",
+        },
+    }
+
+
+def convert_notebook_to_script(
+    notebook_path,
+    script_path,
+):
+    notebook_path = Path(notebook_path)
+    script_path = Path(script_path)
+
+    notebook = json.loads(notebook_path.read_text())
+    lines: list[str] = []
+
+    for cell in notebook.get("cells", []):
+        cell_type = cell.get("cell_type")
+        source_lines = _cell_source_to_lines(cell.get("source"))
+        if cell_type == "markdown":
+            lines.append("#%% [markdown]\n")
+            lines.extend(_format_markdown_lines(source_lines))
+        elif cell_type == "code":
+            if _is_magic_cell(source_lines):
+                lines.append("#%% [markdown]\n")
+                lines.extend(_format_markdown_lines(source_lines))
+            else:
+                if not _code_cell_starts_with_percent_marker(source_lines):
+                    lines.append(_canonical_code_cell_marker())
+                lines.extend(source_lines)
+        else:
+            continue
+        if lines and not lines[-1].endswith("\n"):
+            lines[-1] = f"{lines[-1]}\n"
+        lines.append("\n")
+
+    script_path.parent.mkdir(parents=True, exist_ok=True)
+    script_path.write_text("".join(lines))
+    return script_path
+
+
+def convert_script_to_notebook(
+    script_path,
+    notebook_path,
+    config_data=None,
+    include_magics=True,
+):
+    script_path = Path(script_path)
+    notebook_path = Path(notebook_path)
+
+    script_text = script_path.read_text()
+    script_text = re.sub(r"(?m)^\s*job\.commit\(\)\s*(?:#.*)?$\n?", "", script_text)
+    script_text = _strip_generated_script_config_cells(script_text)
+    cells = _script_to_cells(script_text)
+    if include_magics and config_data:
+        cells = _build_magic_cells(config_data) + cells
+
+    if notebook_path.exists():
+        template = json.loads(notebook_path.read_text())
+        metadata = template.get("metadata") or _default_notebook_metadata()
+        nbformat = template.get("nbformat", 4)
+        nbformat_minor = template.get("nbformat_minor", 5)
+    else:
+        metadata = _default_notebook_metadata()
+        nbformat = 4
+        nbformat_minor = 5
+
+    notebook = {
+        "metadata": metadata,
+        "nbformat": nbformat,
+        "nbformat_minor": nbformat_minor,
+        "cells": cells,
+    }
+
+    notebook_path.parent.mkdir(parents=True, exist_ok=True)
+    notebook_path.write_text(json.dumps(notebook, indent=4))
+    return notebook_path