gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/commands/list.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+import typer
+from .helpers import *
+from ..cli import app, glue_config_app
+
+
+@app.command(
+    "list",
+    epilog=_examples_epilog(
+        "gluekit list --help",
+        "gluekit init --help",
+    ),
+)
+def glue_list() -> None:
+    """List real AWS Glue jobs in the configured AWS account."""
+    raise NotImplementedError("glue list command is not yet implemented.")

gluekit/commands/local_commands.py

@@ -0,0 +1,680 @@
+from __future__ import annotations
+
+import posixpath
+from json import dumps
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from ..cli import glue_local_app
+from ..local.local_fixtures import (
+    LOCAL_RUN_CONFIG_FILE,
+    copy_mapped_file,
+    expand_bucket_mappings,
+    expand_directory_mappings,
+    expand_s3_root_mapping,
+    is_s3_uri,
+    load_local_fixture_config,
+    normalize_s3_bucket_name,
+    normalize_s3_object_uri,
+    parse_s3_uri,
+    s3_destination_uri,
+    s3_directory_destination,
+    s3_prefix,
+    save_local_fixture_config,
+)
+from .helpers import (
+    _examples_epilog,
+    _get_checked_out_local_setup_name,
+    _get_local_checkouts,
+    _save_local_checkout,
+)
+
+local_s3_app = typer.Typer(
+    no_args_is_help=True,
+    rich_markup_mode="markdown",
+    help="Manage local mocked S3 fixture mappings; does not call AWS S3.",
+)
+local_ssm_app = typer.Typer(
+    no_args_is_help=True,
+    rich_markup_mode="markdown",
+    help="Manage local mocked SSM Parameter Store values; does not call AWS SSM.",
+)
+glue_local_app.add_typer(local_s3_app, name="s3")
+glue_local_app.add_typer(local_ssm_app, name="ssm")
+S3_BUCKET_ARG_TEMPLATE = "{s3 bucket in mocked profile}={local directory}"
+SSM_PARAM_ARG_TEMPLATE = "{ssm parameter in mocked profile}={local parameter value}"
+
+
+def _resolve_config_path(config_file: str | Path | None = None) -> Path:
+    if config_file is not None:
+        return Path(config_file)
+    return LOCAL_RUN_CONFIG_FILE
+
+
+def _load_config(config_file: str | Path | None = None) -> dict[str, object]:
+    try:
+        return load_local_fixture_config(_resolve_config_path(config_file))
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+
+
+def _save_config(
+    config: dict[str, object], config_file: str | Path | None = None
+) -> None:
+    try:
+        save_local_fixture_config(config, _resolve_config_path(config_file))
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+
+
+def _s3_section(config: dict[str, object]) -> dict[str, object]:
+    return config["s3"]  # type: ignore[return-value]
+
+
+def _ssm_section(config: dict[str, object]) -> dict[str, object]:
+    return config["ssm"]  # type: ignore[return-value]
+
+
+def _s3_bucket_mappings(config: dict[str, object]) -> dict[str, str]:
+    return _s3_section(config)["buckets"]  # type: ignore[return-value]
+
+
+def _set_s3_root_mapping(config: dict[str, object], local_path: str) -> str:
+    source_dir = Path(local_path)
+    if not source_dir.exists() or not source_dir.is_dir():
+        raise typer.BadParameter(
+            f"Local S3 root directory must be an existing directory: {local_path!r}"
+        )
+    _s3_section(config)["root"] = str(source_dir)
+    return str(source_dir)
+
+
+def _s3_objects(config: dict[str, object]) -> dict[str, str]:
+    return _s3_section(config)["objects"]  # type: ignore[return-value]
+
+
+def _s3_directories(config: dict[str, object]) -> list[dict[str, str]]:
+    return _s3_section(config)["directories"]  # type: ignore[return-value]
+
+
+def _ssm_parameters(config: dict[str, object]) -> dict[str, str]:
+    return _ssm_section(config)["parameters"]  # type: ignore[return-value]
+
+
+def _add_s3_bucket_mapping(
+    config: dict[str, object], *, bucket: str, local_path: str
+) -> str:
+    source_dir = Path(local_path)
+    if not source_dir.exists() or not source_dir.is_dir():
+        raise typer.BadParameter(
+            f"Local S3 bucket directory must be an existing directory: {local_path!r}"
+        )
+    bucket_name = normalize_s3_bucket_name(bucket)
+    _s3_bucket_mappings(config)[bucket_name] = str(source_dir)
+    return bucket_name
+
+
+def _add_directory_mapping(
+    config: dict[str, object], *, local_path: str, destination: str
+) -> str:
+    source_dir = Path(local_path)
+    if not source_dir.exists() or not source_dir.is_dir():
+        raise typer.BadParameter(
+            f"Local source directory must be an existing directory: {local_path!r}"
+        )
+    destination_prefix = s3_directory_destination(destination)
+    directories = [
+        entry
+        for entry in _s3_directories(config)
+        if entry.get("local_path") != local_path
+        or entry.get("destination") != destination_prefix
+    ]
+    directories.append(
+        {
+            "local_path": str(source_dir),
+            "destination": destination_prefix,
+        }
+    )
+    _s3_section(config)["directories"] = sorted(
+        directories, key=lambda entry: (entry["destination"], entry["local_path"])
+    )
+    return destination_prefix
+
+
+def _remove_directory_mapping(
+    config: dict[str, object], target: str
+) -> list[dict[str, str]]:
+    directories = _s3_directories(config)
+    normalized_target = target.strip()
+    destination_target = (
+        s3_directory_destination(normalized_target)
+        if is_s3_uri(normalized_target)
+        else None
+    )
+    kept: list[dict[str, str]] = []
+    removed: list[dict[str, str]] = []
+    for entry in directories:
+        local_path = entry.get("local_path", "")
+        destination = entry.get("destination", "")
+        if normalized_target == local_path or (
+            destination_target is not None and destination == destination_target
+        ):
+            removed.append(entry)
+            continue
+        kept.append(entry)
+    _s3_section(config)["directories"] = kept
+    return removed
+
+
+def _parse_mapping_arg(raw: str, *, template: str) -> tuple[str, str]:
+    source, separator, local_value = raw.partition("=")
+    if not separator:
+        raise typer.BadParameter(f"Mapping must use {template}. Received: {raw!r}")
+    source = source.strip()
+    local_value = local_value.strip()
+    if not source or not local_value:
+        raise typer.BadParameter(f"Mapping must use {template}. Received: {raw!r}")
+    return source, local_value
+
+
+def _require_s3_object(objects: dict[str, str], uri: str) -> str:
+    try:
+        return objects[uri]
+    except KeyError as exc:
+        raise typer.BadParameter(f"No mapped local file found for {uri}.") from exc
+
+
+def _validate_ssm_name(name: str) -> str:
+    normalized = name.strip()
+    if not normalized or " " in normalized:
+        raise typer.BadParameter(
+            "SSM parameter names must be non-empty and contain no spaces."
+        )
+    return normalized
+
+
+def _format_parameter(name: str, value: str) -> str:
+    return f"{name}\t{value}"
+
+
+@glue_local_app.command(
+    "setup",
+    epilog=_examples_epilog(
+        "gluekit local setup --profile local-dev --s3-root sources/s3",
+        "gluekit local setup --s3-bucket local-dev=tests/fixtures",
+        "gluekit local setup --ssm-param /app/runtime=local",
+    ),
+)
+def local_setup(
+    profile: Optional[str] = typer.Option(
+        None,
+        "--profile",
+        help=(
+            "Optional gluekit/AWS profile name to associate with this local "
+            "setup in checkout state and .gluekit/local.json; this command "
+            "does not authenticate to AWS."
+        ),
+    ),
+    s3_bucket: Optional[list[str]] = typer.Option(
+        None,
+        "--s3-bucket",
+        help=(
+            "Map a mocked S3 bucket to a local directory in "
+            f"{S3_BUCKET_ARG_TEMPLATE} form. Repeat as needed."
+        ),
+    ),
+    s3_root: Optional[Path] = typer.Option(
+        None,
+        "--s3-root",
+        help=(
+            "Map a local directory as the full mocked S3 namespace for this "
+            "setup/profile. Expected layout: {local s3 root}/{bucket}/{prefix/key}."
+        ),
+    ),
+    ssm_param: Optional[list[str]] = typer.Option(
+        None,
+        "--ssm-param",
+        help=(
+            "Map a mocked SSM parameter to a local value in "
+            f"{SSM_PARAM_ARG_TEMPLATE} form. Repeat as needed."
+        ),
+    ),
+) -> None:
+    """Create or update and check out the local-only development setup."""
+    setup_data: dict[str, object] = {}
+    if profile is not None:
+        setup_data["profile"] = profile
+
+    config = _load_config(LOCAL_RUN_CONFIG_FILE)
+    if profile is not None:
+        config["profile"] = profile
+    if s3_root is not None:
+        _set_s3_root_mapping(config, str(s3_root))
+    if s3_bucket is not None:
+        _s3_section(config)["buckets"] = {}
+        _s3_section(config)["directories"] = []
+        for entry in s3_bucket:
+            bucket, local_path = _parse_mapping_arg(
+                entry, template=S3_BUCKET_ARG_TEMPLATE
+            )
+            _add_s3_bucket_mapping(config, bucket=bucket, local_path=local_path)
+    if ssm_param is not None:
+        _ssm_section(config)["parameters"] = {}
+        for entry in ssm_param:
+            parameter_name, value = _parse_mapping_arg(
+                entry, template=SSM_PARAM_ARG_TEMPLATE
+            )
+            _ssm_parameters(config)[_validate_ssm_name(parameter_name)] = value
+    _save_config(config, LOCAL_RUN_CONFIG_FILE)
+    _save_local_checkout("local", setup_data)
+    typer.echo(f"Saved and checked out local setup using {LOCAL_RUN_CONFIG_FILE}.")
+
+
+@glue_local_app.command(
+    "status",
+    epilog=_examples_epilog("gluekit local status"),
+)
+def local_status() -> None:
+    """Show the active local development setup."""
+    active_name = _get_checked_out_local_setup_name()
+    setups = _get_local_checkouts()
+    if not setups:
+        typer.echo("No local setup saved.")
+        return
+    for name, setup in sorted(setups.items()):
+        marker = "*" if name == active_name else "-"
+        typer.echo(f"{marker} {name}")
+        typer.echo(dumps(setup, indent=4))
+
+
+@local_s3_app.command(
+    "cp",
+    epilog=_examples_epilog(
+        "gluekit local s3 cp tests/fixtures/input.json s3://my-bucket/input.json",
+        "gluekit local s3 cp s3://my-bucket/input.json s3://my-bucket/copy.json",
+        "gluekit local s3 cp s3://my-bucket/input.json tmp/input.json",
+    ),
+)
+def local_s3_cp(
+    source: str = typer.Argument(..., help="Local path or mocked S3 URI source."),
+    destination: str = typer.Argument(
+        ..., help="Local path or mocked S3 URI destination."
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to update. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Copy a local file or mocked S3 object in the fixture config; does not call AWS."""
+    config = _load_config(config_file)
+    objects = _s3_objects(config)
+
+    if is_s3_uri(source) and is_s3_uri(destination):
+        source_uri = normalize_s3_object_uri(source)
+        source_path = _require_s3_object(objects, source_uri)
+        destination_uri = s3_destination_uri(
+            destination, posixpath.basename(source_uri)
+        )
+        objects[destination_uri] = source_path
+        _save_config(config, config_file)
+        typer.echo(f"copy: {source_uri} to {destination_uri}")
+        return
+
+    if is_s3_uri(source):
+        source_uri = normalize_s3_object_uri(source)
+        source_path = _require_s3_object(objects, source_uri)
+        destination_path = copy_mapped_file(source_path, destination)
+        typer.echo(f"download: {source_uri} to {destination_path}")
+        return
+
+    if is_s3_uri(destination):
+        source_path = Path(source)
+        if not source_path.is_file():
+            raise typer.BadParameter(
+                f"Local source must be an existing file: {source!r}"
+            )
+        destination_uri = s3_destination_uri(destination, source_path.name)
+        objects[destination_uri] = str(source_path)
+        _save_config(config, config_file)
+        typer.echo(f"upload: {source_path} to {destination_uri}")
+        return
+
+    raise typer.BadParameter("At least one path must be an S3 URI.")
+
+
+@local_s3_app.command(
+    "mount",
+    epilog=_examples_epilog(
+        "gluekit local s3 mount tests/fixtures s3://local-dev/input/",
+        "gluekit local s3 mount data/mock s3://bucket-a",
+    ),
+)
+def local_s3_mount(
+    source_dir: str = typer.Argument(..., help="Local directory to map recursively."),
+    destination: str = typer.Argument(
+        ..., help="Mocked S3 bucket or prefix base for mapped files."
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to update. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Map a local directory recursively into mocked S3 keys; does not call AWS."""
+    if not is_s3_uri(destination):
+        raise typer.BadParameter("Destination must be an S3 URI.")
+    config = _load_config(config_file)
+    destination_prefix = _add_directory_mapping(
+        config,
+        local_path=source_dir,
+        destination=destination,
+    )
+    _save_config(config, config_file)
+    typer.echo(f"mount: {source_dir} to {destination_prefix}")
+
+
+@local_s3_app.command(
+    "unmount",
+    epilog=_examples_epilog(
+        "gluekit local s3 unmount tests/fixtures",
+        "gluekit local s3 unmount s3://local-dev/input/",
+    ),
+)
+def local_s3_unmount(
+    target: str = typer.Argument(
+        ..., help="Local source directory or mocked S3 destination to remove."
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to update. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Remove a recursive local-directory-to-mocked-S3 mapping."""
+    config = _load_config(config_file)
+    removed = _remove_directory_mapping(config, target)
+    if not removed:
+        raise typer.BadParameter(f"No mounted directory found for {target!r}.")
+    _save_config(config, config_file)
+    for entry in removed:
+        typer.echo(f"unmount: {entry['local_path']} from {entry['destination']}")
+
+
+@local_s3_app.command(
+    "mv",
+    epilog=_examples_epilog(
+        "gluekit local s3 mv s3://my-bucket/input.json s3://my-bucket/renamed.json",
+        "gluekit local s3 mv s3://my-bucket/input.json tmp/input.json",
+    ),
+)
+def local_s3_mv(
+    source: str = typer.Argument(..., help="Local path or mocked S3 URI source."),
+    destination: str = typer.Argument(
+        ..., help="Local path or mocked S3 URI destination."
+    ),
+    delete_local: bool = typer.Option(
+        False,
+        "--delete-local",
+        help="Delete the local source file after mapping a LocalPath to S3.",
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to update. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Move a local file or mocked S3 object in the fixture config; does not call AWS."""
+    config = _load_config(config_file)
+    objects = _s3_objects(config)
+
+    if is_s3_uri(source) and is_s3_uri(destination):
+        source_uri = normalize_s3_object_uri(source)
+        source_path = _require_s3_object(objects, source_uri)
+        destination_uri = s3_destination_uri(
+            destination, posixpath.basename(source_uri)
+        )
+        del objects[source_uri]
+        objects[destination_uri] = source_path
+        _save_config(config, config_file)
+        typer.echo(f"move: {source_uri} to {destination_uri}")
+        return
+
+    if is_s3_uri(source):
+        source_uri = normalize_s3_object_uri(source)
+        source_path = _require_s3_object(objects, source_uri)
+        destination_path = copy_mapped_file(source_path, destination)
+        del objects[source_uri]
+        _save_config(config, config_file)
+        typer.echo(f"move: {source_uri} to {destination_path}")
+        return
+
+    if is_s3_uri(destination):
+        source_path = Path(source)
+        if not source_path.is_file():
+            raise typer.BadParameter(
+                f"Local source must be an existing file: {source!r}"
+            )
+        destination_uri = s3_destination_uri(destination, source_path.name)
+        objects[destination_uri] = str(source_path)
+        if delete_local:
+            source_path.unlink()
+        _save_config(config, config_file)
+        typer.echo(f"move: {source_path} to {destination_uri}")
+        return
+
+    raise typer.BadParameter("At least one path must be an S3 URI.")
+
+
+@local_s3_app.command(
+    "rm",
+    epilog=_examples_epilog(
+        "gluekit local s3 rm s3://my-bucket/input.json",
+        "gluekit local s3 rm s3://my-bucket/prefix/ --recursive",
+    ),
+)
+def local_s3_rm(
+    uri: str = typer.Argument(..., help="Mapped mocked S3 object or prefix to remove."),
+    recursive: bool = typer.Option(
+        False,
+        "--recursive",
+        "-r",
+        help="Remove all mapped objects under the S3 prefix.",
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to update. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Remove mapped mocked S3 objects from the local fixture config."""
+    config = _load_config(config_file)
+    objects = _s3_objects(config)
+    removed: list[str]
+    if recursive:
+        prefix = s3_prefix(uri)
+        removed = [
+            object_uri for object_uri in objects if object_uri.startswith(prefix)
+        ]
+    else:
+        object_uri = normalize_s3_object_uri(uri)
+        removed = [object_uri] if object_uri in objects else []
+
+    if not removed:
+        raise typer.BadParameter(f"No mapped S3 objects found for {uri}.")
+    for object_uri in removed:
+        del objects[object_uri]
+    _save_config(config, config_file)
+    for object_uri in removed:
+        typer.echo(f"delete: {object_uri}")
+
+
+@local_s3_app.command("ls")
+def local_s3_ls(
+    uri: Optional[str] = typer.Argument(None, help="Optional mocked S3 URI to list."),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to read. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """List mocked S3 buckets inferred from mappings, or mapped objects."""
+    config = _load_config(config_file)
+    objects = expand_s3_root_mapping(config)
+    objects.update(expand_bucket_mappings(config))
+    objects.update(_s3_objects(config))
+    objects.update(expand_directory_mappings(config))
+    if uri is None:
+        bucket_names = sorted({parse_s3_uri(object_uri)[0] for object_uri in objects})
+        for bucket_name in bucket_names:
+            typer.echo(bucket_name)
+        return
+
+    prefix = s3_prefix(uri)
+    for object_uri in sorted(
+        object_uri for object_uri in objects if object_uri.startswith(prefix)
+    ):
+        typer.echo(object_uri)
+
+
+@local_ssm_app.command(
+    "put",
+    epilog=_examples_epilog("gluekit local ssm put /app/env dev"),
+)
+def local_ssm_put(
+    name: str = typer.Argument(..., help="Mocked SSM parameter name."),
+    value: str = typer.Argument(..., help="Mocked SSM parameter value."),
+    overwrite: bool = typer.Option(
+        True,
+        "--overwrite/--no-overwrite",
+        help="Overwrite an existing parameter.",
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to update. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Create or update a mocked SSM parameter with local-only syntax."""
+    _put_ssm_parameter(
+        name,
+        value,
+        overwrite=overwrite,
+        config_file=config_file,
+    )
+
+
+def _put_ssm_parameter(
+    name: str,
+    value: str,
+    *,
+    overwrite: bool,
+    config_file: str | Path | None = None,
+) -> None:
+    parameter_name = _validate_ssm_name(name)
+    config = _load_config(config_file)
+    parameters = _ssm_parameters(config)
+    if parameter_name in parameters and not overwrite:
+        raise typer.BadParameter(
+            f"SSM parameter {parameter_name!r} already exists. Use --overwrite to replace it."
+        )
+    parameters[parameter_name] = value
+    _save_config(config, config_file)
+    typer.echo(f"put_parameter: {parameter_name}")
+
+
+@local_ssm_app.command("get")
+def local_ssm_get(
+    name: str = typer.Argument(..., help="Mocked SSM parameter name."),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to read. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Show one mocked SSM parameter with local-only syntax."""
+    _get_ssm_parameter(name, config_file=config_file)
+
+
+def _get_ssm_parameter(name: str, *, config_file: str | Path | None = None) -> None:
+    parameter_name = _validate_ssm_name(name)
+    parameter = _ssm_parameters(_load_config(config_file)).get(parameter_name)
+    if parameter is None:
+        raise typer.BadParameter(f"No SSM parameter found for {parameter_name!r}.")
+    typer.echo(_format_parameter(parameter_name, parameter))
+
+
+@local_ssm_app.command("rm")
+def local_ssm_rm(
+    name: str = typer.Argument(..., help="Mocked SSM parameter name."),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to update. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """Delete one mocked SSM parameter with local-only syntax."""
+    _delete_ssm_parameter(name, config_file=config_file)
+
+
+def _delete_ssm_parameter(name: str, *, config_file: str | Path | None = None) -> None:
+    parameter_name = _validate_ssm_name(name)
+    config = _load_config(config_file)
+    parameters = _ssm_parameters(config)
+    if parameter_name not in parameters:
+        raise typer.BadParameter(f"No SSM parameter found for {parameter_name!r}.")
+    del parameters[parameter_name]
+    _save_config(config, config_file)
+    typer.echo(f"delete_parameter: {parameter_name}")
+
+
+@local_ssm_app.command("ls")
+def local_ssm_ls(
+    path: Optional[str] = typer.Argument(
+        None, help="Optional mocked SSM parameter hierarchy path."
+    ),
+    recursive: bool = typer.Option(
+        False,
+        "--recursive/--no-recursive",
+        help="List all nested parameters below the path.",
+    ),
+    config_file: Optional[Path] = typer.Option(
+        None,
+        "--config-file",
+        help="Local fixture config file to read. Defaults to .gluekit/local.json.",
+    ),
+) -> None:
+    """List mocked SSM parameters with local-only syntax."""
+    _list_ssm_parameters(path, recursive=recursive, config_file=config_file)
+
+
+def _list_ssm_parameters(
+    path: str | None, *, recursive: bool, config_file: str | Path | None = None
+) -> None:
+    parameters = _ssm_parameters(_load_config(config_file))
+    if path is None:
+        for name, parameter in sorted(parameters.items()):
+            typer.echo(_format_parameter(name, parameter))
+        return
+
+    parameter_path = _validate_ssm_name(path).rstrip("/")
+    prefix = f"{parameter_path}/"
+    for name, parameter in sorted(parameters.items()):
+        if name == parameter_path:
+            typer.echo(_format_parameter(name, parameter))
+            continue
+        if not name.startswith(prefix):
+            continue
+        remainder = name[len(prefix) :]
+        if recursive or "/" not in remainder:
+            typer.echo(_format_parameter(name, parameter))
+
+
+@glue_local_app.callback(invoke_without_command=True)
+def local_callback(ctx: typer.Context) -> None:
+    """Manage local-only Glue development fixtures."""
+    if ctx.invoked_subcommand is None:
+        typer.echo(ctx.get_help())
+        raise typer.Exit()