gen3-dataops-toolkit 2.0.0__py3-none-any.whl

g3dt/resolver.py

@@ -0,0 +1,163 @@
+"""Resolve a project's deployed resource names from AWS SSM Parameter Store.
+
+The CDK app (gen3-aws-data-pipeline) writes one SSM parameter per resource it
+creates, under the tree ``/{project}/{env}/...``, and mirrors the human-authored
+Gen3 app facts under ``/{project}/{env}/app/*``. This module reads that tree
+once per process and exposes it as a typed :class:`ResolvedConfig`, so nothing
+else in the toolkit ever hard-codes an AWS resource name.
+
+The tree's exact shape is enforced on the infrastructure side by the CDK repo's
+drift-guard test (``test/ssm-publishing.test.ts``): 38 parameters from the SSM
+stack plus ``ec2/instanceId`` published by the EC2 stack (39 total).
+"""
+from __future__ import annotations
+
+import functools
+from dataclasses import dataclass, field
+from typing import Mapping, Optional
+
+import boto3
+
+from g3dt.config import ConfigError
+
+
+def _fetch_params(project: str, env: str, *, session: boto3.Session) -> dict:
+    """Return ``{relative_name: value}`` for every parameter under /{project}/{env}.
+
+    ``relative_name`` is the path with the ``/{project}/{env}/`` prefix removed,
+    e.g. ``/etl/test/buckets/metadata`` -> ``buckets/metadata``.
+    """
+    client = session.client("ssm")
+    base = f"/{project}/{env}"
+    # get_parameters_by_path returns at most 10 params per call, so paginate.
+    paginator = client.get_paginator("get_parameters_by_path")
+    out: dict = {}
+    for page in paginator.paginate(Path=base, Recursive=True, WithDecryption=True):
+        for p in page["Parameters"]:
+            out[p["Name"][len(base) + 1 :]] = p["Value"]
+    if not out:
+        raise ConfigError(
+            f"No SSM parameters found under {base} — has CDK been deployed for "
+            f"this env? Run `cdk deploy --all -c project={project} -c env={env}` "
+            f"in gen3-aws-data-pipeline, then verify with:\n"
+            f"    aws ssm get-parameters-by-path --path {base} --recursive"
+        )
+    return out
+
+
+@dataclass(frozen=True)
+class ResolvedConfig:
+    """All resource names CDK published for one project/env.
+
+    ``params`` is the raw ``{relative_name: value}`` map (used by ``config show``
+    to print the whole subtree). The properties are typed accessors for the
+    names the toolkit reads most often; each raises a friendly
+    :class:`ConfigError` if its parameter is missing, which surfaces an
+    incomplete CDK deploy immediately instead of a cryptic AWS error later.
+    """
+
+    project: str
+    env: str
+    params: Mapping[str, str] = field(repr=False)
+
+    def _req(self, key: str) -> str:
+        try:
+            return self.params[key]
+        except KeyError:
+            raise ConfigError(
+                f"SSM parameter /{self.project}/{self.env}/{key} is missing. "
+                f"Found: {', '.join(sorted(self.params)) or '(none)'}"
+            )
+
+    # --- OUTPUT names (CDK-created) ---
+    @property
+    def metadata_bucket(self) -> str:
+        return self._req("buckets/metadata")
+
+    @property
+    def metadata_db(self) -> str:
+        return self._req("glue/db/metadata")
+
+    @property
+    def release_db(self) -> str:
+        return self._req("release/db")
+
+    @property
+    def release_table(self) -> str:
+        return self._req("release/table")
+
+    @property
+    def athena_workgroup(self) -> str:
+        return self._req("athena/workgroup")
+
+    @property
+    def athena_output_location(self) -> str:
+        return self._req("athena/outputLocation")
+
+    @property
+    def ec2_instance_id(self) -> str:
+        return self._req("ec2/instanceId")
+
+    @property
+    def ec2_log_group(self) -> str:
+        return self._req("ec2/logGroup")
+
+    @property
+    def ec2_log_bucket(self) -> str:
+        return self._req("ec2/logBucket")
+
+    @property
+    def ec2_log_prefix(self) -> str:
+        return self._req("ec2/logPrefix")
+
+    @property
+    def region(self) -> str:
+        return self._req("meta/region")
+
+    @property
+    def toolkit_version(self) -> str:
+        return self._req("meta/toolkitVersion")
+
+    def app(self, key: str) -> str:
+        """A mirrored Gen3 app fact, e.g. ``app("domain")`` (snake_case keys)."""
+        return self._req(f"app/{key}")
+
+    def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
+        """A raw leaf by relative name, or ``default`` if absent."""
+        return self.params.get(key, default)
+
+
+@functools.lru_cache(maxsize=None)
+def resolve(project: str, env: str, profile: Optional[str] = None) -> ResolvedConfig:
+    """Resolve all deployed names for ``project``/``env`` from SSM (cached).
+
+    Cached on ``(project, env, profile)`` for the life of the process, so a
+    single CLI invocation makes exactly one ``get_parameters_by_path``
+    round-trip no matter how many commands ask for a name. Call
+    ``resolve.cache_clear()`` in tests.
+
+    ``profile`` is the local AWS named profile to authenticate the read (e.g.
+    ``etl_test``); ``None`` uses the default credential chain — which is the
+    instance profile on the EC2 job box and the build role in CodeBuild.
+    """
+    session = boto3.Session(profile_name=profile) if profile else boto3.Session()
+    return ResolvedConfig(
+        project=project, env=env, params=_fetch_params(project, env, session=session)
+    )
+
+
+def list_envs(project: str, profile: Optional[str] = None) -> list:
+    """Return the environment names that have an SSM tree under /{project}/.
+
+    Reads the parameter names one level down (e.g. ``/etl/test/...`` -> ``test``).
+    """
+    session = boto3.Session(profile_name=profile) if profile else boto3.Session()
+    client = session.client("ssm")
+    paginator = client.get_paginator("get_parameters_by_path")
+    envs = set()
+    for page in paginator.paginate(Path=f"/{project}", Recursive=True):
+        for p in page["Parameters"]:
+            parts = p["Name"].split("/")  # ['', project, env, ...]
+            if len(parts) > 2:
+                envs.add(parts[2])
+    return sorted(envs)

g3dt/services/delete/delete_all_metadata_for_project.py

@@ -0,0 +1,170 @@
+import sys
+import logging
+import argparse
+import yaml
+from g3dt.upload.metadata_submitter import (
+    create_boto3_session,
+    get_gen3_api_key_aws_secret,
+    create_gen3_submission_class,
+)
+from g3dt.upload.metadata_deleter import (
+    delete_project_metadata,
+)
+
+# ANSI colour codes
+GREEN = "\033[92m"
+RED = "\033[91m"
+YELLOW = "\033[93m"
+BLUE = "\033[94m"
+RESET = "\033[0m"
+
+EXCLUDE_NODES = [
+    "program",
+    "project",
+    "acknowledgement",
+    "publication",
+]
+
+
+def setup_logger():
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    return logger
+
+
+# Shared config resolution (SSM-backed) — see src/g3dt/config.py
+from g3dt import config as g3dt_config  # noqa: E402
+
+
+def load_import_order(import_order_path, exclude_nodes=None):
+    """
+    Reads the DataImportOrder.txt file and returns the node list
+    in deletion order (reversed, with excluded nodes removed).
+    """
+    if exclude_nodes is None:
+        exclude_nodes = EXCLUDE_NODES
+    with open(import_order_path, 'r', encoding='utf-8') as f:
+        nodes = [line.strip() for line in f if line.strip()]
+    nodes = [n for n in nodes if n not in exclude_nodes]
+    nodes.reverse()
+    return nodes
+
+
+def main():
+    logger = setup_logger()
+
+    parser = argparse.ArgumentParser(
+        description=(
+            "Delete ALL metadata for a Gen3 project. Iterates "
+            "through nodes in reverse DataImportOrder and calls "
+            "Gen3's delete_nodes API for each."
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--study",
+        required=True,
+        help=(
+            "Study key (bare or env-suffixed) "
+            "(e.g. ausdiab, caughtcad, edcad, cdah)"
+        ),
+    )
+    parser.add_argument(
+        "--env",
+        required=True,
+        help="Environment to use (selects AWS secret, profile, etc.)",
+    )
+    parser.add_argument(
+        "--import-order",
+        default="DataImportOrder.txt",
+        help="Path to DataImportOrder.txt",
+    )
+    parser.add_argument(
+        "--node",
+        default=None,
+        help=(
+            "Delete only a specific node (e.g. 'subject'). "
+            "If omitted, all nodes are processed in reverse "
+            "DataImportOrder."
+        ),
+    )
+    parser.add_argument(
+        "--prompt",
+        action="store_true",
+        default=False,
+        help="Prompt for confirmation before deleting.",
+    )
+
+    args = parser.parse_args()
+
+    # Env facts from SSM; the study registry from the marker or
+    # s3://<metadata-bucket>/config/studies.yaml.
+    try:
+        env_cfg = g3dt_config.resolve_env(args.env)
+        study_cfg = g3dt_config.resolve_study(args.study, args.env)
+    except g3dt_config.ConfigError as exc:
+        logger.error(str(exc))
+        sys.exit(1)
+
+    project_id = study_cfg.project_id
+    program_id = study_cfg.program_id
+
+    aws_secret_name = env_cfg.aws_secret_name
+    aws_profile = env_cfg.aws_profile
+    aws_region = env_cfg.region
+
+    logger.info(
+        "Study: %s | Env: %s | Program: %s | Project: %s",
+        args.study, args.env, program_id, project_id,
+    )
+
+    # AWS and Gen3 authentication
+    session = create_boto3_session(aws_profile=aws_profile)
+    api_key = get_gen3_api_key_aws_secret(
+        secret_name=aws_secret_name,
+        region_name=aws_region,
+        session=session,
+    )
+    sub = create_gen3_submission_class(api_key)
+
+    # Determine node list
+    if args.node:
+        nodes_to_delete = [args.node]
+        logger.info(
+            "%s[SINGLE NODE]%s Targeting node: %s",
+            BLUE, RESET, args.node,
+        )
+    else:
+        nodes_to_delete = load_import_order(args.import_order)
+        logger.info(
+            "Loaded %s nodes from %s (deletion order, "
+            "excluding %s)",
+            len(nodes_to_delete),
+            args.import_order,
+            EXCLUDE_NODES,
+        )
+
+    # Delete
+    delete_project_metadata(
+        gen3_submission=sub,
+        program_id=program_id,
+        project_id=project_id,
+        nodes=nodes_to_delete,
+        prompt_for_confirmation=args.prompt,
+    )
+
+    logger.info(
+        "=========================================="
+    )
+    logger.info("Deletion complete.")
+
+
+if __name__ == "__main__":
+    main()

g3dt/services/delete/delete_metadata.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Exit code the per-study worker uses to signal "no data at this version —
+# skipped" (see delete_metadata_by_guid.py SKIP_EXIT_CODE, shipped alongside
+# this script).
+SKIP_EXIT_CODE=3
+
+usage() {
+    cat <<EOF
+Usage: $(basename "$0") --studies <comma-separated-studies> --env <environment> --version <version|all> [--node <node>]
+
+Delete metadata for each study sequentially, in a single job.
+
+Arguments:
+  --studies   Comma-separated list of study config keys (e.g. ausdiab_staging,caughtcad_staging)
+  --env       Environment string passed to the Python worker (e.g. staging_ec2)
+  --version   Metadata version to delete (e.g. 0.9.8), or 'all' for every version
+  --node      (optional) Restrict deletion to a single node type
+
+Behaviour:
+  * --version all          -> delete_all_metadata_for_project.py (deletes whole nodes)
+  * --version <x.y.z>      -> delete_metadata_by_guid.py (Athena GUID lookup for that version)
+
+  A study that exists but has no data at the requested version is skipped and the
+  loop continues. Only genuine errors (Gen3/AWS failures) count as failures.
+
+Run via the g3dt CLI:
+  g3dt delete metadata \\
+    --studies ausdiab_staging,caughtcad_staging \\
+    --env staging_ec2 \\
+    --version 0.9.8
+
+Failure logs are written under ~/.g3dt/logs/.
+EOF
+    exit 1
+}
+
+# ---------- Parse arguments ----------
+STUDIES=""
+ENV=""
+VERSION=""
+NODE=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --studies)
+            STUDIES="$2"
+            shift 2
+            ;;
+        --env)
+            ENV="$2"
+            shift 2
+            ;;
+        --version)
+            VERSION="$2"
+            shift 2
+            ;;
+        --node)
+            NODE="$2"
+            shift 2
+            ;;
+        *)
+            echo "ERROR: Unknown argument: $1"
+            usage
+            ;;
+    esac
+done
+
+if [[ -z "$STUDIES" || -z "$ENV" || -z "$VERSION" ]]; then
+    echo "ERROR: --studies, --env and --version are required."
+    usage
+fi
+
+# Lower-case the version so 'ALL'/'All' are treated as 'all'.
+VERSION_LC="$(echo "$VERSION" | tr '[:upper:]' '[:lower:]')"
+
+# ---------- Setup ----------
+# Logs go outside the installed package.
+LOG_DIR="$HOME/.g3dt/logs"
+TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
+FAILED_LOG="${LOG_DIR}/${TIMESTAMP}_delete_failed.log"
+mkdir -p "${LOG_DIR}"
+
+IFS=',' read -ra STUDY_LIST <<< "$STUDIES"
+DELETED_COUNT=0
+SKIPPED_COUNT=0
+FAIL_COUNT=0
+
+echo "============================================"
+echo "Metadata delete started at $(date)"
+echo "Environment : ${ENV}"
+echo "Studies     : ${STUDIES}"
+echo "Version     : ${VERSION}"
+[[ -n "$NODE" ]] && echo "Node        : ${NODE}"
+echo "Failure log : ${FAILED_LOG}"
+echo "============================================"
+echo ""
+
+# ---------- Sequential execution ----------
+for study in "${STUDY_LIST[@]}"; do
+    echo "--------------------------------------------"
+    echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Starting deletion for study: ${study} (version: ${VERSION})"
+    echo "--------------------------------------------"
+
+    if [[ "$VERSION_LC" == "all" ]]; then
+        CMD=(python3 "${SCRIPT_DIR}/delete_all_metadata_for_project.py"
+             --study "$study" --env "$ENV")
+        [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
+    else
+        CMD=(python3 "${SCRIPT_DIR}/delete_metadata_by_guid.py"
+             --study "$study" --env "$ENV" --version "$VERSION" --skip-if-empty)
+        [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
+    fi
+
+    # Run the worker without aborting the loop on a non-zero exit.
+    set +e
+    "${CMD[@]}"
+    EXIT_CODE=$?
+    set -e
+
+    if [[ $EXIT_CODE -eq 0 ]]; then
+        echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Completed: ${study}"
+        DELETED_COUNT=$((DELETED_COUNT + 1))
+    elif [[ $EXIT_CODE -eq $SKIP_EXIT_CODE ]]; then
+        echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Skipped (no data at version ${VERSION}): ${study}"
+        SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    else
+        FAIL_COUNT=$((FAIL_COUNT + 1))
+        echo "[$(date +%Y-%m-%d\ %H:%M:%S)] FAILED: ${study} (exit code ${EXIT_CODE})"
+        echo "[$(date +%Y-%m-%d\ %H:%M:%S)] ${study} exit_code=${EXIT_CODE}" >> "$FAILED_LOG"
+    fi
+
+    echo ""
+done
+
+# ---------- Summary ----------
+echo "============================================"
+echo "Metadata delete finished at $(date)"
+echo "Total studies : ${#STUDY_LIST[@]}"
+echo "Deleted       : ${DELETED_COUNT}"
+echo "Skipped       : ${SKIPPED_COUNT}"
+echo "Failures      : ${FAIL_COUNT}"
+
+if [[ $FAIL_COUNT -gt 0 ]]; then
+    echo "See failure details: ${FAILED_LOG}"
+    exit 1
+fi
+
+echo "All studies processed (deleted ${DELETED_COUNT}, skipped ${SKIPPED_COUNT})."
+exit 0