gen3-dataops-toolkit 2.0.0__py3-none-any.whl

g3dt/services/delete/delete_metadata_by_guid.py

@@ -0,0 +1,338 @@
+import sys
+import time
+import logging
+import argparse
+import yaml
+from g3dt.upload.metadata_submitter import (
+    create_boto3_session,
+    get_gen3_api_key_aws_secret,
+    infer_api_endpoint_from_jwt,
+    create_gen3_submission_class,
+)
+from g3dt.upload.metadata_deleter import (
+    query_metadata_upload_guids,
+    delete_records_by_guid,
+)
+
+# ANSI colour codes (matching metadata_submitter.py style)
+GREEN = "\033[92m"
+RED = "\033[91m"
+YELLOW = "\033[93m"
+BLUE = "\033[94m"
+RESET = "\033[0m"
+
+EXCLUDE_NODES = [
+    "program",
+    "project",
+    "acknowledgement",
+    "publication",
+]
+
+# Exit code that signals "study exists but has no data at this version —
+# skipped". A bulk caller (services/delete/delete_metadata.sh) treats this as a
+# skip-and-continue rather than a failure. Only emitted with --skip-if-empty.
+SKIP_EXIT_CODE = 3
+
+
+def setup_logger():
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    return logger
+
+
+# Shared config resolution (SSM-backed) — see src/g3dt/config.py
+from g3dt import config as g3dt_config, resolver  # noqa: E402
+
+
+def load_import_order(import_order_path, exclude_nodes=None):
+    """
+    Reads the DataImportOrder.txt file and returns the node list
+    in deletion order (reversed, with excluded nodes removed).
+    """
+    if exclude_nodes is None:
+        exclude_nodes = EXCLUDE_NODES
+    with open(import_order_path, 'r', encoding='utf-8') as f:
+        nodes = [line.strip() for line in f if line.strip()]
+    nodes = [n for n in nodes if n not in exclude_nodes]
+    nodes.reverse()
+    return nodes
+
+
+def main():
+    logger = setup_logger()
+
+    parser = argparse.ArgumentParser(
+        description=(
+            "Delete Gen3 metadata records by GUID. Queries the Athena "
+            "metadata_upload table for matching records per node "
+            "(in reverse DataImportOrder) and deletes them from Gen3."
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--study",
+        required=True,
+        help=(
+            "Study key (bare or env-suffixed) "
+            "(e.g. ausdiab, caughtcad, edcad, cdah)"
+        ),
+    )
+    parser.add_argument(
+        "--env",
+        required=True,
+        help="Environment to use (selects AWS secret, profile, etc.)",
+    )
+    parser.add_argument(
+        "--version",
+        required=True,
+        help="Metadata version to filter on (e.g. 0.8.1)",
+    )
+    parser.add_argument(
+        "--import-order",
+        default="DataImportOrder.txt",
+        help="Path to DataImportOrder.txt",
+    )
+    parser.add_argument(
+        "--node",
+        default=None,
+        help=(
+            "Delete only a specific node (e.g. 'subject'). "
+            "If omitted, all nodes are processed in reverse "
+            "DataImportOrder."
+        ),
+    )
+    parser.add_argument(
+        "--prompt",
+        action="store_true",
+        default=False,
+        help="Prompt for confirmation before deleting.",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=40,
+        help="Number of UUIDs per DELETE request.",
+    )
+    parser.add_argument(
+        "--batch-delay",
+        type=float,
+        default=0.5,
+        help="Seconds to pause between batches.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Log full API response JSON for each request.",
+    )
+    parser.add_argument(
+        "--delay",
+        type=float,
+        default=1.0,
+        help="Seconds to wait between nodes.",
+    )
+    parser.add_argument(
+        "--skip-if-empty",
+        action="store_true",
+        default=False,
+        help=(
+            "If the study has no data at the given version, exit with the skip "
+            "code (3) instead of 0. Used by the bulk delete loop to "
+            "skip-and-continue rather than treat it as a failure."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+
+    # Env facts + resource names from SSM; the study registry from the marker
+    # or s3://<metadata-bucket>/config/studies.yaml.
+    try:
+        env_cfg = g3dt_config.resolve_env(args.env)
+    except g3dt_config.ConfigError as exc:
+        logger.error(str(exc))
+        sys.exit(1)
+    try:
+        study_cfg = g3dt_config.resolve_study(args.study, args.env)
+    except g3dt_config.ConfigError as exc:
+        if args.skip_if_empty:
+            logger.warning(
+                "Study '%s' not found in configuration — skipping.", args.study
+            )
+            sys.exit(SKIP_EXIT_CODE)
+        logger.error(str(exc))
+        sys.exit(1)
+
+    project_id = study_cfg.project_id
+    program_id = study_cfg.program_id
+
+    aws_secret_name = env_cfg.aws_secret_name
+    aws_profile = env_cfg.aws_profile
+    aws_region = env_cfg.region
+
+    rc = resolver.resolve(
+        g3dt_config.require_project(),
+        g3dt_config.env_base(args.env),
+        profile=aws_profile,
+    )
+    # Upload-tracking table: conventional name in the env's metadata DB
+    # (exactly like the CDK's `releases` table).
+    database = rc.metadata_db
+    table = g3dt_config.METADATA_UPLOAD_TABLE
+    athena_s3_output = rc.athena_output_location
+    workgroup = rc.athena_workgroup
+
+    # Construct compound project_id for Athena query
+    compound_project_id = f"{program_id}-{project_id}"
+
+    logger.info(
+        "Study: %s | Env: %s | Project: %s | Version: %s",
+        args.study,
+        args.env,
+        compound_project_id,
+        args.version,
+    )
+
+    # AWS and Gen3 authentication
+    session = create_boto3_session(aws_profile=aws_profile)
+    api_key = get_gen3_api_key_aws_secret(
+        secret_name=aws_secret_name,
+        region_name=aws_region,
+        session=session,
+    )
+
+    # Derive api_endpoint from JWT
+    api_endpoint = infer_api_endpoint_from_jwt(api_key['api_key'])
+    logger.info("Derived API endpoint: %s", api_endpoint)
+
+    # Create Gen3Submission instance
+    sub = create_gen3_submission_class(api_key)
+
+    # Determine node list
+    if args.node:
+        nodes_to_delete = [args.node]
+        logger.info(
+            "%s[SINGLE NODE]%s Targeting node: %s",
+            BLUE, RESET, args.node,
+        )
+    else:
+        nodes_to_delete = load_import_order(args.import_order)
+        logger.info(
+            "Loaded %s nodes from %s (deletion order, "
+            "excluding %s)",
+            len(nodes_to_delete),
+            args.import_order,
+            EXCLUDE_NODES,
+        )
+
+    if args.prompt:
+        confirm = input(
+            f"Proceed with deletion for project "
+            f"{compound_project_id}, version {args.version}, "
+            f"{len(nodes_to_delete)} node(s)? (yes/no): "
+        ).strip().lower()
+        if confirm != "yes":
+            logger.info("Deletion cancelled by user.")
+            return
+
+    # Process each node
+    total_deleted = 0
+    total_skipped = 0
+    total_nodes = len(nodes_to_delete)
+
+    for idx, node in enumerate(nodes_to_delete, start=1):
+        logger.info(
+            "%s[Node %d/%d]%s | Project: %-10s | "
+            "Node: %-25s | Querying...",
+            BLUE, idx, total_nodes, RESET,
+            compound_project_id, node,
+        )
+
+        df = query_metadata_upload_guids(
+            database=database,
+            table=table,
+            project_id=compound_project_id,
+            api_endpoint=api_endpoint,
+            version=args.version,
+            athena_s3_output=athena_s3_output,
+            workgroup=workgroup,
+            aws_region=aws_region,
+            aws_profile=aws_profile,
+            node=node,
+        )
+
+        if df.empty:
+            logger.info(
+                "%s[SKIP]%s    | Project: %-10s | "
+                "Node: %-25s | No records found",
+                YELLOW, RESET,
+                compound_project_id, node,
+            )
+            total_skipped += 1
+            continue
+
+        uuids = df['gen3_guid'].dropna().unique().tolist()
+        logger.info(
+            "%s[DELETE]%s  | Project: %-10s | "
+            "Node: %-25s | Records: %s",
+            BLUE, RESET,
+            compound_project_id, node, len(uuids),
+        )
+
+        delete_records_by_guid(
+            gen3_submission=sub,
+            program_id=program_id,
+            project_id=project_id,
+            uuids=uuids,
+            batch_size=args.batch_size,
+            batch_delay=args.batch_delay,
+            verbose=args.verbose,
+        )
+
+        logger.info(
+            "%s[SUCCESS]%s | Project: %-10s | "
+            "Node: %-25s | Deleted: %s",
+            GREEN, RESET,
+            compound_project_id, node, len(uuids),
+        )
+        total_deleted += len(uuids)
+
+        if idx < total_nodes:
+            time.sleep(args.delay)
+
+    logger.info(
+        "=========================================="
+    )
+    logger.info(
+        "Deletion complete. Total deleted: %s | "
+        "Nodes skipped: %s | Nodes processed: %s",
+        total_deleted,
+        total_skipped,
+        total_nodes - total_skipped,
+    )
+
+    # No records matched the requested version across any node. This usually
+    # means the version was never uploaded (or the data was uploaded without a
+    # version), so surface an actionable hint rather than a silent "0 deleted".
+    if total_deleted == 0:
+        logger.warning(
+            "Data version '%s' not found for study '%s'. Ensure each data node "
+            "has a `data_version` property for versioning to work.",
+            args.version,
+            args.study,
+        )
+        if args.skip_if_empty:
+            sys.exit(SKIP_EXIT_CODE)
+
+
+if __name__ == "__main__":
+    main()

g3dt/services/dictionary/deploy_dd.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+set -o pipefail
+
+# 1. Pull the dictionary at a specific version.
+# 2. Upload the dictionary to S3.
+# 3. Restart microservices (schema).
+
+# Usage: bash deploy_dd.sh <profile>
+# <profile> is display-only; all configuration comes from G3DT_* environment
+# variables exported by the g3dt CLI (g3dt.config.script_env).
+PROFILE=$1
+
+if [ -z "$PROFILE" ]; then
+    echo "Usage: $0 <profile> (test|staging|prod) — run via the g3dt CLI"
+    exit 1
+fi
+
+# Defining script paths (sibling scripts ship together inside the package)
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+SERVICE_DIR="${SCRIPT_DIR}/.."
+
+# Configuration from G3DT_* env vars (fail loudly if a required one is missing)
+VERSION="${G3DT_DICTIONARY_VERSION:?G3DT_DICTIONARY_VERSION not set — run via the g3dt CLI}"
+SCHEMA_S3_URI="${G3DT_SCHEMA_S3_URI:?G3DT_SCHEMA_S3_URI not set — run via the g3dt CLI}"
+DOMAIN="${G3DT_DOMAIN:?G3DT_DOMAIN not set — run via the g3dt CLI}"
+APP_NAME="${G3DT_APP_NAME:?G3DT_APP_NAME not set — run via the g3dt CLI}"
+NAMESPACE="${G3DT_NAMESPACE:?G3DT_NAMESPACE not set — run via the g3dt CLI}"
+CLUSTER_NAME="${G3DT_CLUSTER_NAME:?G3DT_CLUSTER_NAME not set — run via the g3dt CLI}"
+SCHEMA_REPO="${G3DT_SCHEMA_REPO:?G3DT_SCHEMA_REPO not set — run via the g3dt CLI}"
+REGION="${G3DT_REGION:-ap-southeast-2}"
+EKS_ARN="${G3DT_EKS_ARN:-}"
+# Downloaded schemas live outside the installed package.
+SCHEMA_DIR="${G3DT_SCHEMA_DIR:-$HOME/.g3dt/schemas}"
+ARGO_SCRIPT_DIR="${SERVICE_DIR}/k8s_ops"
+
+# Never export an empty AWS_PROFILE (empty means ambient credentials).
+if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+    export AWS_PROFILE="${G3DT_AWS_PROFILE}"
+    echo "==== [0] Configuring AWS PROFILE as '${AWS_PROFILE}' for profile '${PROFILE}' ===="
+else
+    echo "==== [0] No AWS profile set for '${PROFILE}' — using ambient AWS credentials ===="
+fi
+
+echo "Updating kubeconfig for cluster: ${CLUSTER_NAME}"
+# eks_arn is optional; only pass --role-arn when it is set.
+ROLE_ARG=""
+if [ -n "${EKS_ARN}" ] && [ "${EKS_ARN}" != "null" ]; then
+    ROLE_ARG="--role-arn ${EKS_ARN}"
+fi
+if ! aws eks update-kubeconfig --name "${CLUSTER_NAME}" --region "${REGION}" ${ROLE_ARG}; then
+    echo "Error: Failed to update kubeconfig for cluster ${CLUSTER_NAME}"
+    exit 1
+fi
+
+echo "==== [1] Pulling dictionary for version ${VERSION} ===="
+bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "https://raw.githubusercontent.com/${SCHEMA_REPO}/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
+
+echo "==== [2] Uploading dictionary to S3: s3://${SCHEMA_S3_URI} ===="
+python3 "${SERVICE_DIR}/dictionary/upload_dictionary.py" "${SCHEMA_DIR}/acdc_schema_${VERSION}.json" "s3://${SCHEMA_S3_URI}"
+
+echo "==== [3] Restarting microservices (schema) ===="
+bash "${ARGO_SCRIPT_DIR}/argocd_restart_schema.sh" -d "${DOMAIN}" -a "${APP_NAME}" -n "${NAMESPACE}"

g3dt/services/dictionary/pull_dict.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+show_help() {
+    echo "Usage: $0 <dict_url> [output_file]"
+    echo
+    echo "Download a dictionary JSON file from the specified URL."
+    echo
+    echo "Arguments:"
+    echo "  dict_url       The URL to download the dictionary JSON from."
+    echo "  output_file    Optional. The file to save the dictionary as (default: acdc_schema.json)."
+}
+
+if [[ "$1" == "-h" || "$1" == "--help" || "$#" -lt 1 ]]; then
+    show_help
+    exit 1
+fi
+
+DICT_URL="$1"
+
+# Downloads are written outside the installed package (never into site-packages).
+SCHEMA_DIR="${G3DT_SCHEMA_DIR:-$HOME/.g3dt/schemas}"
+mkdir -p "$SCHEMA_DIR"
+
+# Extract version tag after '/tags/' or '/tag/' or '/refs/tags/' in the URL
+if [[ "$DICT_URL" =~ /tags/([^/]+)/ ]]; then
+    VERSION="${BASH_REMATCH[1]}"
+elif [[ "$DICT_URL" =~ /refs/tags/([^/]+)/ ]]; then
+    VERSION="${BASH_REMATCH[1]}"
+else
+    VERSION="unknown"
+fi
+
+# Set output filename, including version tag if not overridden by user
+if [[ -n "$2" ]]; then
+    OUTPUT_BASENAME="$2"
+else
+    # Get the filename from the URL
+    BASE_NAME=$(basename "$DICT_URL")
+    # Insert version before file extension if possible
+    if [[ "$BASE_NAME" == *.* ]]; then
+        EXTENSION="${BASE_NAME##*.}"
+        NAME_NO_EXT="${BASE_NAME%.*}"
+        OUTPUT_BASENAME="${NAME_NO_EXT}_${VERSION}.${EXTENSION}"
+    else
+        OUTPUT_BASENAME="${BASE_NAME}_${VERSION}"
+    fi
+fi
+
+OUTPUT_FILE="${SCHEMA_DIR}/${OUTPUT_BASENAME}"
+
+echo "Downloading dictionary from $DICT_URL..."
+wget -O "$OUTPUT_FILE" "$DICT_URL"
+
+if [ $? -eq 0 ]; then
+    echo "Download complete: $OUTPUT_FILE"
+else
+    echo "Download failed." >&2
+    exit 1
+fi

g3dt/services/dictionary/upload_dictionary.py

@@ -0,0 +1,109 @@
+import boto3
+import json
+import logging
+import sys
+from botocore.exceptions import ClientError
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s %(message)s",
+    handlers=[logging.StreamHandler()]
+)
+logger = logging.getLogger(__name__)
+
+
+def get_s3_client(profile_name=None):
+    """
+    Creates an S3 client with optional AWS profile.
+
+    Args:
+        profile_name (str, optional): AWS profile name.
+
+    Returns:
+        boto3.client: S3 client.
+    """
+    if profile_name:
+        session = boto3.Session(profile_name=profile_name)
+        return session.client("s3")
+    return boto3.client("s3")
+
+
+def get_dict_version(dict_file_path):
+    """
+    Extracts the dictionary version from the provided JSON/YAML combo settings.
+
+    Args:
+        dict_file_path (str): Path to the dictionary file.
+
+    Returns:
+        str: Dictionary version.
+    """
+    with open(dict_file_path, "r", encoding="utf-8") as f:
+        dict_data = json.load(f)
+    return dict_data.get("_settings.yaml", {}).get("_dict_version", None)
+
+
+def upload_dict_to_s3(dict_file_path, s3_target_uri, dict_version, profile_name=None):
+    """
+    Uploads a dictionary file to S3 with metadata.
+
+    Args:
+        dict_file_path (str): Local dictionary path.
+        s3_target_uri (str): URI like s3://bucket/key.
+        dict_version (str): Dictionary version string.
+        profile_name (str, optional): AWS profile name.
+
+    Returns:
+        bool: True on success, False otherwise.
+    """
+    if not s3_target_uri.startswith("s3://"):
+        logger.error(f"Invalid S3 URI: {s3_target_uri}")
+        return False
+
+    try:
+        s3_path = s3_target_uri[len("s3://") :]
+        if "/" not in s3_path:
+            logger.error(f"S3 URI missing key: {s3_target_uri}")
+            return False
+        bucket, key = s3_path.split("/", 1)
+        # Set S3 metadata key to "version" instead of "dict_version"
+        extra_args = {"Metadata": {"version": dict_version or "unknown"}}
+        s3_client = get_s3_client(profile_name)
+        s3_client.upload_file(dict_file_path, bucket, key, ExtraArgs=extra_args)
+        logger.info(
+            f"Successfully uploaded '{dict_file_path}' (version: {dict_version}) to {s3_target_uri}"
+        )
+        return True
+    except ClientError as e:
+        logger.error(f"Failed to upload file to {s3_target_uri}: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error uploading file: {e}")
+        raise
+
+
+def main():
+    if len(sys.argv) < 3 or len(sys.argv) > 4:
+        print(
+            "Usage: python upload_dictionary.py <local_file_path> <s3_uri> [aws_profile]",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    local_file_path = sys.argv[1]
+    s3_uri = sys.argv[2]
+    profile_name = sys.argv[3] if len(sys.argv) == 4 else None
+
+    dict_version = get_dict_version(local_file_path)
+    if dict_version is None:
+        logger.error(f"Could not determine dictionary version from {local_file_path}")
+        sys.exit(1)
+
+    success = upload_dict_to_s3(local_file_path, s3_uri, dict_version, profile_name)
+    if not success:
+        sys.exit(1)
+    # You may add a completion message if desired
+
+
+if __name__ == "__main__":
+    main()