gen3-dataops-toolkit 2.0.0__py3-none-any.whl

g3dt/services/upload/metadata/upload_all_studies.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+usage() {
+    cat <<EOF
+Usage: $(basename "$0") --studies <comma-separated-studies> --env <environment>
+
+Run upload_metadata.py sequentially for each study.
+
+Arguments:
+  --studies   Comma-separated list of study config keys (e.g. ausdiab_staging,caughtcad_staging)
+  --env       Environment string passed to the Python script (e.g. staging_ec2)
+
+Run via the g3dt CLI:
+  g3dt metadata upload-all \\
+    --studies ausdiab_staging,caughtcad_staging,edcad_staging \\
+    --env staging_ec2
+
+Failure logs are written under ~/.g3dt/logs/.
+EOF
+    exit 1
+}
+
+# ---------- Parse arguments ----------
+STUDIES=""
+ENV=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --studies)
+            STUDIES="$2"
+            shift 2
+            ;;
+        --env)
+            ENV="$2"
+            shift 2
+            ;;
+        *)
+            echo "ERROR: Unknown argument: $1"
+            usage
+            ;;
+    esac
+done
+
+if [[ -z "$STUDIES" || -z "$ENV" ]]; then
+    echo "ERROR: --studies and --env are required."
+    usage
+fi
+
+# ---------- Production safety check ----------
+if echo "$ENV $STUDIES" | grep -qi "prod"; then
+    echo "ERROR: Production environment detected in arguments."
+    echo "       This script is not intended for production use. Aborting."
+    exit 1
+fi
+
+# ---------- Setup ----------
+# Logs go outside the installed package.
+LOG_DIR="$HOME/.g3dt/logs"
+TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
+FAILED_LOG="${LOG_DIR}/${TIMESTAMP}_bulk_upload_failed.log"
+mkdir -p "${LOG_DIR}"
+
+IFS=',' read -ra STUDY_LIST <<< "$STUDIES"
+FAIL_COUNT=0
+
+echo "============================================"
+echo "Bulk upload started at $(date)"
+echo "Environment : ${ENV}"
+echo "Studies     : ${STUDIES}"
+echo "Failure log : ${FAILED_LOG}"
+echo "============================================"
+echo ""
+
+# ---------- Sequential execution ----------
+for study in "${STUDY_LIST[@]}"; do
+    echo "--------------------------------------------"
+    echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Starting upload for study: ${study}"
+    echo "--------------------------------------------"
+
+    if python3 "${SCRIPT_DIR}/upload_metadata.py" \
+        --study "$study" --env "$ENV"; then
+        echo "[$(date +%Y-%m-%d\ %H:%M:%S)] Completed successfully: ${study}"
+    else
+        EXIT_CODE=$?
+        FAIL_COUNT=$((FAIL_COUNT + 1))
+        echo "[$(date +%Y-%m-%d\ %H:%M:%S)] FAILED: ${study} (exit code ${EXIT_CODE})"
+        echo "[$(date +%Y-%m-%d\ %H:%M:%S)] ${study} exit_code=${EXIT_CODE}" >> "$FAILED_LOG"
+    fi
+
+    echo ""
+done
+
+# ---------- Summary ----------
+echo "============================================"
+echo "Bulk upload finished at $(date)"
+echo "Total studies : ${#STUDY_LIST[@]}"
+echo "Failures      : ${FAIL_COUNT}"
+
+if [[ $FAIL_COUNT -gt 0 ]]; then
+    echo "See failure details: ${FAILED_LOG}"
+    exit 1
+fi
+
+echo "All studies uploaded successfully."
+exit 0

g3dt/services/upload/metadata/upload_metadata.py

@@ -0,0 +1,152 @@
+import sys
+import logging
+import argparse
+import yaml
+import json
+from g3dt.upload.metadata_submitter import (
+    list_metadata_jsons_s3,
+    find_data_import_order_file_s3,
+    create_boto3_session,
+    get_gen3_api_key_aws_secret,
+    MetadataSubmitter,
+)
+
+
+def setup_logger(debug=False):
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG if debug else logging.INFO)
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    return logger
+
+
+# Shared config resolution (SSM-backed) — see src/g3dt/config.py
+from g3dt import config as g3dt_config, resolver  # noqa: E402
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Unified Metadata Upload Script")
+    parser.add_argument(
+        "--study", required=True, help="Study name (e.g., ausdiab, caughtcad, edcad)"
+    )
+    parser.add_argument(
+        "--env", required=True,
+        help="Environment to upload to (e.g., test, staging, prod, staging_ec2, prod_ec2)"
+    )
+    parser.add_argument("--specific-node", help="Submit only a specific node")
+    parser.add_argument(
+        "--debug", action="store_true",
+        help="Enable debug logging"
+    )
+
+    args = parser.parse_args()
+
+    logger = setup_logger(debug=args.debug)
+
+    # Env facts + resource names from SSM; the study registry from the marker
+    # or s3://<metadata-bucket>/config/studies.yaml.
+    try:
+        env_cfg = g3dt_config.resolve_env(args.env)
+        study_cfg = g3dt_config.resolve_study(args.study, args.env)
+    except g3dt_config.ConfigError as exc:
+        logger.error(str(exc))
+        sys.exit(1)
+
+    project_id = study_cfg.project_id
+    program_id = study_cfg.program_id
+    base_dir = study_cfg.s3_metadata_path
+
+    aws_secret_name = env_cfg.aws_secret_name
+    aws_profile = env_cfg.aws_profile
+    aws_region = env_cfg.region
+
+    rc = resolver.resolve(
+        g3dt_config.require_project(),
+        g3dt_config.env_base(args.env),
+        profile=aws_profile,
+    )
+    # Upload-tracking table: conventional name in the env's metadata DB/bucket
+    # (exactly like the CDK's `releases` table).
+    database = rc.metadata_db
+    table = g3dt_config.METADATA_UPLOAD_TABLE
+    table_location = f"s3://{rc.metadata_bucket}/{g3dt_config.METADATA_UPLOAD_PREFIX}"
+    athena_s3_output = rc.athena_output_location
+    workgroup = rc.athena_workgroup
+
+    logger.info(
+        f"Starting metadata submission for {args.study} "
+        f"in {args.env} environment."
+    )
+    logger.info(f"Project ID: {project_id}, Program ID: {program_id}")
+    logger.info(f"S3 Path: {base_dir}")
+
+    session = create_boto3_session(
+        aws_profile=aws_profile, aws_region=aws_region
+    )
+
+    logger.info("Listing metadata JSON files in S3 directory.")
+    metadata_files = list_metadata_jsons_s3(s3_uri=base_dir, session=session)
+    if not metadata_files:
+        raise FileNotFoundError(
+            f"No metadata *.json files found in {base_dir}"
+        )
+    logger.info(
+        f"Located {len(metadata_files)} metadata *.json files in {base_dir}"
+    )
+
+    logger.info("Finding DataImportOrder.txt in the S3 directory.")
+    import_order_file_path = find_data_import_order_file_s3(
+        s3_uri=base_dir, session=session
+    )
+    logger.info(f"Import order file found: {import_order_file_path}")
+
+    # Fetch Gen3 API key
+    if aws_secret_name.startswith('/'):
+        logger.info(f"Loading Gen3 API key from local file: {aws_secret_name}")
+        with open(aws_secret_name, 'r') as f:
+            api_key = json.load(f)
+    else:
+        logger.info(
+            f"Fetching Gen3 API key from AWS Secrets Manager: {aws_secret_name}"
+        )
+        api_key = get_gen3_api_key_aws_secret(
+            secret_name=aws_secret_name,
+            region_name=aws_region,
+            session=session,
+        )
+
+    submitter = MetadataSubmitter(
+        metadata_file_list=metadata_files,
+        api_key=api_key,
+        project_id=project_id,
+        data_import_order_path=import_order_file_path,
+        database=database,
+        table=table,
+        athena_s3_output=athena_s3_output,
+        workgroup=workgroup,
+        table_location=table_location,
+        program_id=program_id,
+        max_size_kb=100,  # Default to 100 as seen in most scripts
+        max_retries=3,
+        aws_profile=aws_profile,
+        aws_region=aws_region,
+    )
+
+    try:
+        logger.info("Submitting metadata to Gen3.")
+        submitter.submit_metadata(specific_node=args.specific_node)
+        logger.info(
+            f"Finished submitting metadata for {args.study} ({project_id})"
+        )
+    except Exception as e:
+        logger.error(f"An unexpected error occurred during submission: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

g3dt/upload/__init__.py

@@ -0,0 +1 @@
+from .upload_synthdata_s3 import *

g3dt/upload/metadata_deleter.py

@@ -0,0 +1,265 @@
+import os
+import sys
+import time
+import logging
+from gen3.submission import Gen3Submission
+from typing import Optional, List
+import pandas as pd
+from g3dt.utils.athena_utils import AthenaQuery, AthenaConfig
+
+logger = logging.getLogger(__name__)
+
+
+def query_metadata_upload_guids(
+    database: str,
+    table: str,
+    project_id: str,
+    api_endpoint: str,
+    version: str,
+    athena_s3_output: str,
+    workgroup: str = "primary",
+    aws_region: str = "ap-southeast-2",
+    aws_profile: Optional[str] = None,
+    node: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Queries the Athena metadata upload table for records matching a given
+    project_id, version, and api_endpoint. Optionally filters by node.
+
+    Args:
+        database (str): The Athena database name.
+        table (str): The Athena table name.
+        project_id (str): The compound project ID (e.g. "program1-CDAH").
+        api_endpoint (str): The Gen3 API endpoint URL.
+        version (str): The metadata version to filter on (e.g. "0.8.1").
+        athena_s3_output (str): S3 URI for Athena query results.
+        workgroup (str, optional): Athena workgroup. Default is "primary".
+        aws_region (str, optional): AWS region. Default is "ap-southeast-2".
+        aws_profile (str, optional): AWS profile name.
+        node (str, optional): Node name to filter on (e.g. "subject").
+
+    Returns:
+        pd.DataFrame: DataFrame of matching records including gen3_guid.
+    """
+    athena_config = AthenaConfig(
+        aws_region=aws_region,
+        aws_profile=aws_profile,
+        athena_s3_output=athena_s3_output,
+    )
+    athena_query = AthenaQuery(athena_config)
+
+    sql = (
+        f'SELECT * FROM "{database}"."{table}" '
+        f"WHERE project_id = '{project_id}' "
+        f"AND version = '{version}' "
+        f"AND api_endpoint = '{api_endpoint}'"
+    )
+    if node:
+        sql += f" AND node = '{node}'"
+
+    logger.info(
+        "Querying Athena for metadata upload records: "
+        "project_id=%s, version=%s, api_endpoint=%s, node=%s",
+        project_id,
+        version,
+        api_endpoint,
+        node or "ALL",
+    )
+    df = athena_query.query_athena(
+        sql=sql,
+        athena_database=database,
+        ctas_approach=False,
+    )
+    logger.info("Query returned %s records.", len(df))
+    return df
+
+
+def delete_records_by_guid(
+    gen3_submission: Gen3Submission,
+    program_id: str,
+    project_id: str,
+    uuids: List[str],
+    batch_size: int = 40,
+    batch_delay: float = 0.5,
+    verbose: bool = False,
+):
+    """
+    Deletes Gen3 records one at a time using the SDK's
+    delete_record method. UUIDs are grouped into batches
+    for rate-limiting only (a pause between each batch).
+
+    Errors are caught and logged per-UUID so that one
+    failure does not stop the rest of the deletions.
+
+    Args:
+        gen3_submission (Gen3Submission): An authenticated
+            Gen3Submission instance.
+        program_id (str): The Gen3 program name.
+        project_id (str): The Gen3 project name.
+        uuids (list[str]): List of gen3_guid UUIDs to delete.
+        batch_size (int, optional): Number of UUIDs to process
+            before pausing. Default is 40.
+        batch_delay (float, optional): Seconds to pause between
+            batches. Default is 0.5.
+        verbose (bool, optional): If True, log the full API
+            response JSON for each request. Default is False.
+    """
+    if not uuids:
+        logger.info("No UUIDs provided for deletion. Skipping.")
+        return
+
+    batches = [
+        uuids[i: i + batch_size]
+        for i in range(0, len(uuids), batch_size)
+    ]
+    total_batches = len(batches)
+    total = len(uuids)
+
+    logger.info(
+        "Deleting %s records in %s batches "
+        "(batch_size=%s)...",
+        total, total_batches, batch_size,
+    )
+
+    success_count = 0
+    failed_ids = []
+
+    for idx, batch in enumerate(batches, start=1):
+        batch_success = 0
+        for uuid in batch:
+            try:
+                if verbose:
+                    resp = gen3_submission.delete_record(
+                        program_id, project_id, uuid,
+                    )
+                    logger.debug(
+                        "%s | response: %s",
+                        uuid, resp,
+                    )
+                else:
+                    with open(os.devnull, "w") as devnull:
+                        old_stdout = sys.stdout
+                        sys.stdout = devnull
+                        try:
+                            gen3_submission.delete_record(
+                                program_id, project_id,
+                                uuid,
+                            )
+                        finally:
+                            sys.stdout = old_stdout
+                batch_success += 1
+            except Exception as e:
+                failed_ids.append(uuid)
+                try:
+                    body = e.response.json()
+                    code = body.get("code", "?")
+                    if verbose:
+                        msg = body
+                    else:
+                        ents = body.get("entities", [])
+                        errs = ents[0].get("errors", [])
+                        msg = errs[0].get(
+                            "message", str(e),
+                        )
+                except Exception:
+                    code = "?"
+                    msg = str(e)
+                logger.warning(
+                    "\033[91m[FAIL]\033[0m %s | %s | %s",
+                    uuid, code, msg,
+                )
+
+        success_count += batch_success
+        if batch_success == len(batch):
+            logger.info(
+                "\033[92m[Batch %d/%d]\033[0m "
+                "Deleted %d/%d",
+                idx, total_batches,
+                batch_success, len(batch),
+            )
+        else:
+            logger.info(
+                "[Batch %d/%d] Deleted %d/%d",
+                idx, total_batches,
+                batch_success, len(batch),
+            )
+
+        if idx < total_batches:
+            time.sleep(batch_delay)
+
+    if failed_ids:
+        logger.info(
+            "Deletion complete. "
+            "\033[92mSuccessful: %s\033[0m, "
+            "\033[91mFailed: %s\033[0m",
+            success_count, len(failed_ids),
+        )
+    else:
+        logger.info(
+            "\033[92mDeletion complete. "
+            "Successful: %s, Failed: 0\033[0m",
+            success_count,
+        )
+
+
+def delete_project_metadata(
+    gen3_submission: Gen3Submission,
+    program_id: str,
+    project_id: str,
+    nodes: List[str],
+    prompt_for_confirmation: bool = True,
+):
+    """
+    Deletes all metadata for a project by iterating through nodes
+    and calling Gen3's delete_nodes API.
+
+    Nodes should be provided in deletion order (reverse of import
+    order), with any excluded nodes already filtered out.
+
+    Args:
+        gen3_submission (Gen3Submission): An authenticated
+            Gen3Submission instance.
+        program_id (str): The Gen3 program name.
+        project_id (str): The Gen3 project name.
+        nodes (list[str]): Ordered list of node names to delete.
+        prompt_for_confirmation (bool): Whether to prompt for
+            confirmation before deletion.
+
+    Returns:
+        None
+    """
+    if not nodes:
+        logger.info("No nodes provided for deletion. Skipping.")
+        return
+
+    if prompt_for_confirmation:
+        confirm = input(
+            "Do you want to delete the metadata? (yes/no): "
+        ).strip().lower()
+        if confirm != "yes":
+            logger.info("Deletion cancelled by user.")
+            return
+
+    total_nodes = len(nodes)
+    for idx, node in enumerate(nodes, start=1):
+        logger.info(
+            "\033[94m[Node %d/%d]\033[0m | "
+            "Project: %-10s | Node: %-25s | Deleting...",
+            idx, total_nodes, project_id, node,
+        )
+        try:
+            gen3_submission.delete_nodes(
+                program_id, project_id, [node]
+            )
+            logger.info(
+                "\033[92m[SUCCESS]\033[0m | "
+                "Project: %-10s | Node: %-25s",
+                project_id, node,
+            )
+        except Exception as e:
+            logger.error(
+                "\033[91m[FAILED]\033[0m  | "
+                "Project: %-10s | Node: %-25s | "
+                "Error: %s",
+                project_id, node, e,
+            )