gen3-dataops-toolkit 2.0.0__py3-none-any.whl

g3dt/services/k8s_ops/restart_etl_and_ms.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+# Ensure pipeline failures are captured
+set -o pipefail
+
+# Usage: bash restart_etl_and_ms.sh <profile>
+# <profile> is display-only; all configuration comes from G3DT_* environment
+# variables exported by the g3dt CLI (g3dt.config.script_env).
+PROFILE="${1:-}"
+
+if [ -z "$PROFILE" ]; then
+    echo "Usage: $0 <profile> (test|staging|prod) — run via the g3dt CLI"
+    exit 1
+fi
+
+# Defining script paths
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+# Configuration from G3DT_* env vars (fail loudly if a required one is missing)
+CLUSTER_NAME="${G3DT_CLUSTER_NAME:?G3DT_CLUSTER_NAME not set — run via the g3dt CLI}"
+DOMAIN="${G3DT_DOMAIN:?G3DT_DOMAIN not set — run via the g3dt CLI}"
+APP_NAME="${G3DT_APP_NAME:?G3DT_APP_NAME not set — run via the g3dt CLI}"
+NAMESPACE="${G3DT_NAMESPACE:?G3DT_NAMESPACE not set — run via the g3dt CLI}"
+REGION="${G3DT_REGION:-ap-southeast-2}"
+# This script lives alongside the argo scripts inside the package.
+ARGO_SCRIPT_DIR="${SCRIPT_DIR}"
+
+# Never export an empty AWS_PROFILE (empty means ambient credentials).
+if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+    export AWS_PROFILE="${G3DT_AWS_PROFILE}"
+    echo "==== Configuring AWS PROFILE as '${AWS_PROFILE}' for profile '${PROFILE}' ===="
+else
+    echo "==== No AWS profile set for '${PROFILE}' — using ambient AWS credentials ===="
+fi
+
+echo "Updating kubeconfig for cluster: ${CLUSTER_NAME}"
+if ! aws eks update-kubeconfig --name "${CLUSTER_NAME}" --region "${REGION}"; then
+    echo "ERROR: Failed to update kubeconfig. Check your AWS credentials/profile."
+    exit 1
+fi
+
+echo "==== Restarting microservices (etl) ===="
+bash "${ARGO_SCRIPT_DIR}/argocd_restart_etl.sh" \
+    -d "${DOMAIN}" \
+    -a "${APP_NAME}" \
+    -n "${NAMESPACE}" \
+    -s
+
+echo "==== Restarting microservices (schema) ===="
+bash "${ARGO_SCRIPT_DIR}/argocd_restart_ms.sh" \
+    -d "${DOMAIN}" \
+    -a "${APP_NAME}" \
+    -n "${NAMESPACE}" \
+    -r "sheepdog-deployment,guppy-deployment,peregrine-deployment,portal-deployment"

g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py

@@ -0,0 +1,183 @@
+import os
+import argparse
+import logging
+from g3dt.upload.metadata_deleter import (
+    delete_project_metadata,
+)
+from g3dt.upload.metadata_submitter import (
+    create_boto3_session,
+    get_gen3_api_key_aws_secret,
+    create_gen3_submission_class,
+)
+
+EXCLUDE_NODES = [
+    "program",
+    "project",
+    "acknowledgement",
+    "publication",
+]
+
+
+def load_import_order(import_order_path, exclude_nodes=None):
+    """
+    Reads the DataImportOrder.txt file and returns the node list
+    in deletion order (reversed, with excluded nodes removed).
+    """
+    if exclude_nodes is None:
+        exclude_nodes = EXCLUDE_NODES
+    with open(import_order_path, 'r', encoding='utf-8') as f:
+        nodes = [line.strip() for line in f if line.strip()]
+    nodes = [n for n in nodes if n not in exclude_nodes]
+    nodes.reverse()
+    return nodes
+
+
+def delete_for_projects(
+    gen3_submission,
+    program_id,
+    project_ids,
+    nodes,
+    prompt_for_confirmation,
+):
+    for project_id in project_ids:
+        logging.info(
+            "Attempting to delete metadata for project '%s'",
+            project_id,
+        )
+        try:
+            delete_project_metadata(
+                gen3_submission=gen3_submission,
+                program_id=program_id,
+                project_id=project_id,
+                nodes=nodes,
+                prompt_for_confirmation=prompt_for_confirmation,
+            )
+            logging.info(
+                "Successfully deleted metadata for project '%s'.",
+                project_id,
+            )
+        except Exception as e:
+            logging.error(
+                "Error deleting metadata for project '%s': %s",
+                project_id, e,
+            )
+            raise e
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format='[%(asctime)s] [%(levelname)s] %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S',
+    )
+
+    script_path = os.path.dirname(os.path.abspath(__file__))
+    os.chdir(script_path)
+    logging.info("Changed working directory to %s", script_path)
+
+    default_projects = [
+        "AusDiab_Simulated",
+        "EDCAD-PMS_Simulated",
+        "Baker-Biobank_Simulated",
+        "CAUGHT-CAD_Simulated",
+        "BioHeart-CT_Simulated",
+    ]
+    default_program_id = "program1"
+    default_aws_profile = "default"
+    default_aws_region = "ap-southeast-2"
+    default_prompt_for_confirmation = False
+
+    parser = argparse.ArgumentParser(
+        description=(
+            "Delete synthetic project metadata from Gen3 "
+            "Sheepdog API."
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        '-i', '--data-import-order-file',
+        type=str,
+        required=True,
+        help="Path to the DataImportOrder.txt file.",
+    )
+    parser.add_argument(
+        '-p', '--projects',
+        type=str,
+        default=",".join(default_projects),
+        help="Comma separated list of Project IDs.",
+    )
+    parser.add_argument(
+        '--program-id',
+        type=str,
+        default=default_program_id,
+        help="Gen3 program name.",
+    )
+    parser.add_argument(
+        '-s', '--aws-secret-name',
+        type=str,
+        help="AWS secret name for Gen3 API key.",
+    )
+    parser.add_argument(
+        '-profile', '--aws-profile',
+        type=str,
+        default=default_aws_profile,
+        help="AWS profile name.",
+    )
+    parser.add_argument(
+        '-r', '--aws-region',
+        type=str,
+        default=default_aws_region,
+        help="AWS region.",
+    )
+    parser.add_argument(
+        '-confirm', '--prompt',
+        action='store_true',
+        default=default_prompt_for_confirmation,
+        help="Prompt for confirmation before deleting.",
+    )
+
+    args = parser.parse_args()
+
+    parsed_projects = [
+        proj.strip()
+        for proj in args.projects.split(",")
+        if proj.strip()
+    ]
+
+    logging.info("Projects to delete: %s", parsed_projects)
+    logging.info(
+        "Data import order file: %s",
+        args.data_import_order_file,
+    )
+    logging.info("AWS Secret Name: %s", args.aws_secret_name)
+    logging.info("AWS Profile: %s", args.aws_profile)
+    logging.info("AWS Region: %s", args.aws_region)
+    logging.info(
+        "Prompt for confirmation: %s", args.prompt,
+    )
+
+    if not os.path.isfile(args.data_import_order_file):
+        raise FileNotFoundError(
+            f"Import order file does not exist: "
+            f"{args.data_import_order_file}"
+        )
+
+    nodes = load_import_order(args.data_import_order_file)
+
+    session = create_boto3_session(
+        aws_profile=args.aws_profile,
+    )
+    api_key = get_gen3_api_key_aws_secret(
+        secret_name=args.aws_secret_name,
+        region_name=args.aws_region,
+        session=session,
+    )
+    sub = create_gen3_submission_class(api_key)
+
+    delete_for_projects(
+        gen3_submission=sub,
+        program_id=args.program_id,
+        project_ids=parsed_projects,
+        nodes=nodes,
+        prompt_for_confirmation=args.prompt,
+    )

g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+# Ensure pipeline failures are captured
+set -o pipefail
+
+# Usage: bash full_deploy_dd_and_synth.sh <profile>
+# <profile> is display-only; all configuration comes from G3DT_* environment
+# variables exported by the g3dt CLI (g3dt.config.script_env).
+PROFILE=$1
+
+if [ -z "$PROFILE" ]; then
+    echo "Usage: $0 <profile> (test|staging|prod) — run via the g3dt CLI"
+    exit 1
+fi
+
+# Any configured profile is allowed here. The production warning/confirmation is
+# enforced by the CLI entrypoint (`g3dt synth deploy`) before this script runs,
+# since that has a TTY for the prompt.
+
+# Defining script paths (sibling scripts ship together inside the package)
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+SERVICE_DIR="${SCRIPT_DIR}/.."
+
+# Configuration from G3DT_* env vars (fail loudly if a required one is missing)
+VERSION="${G3DT_DICTIONARY_VERSION:?G3DT_DICTIONARY_VERSION not set — run via the g3dt CLI}"
+AWS_SECRET_NAME="${G3DT_AWS_SECRET_NAME:?G3DT_AWS_SECRET_NAME not set — run via the g3dt CLI}"
+SCHEMA_S3_URI="${G3DT_SCHEMA_S3_URI:?G3DT_SCHEMA_S3_URI not set — run via the g3dt CLI}"
+DOMAIN="${G3DT_DOMAIN:?G3DT_DOMAIN not set — run via the g3dt CLI}"
+APP_NAME="${G3DT_APP_NAME:?G3DT_APP_NAME not set — run via the g3dt CLI}"
+NAMESPACE="${G3DT_NAMESPACE:?G3DT_NAMESPACE not set — run via the g3dt CLI}"
+CLUSTER_NAME="${G3DT_CLUSTER_NAME:?G3DT_CLUSTER_NAME not set — run via the g3dt CLI}"
+SCHEMA_REPO="${G3DT_SCHEMA_REPO:?G3DT_SCHEMA_REPO not set — run via the g3dt CLI}"
+REGION="${G3DT_REGION:-ap-southeast-2}"
+EKS_ARN="${G3DT_EKS_ARN:-}"
+ARGO_SCRIPT_DIR="${SERVICE_DIR}/k8s_ops"
+
+# Writable data lives outside the installed package.
+SCHEMA_DIR="${G3DT_SCHEMA_DIR:-$HOME/.g3dt/schemas}"
+SYNTH_BASE="${G3DT_SYNTH_DIR:-$HOME/.g3dt/synth_metadata}"
+
+# Derived variables
+PREV_VERSION="v1.0.0"
+SYNTH_META_DIR="${SYNTH_BASE}/${VERSION}/"
+DATA_IMPORT_ORDER_FILE="${SYNTH_BASE}/${PREV_VERSION}/AusDiab_Simulated/DataImportOrder.txt"
+
+# Never export an empty AWS_PROFILE (empty means ambient credentials).
+if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+    export AWS_PROFILE="${G3DT_AWS_PROFILE}"
+    echo "==== [0] Configuring AWS PROFILE as '${AWS_PROFILE}' for profile '${PROFILE}' ===="
+else
+    echo "==== [0] No AWS profile set for '${PROFILE}' — using ambient AWS credentials ===="
+fi
+
+echo "Updating kubeconfig for cluster: ${CLUSTER_NAME}"
+# eks_arn is optional; only pass --role-arn when it is set.
+ROLE_ARG=""
+if [ -n "${EKS_ARN}" ] && [ "${EKS_ARN}" != "null" ]; then
+    ROLE_ARG="--role-arn ${EKS_ARN}"
+fi
+if ! aws eks update-kubeconfig \
+    --name "${CLUSTER_NAME}" \
+    --region "${REGION}" \
+    ${ROLE_ARG}; then
+    echo "ERROR: Failed to update kubeconfig. Check your AWS credentials/profile."
+    exit 1
+fi
+
+echo "==== [1] Pulling dictionary for version ${VERSION} ===="
+DICT_URL="https://raw.githubusercontent.com/${SCHEMA_REPO}"
+DICT_URL="${DICT_URL}/refs/tags/${VERSION}"
+DICT_URL="${DICT_URL}/dictionary/prod_dict/acdc_schema.json"
+bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "${DICT_URL}"
+
+echo "==== [2] Uploading dictionary to S3: s3://${SCHEMA_S3_URI} ===="
+UPLOAD_DICT_ARGS=("${SCHEMA_DIR}/acdc_schema_${VERSION}.json" "s3://${SCHEMA_S3_URI}")
+# The optional trailing positional is the AWS profile; omit it for ambient credentials.
+if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+    UPLOAD_DICT_ARGS+=("${G3DT_AWS_PROFILE}")
+fi
+python3 "${SERVICE_DIR}/dictionary/upload_dictionary.py" "${UPLOAD_DICT_ARGS[@]}"
+
+echo "==== [3] Restarting microservices (schema) ===="
+bash "${ARGO_SCRIPT_DIR}/argocd_restart_schema.sh" \
+    -d "${DOMAIN}" \
+    -a "${APP_NAME}" \
+    -n "${NAMESPACE}"
+
+echo "==== [4] Deleting old synthetic data for version ${PREV_VERSION} ===="
+DELETE_SYNTH_ARGS=(
+    -p "AusDiab_Simulated,EDCAD-PMS_Simulated,PREDICT_Simulated,Baker-Biobank_Simulated,CAUGHT-CAD_Simulated,BioHeart-CT_Simulated"
+    -s "${AWS_SECRET_NAME}"
+    -i "${DATA_IMPORT_ORDER_FILE}"
+)
+if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+    DELETE_SYNTH_ARGS+=(-profile "${G3DT_AWS_PROFILE}")
+fi
+python3 "${SERVICE_DIR}/synthetic_data/delete_synth_metadata_sheepdog.py" "${DELETE_SYNTH_ARGS[@]}"
+
+echo "==== [5] Generating new synthetic data for version ${VERSION} (LLM-realistic) ===="
+bash "${SERVICE_DIR}/synthetic_data/generate_synth_metadata.sh" \
+    --schema "${SCHEMA_DIR}/acdc_schema_${VERSION}.json" \
+    --version "${VERSION}" \
+    --provider llm \
+    --num-records "30,60,20,55" \
+    --output-root "${SYNTH_BASE}"
+
+echo "==== [6] Uploading new synthetic data for version ${VERSION} ===="
+UPLOAD_SYNTH_ARGS=(
+    --base-dir "${SYNTH_META_DIR}"
+    --aws-secret-name "${AWS_SECRET_NAME}"
+)
+if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+    UPLOAD_SYNTH_ARGS+=(--aws-profile "${G3DT_AWS_PROFILE}")
+fi
+python3 "${SERVICE_DIR}/synthetic_data/upload_synth_metadata_sheepdog.py" "${UPLOAD_SYNTH_ARGS[@]}"
+
+echo "==== [7] Restarting microservices (schema and etl) ===="
+bash "${ARGO_SCRIPT_DIR}/argocd_restart_etl.sh" \
+    -d "${DOMAIN}" \
+    -a "${APP_NAME}" \
+    -n "${NAMESPACE}" \
+    -s

g3dt/services/synthetic_data/generate_synth_metadata.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+#
+# Generate schema-valid synthetic Gen3 metadata using gen3-metadata-simulator.
+# One run per study writes a self-validated folder containing <node>.json +
+# project.json + DataImportOrder.txt — the exact layout the upload step
+# (upload_synth_metadata_sheepdog.py) consumes.
+#
+# The tool takes a LOCAL bundled Gen3 schema file (pulled by pull_dict.sh into
+# ~/.g3dt/schemas/acdc_schema_<version>.json, or $G3DT_SCHEMA_DIR if set). The
+# default provider is keyless 'random'; pass --provider llm for LLM-realistic
+# values, in which case LLM config is read from ~/.g3dt/.env (or $G3DT_ENV_FILE
+# if set): LLM_PROVIDER / LLM_MODEL / LLM_API_KEY_FILE.
+
+set -euo pipefail
+
+# LLM provider config file (lives outside the installed package).
+ENV_FILE="${G3DT_ENV_FILE:-$HOME/.g3dt/.env}"
+
+usage() {
+    cat <<EOF
+Usage: $(basename "$0") --schema <path> --version <ver> [options]
+
+Generate synthetic Gen3 metadata (one folder per study) with gen3-metadata-simulator.
+
+Required:
+  --schema <path>        Path to the bundled Gen3 JSON schema (cad.json).
+  --version <ver>        Version label for the output dir (e.g. v1.1.5).
+
+Options:
+  --studies s1,s2        Comma-separated study/project names.
+                         Default: ${DEFAULT_STUDIES}
+  --num-records N|n1,n2  Records per study: one number for all, or a comma list
+                         (one per study). Default: ${DEFAULT_NUM_RECORDS}
+  --provider random|llm  Value strategy. Default: ${DEFAULT_PROVIDER}
+                         'random' needs no key; 'llm' reads LLM config from
+                         ${ENV_FILE}.
+  --seed N               RNG seed for reproducible output.
+  --output-root DIR      Root output dir. Default: ${DEFAULT_OUTPUT_ROOT}
+  -h, --help             Show this help and exit.
+
+Examples:
+  $(basename "$0") --schema ~/.g3dt/schemas/acdc_schema_v1.1.5.json --version v1.1.5
+  $(basename "$0") --schema schema.json --version v1.1.5 --provider random --num-records 5
+  $(basename "$0") --schema schema.json --version v1.1.5 --num-records "30,60,20,55"
+EOF
+}
+
+# Defaults (kept in sync with full_deploy_dd_and_synth.sh's 4-study record list)
+DEFAULT_STUDIES="AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated"
+DEFAULT_NUM_RECORDS=30
+DEFAULT_PROVIDER=random
+# Generated data goes outside the installed package.
+DEFAULT_OUTPUT_ROOT="${G3DT_SYNTH_DIR:-$HOME/.g3dt/synth_metadata}"
+
+SCHEMA=""
+VERSION=""
+STUDIES="${DEFAULT_STUDIES}"
+NUM_RECORDS="${DEFAULT_NUM_RECORDS}"
+PROVIDER="${DEFAULT_PROVIDER}"
+SEED=""
+OUTPUT_ROOT="${DEFAULT_OUTPUT_ROOT}"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --schema)       SCHEMA="$2"; shift 2 ;;
+        --version)      VERSION="$2"; shift 2 ;;
+        --studies)      STUDIES="$2"; shift 2 ;;
+        --num-records)  NUM_RECORDS="$2"; shift 2 ;;
+        --provider)     PROVIDER="$2"; shift 2 ;;
+        --seed)         SEED="$2"; shift 2 ;;
+        --output-root)  OUTPUT_ROOT="$2"; shift 2 ;;
+        -h|--help)      usage; exit 0 ;;
+        *) echo "Unknown argument: $1" >&2; usage; exit 1 ;;
+    esac
+done
+
+if [[ -z "$SCHEMA" || -z "$VERSION" ]]; then
+    echo "Error: --schema and --version are required." >&2
+    usage
+    exit 1
+fi
+if [[ ! -f "$SCHEMA" ]]; then
+    echo "Error: schema file not found: ${SCHEMA}" >&2
+    echo "Hint: pull it first, e.g. 'g3dt dict pull'." >&2
+    exit 1
+fi
+if ! command -v gen3-metadata-simulator &>/dev/null; then
+    echo "Error: gen3-metadata-simulator not found. Run 'g3dt synth install-simulator'." >&2
+    exit 1
+fi
+
+IFS=',' read -r -a STUDY_ARRAY <<< "$STUDIES"
+
+# --num-records is either a single count applied to every study, or a comma list
+# with one count per study (which must line up with --studies).
+PER_STUDY_COUNTS=0
+if [[ "$NUM_RECORDS" == *,* ]]; then
+    PER_STUDY_COUNTS=1
+    IFS=',' read -r -a NUM_RECORDS_ARRAY <<< "$NUM_RECORDS"
+    if [[ ${#NUM_RECORDS_ARRAY[@]} -ne ${#STUDY_ARRAY[@]} ]]; then
+        echo "Error: --num-records has ${#NUM_RECORDS_ARRAY[@]} values but there are ${#STUDY_ARRAY[@]} studies." >&2
+        exit 1
+    fi
+fi
+
+echo "Generating synthetic metadata: provider=${PROVIDER}, version=${VERSION}, studies=${STUDIES}"
+
+for i in "${!STUDY_ARRAY[@]}"; do
+    STUDY="${STUDY_ARRAY[$i]}"
+    if [[ "$PER_STUDY_COUNTS" -eq 1 ]]; then
+        N="${NUM_RECORDS_ARRAY[$i]}"
+    else
+        N="$NUM_RECORDS"
+    fi
+    OUT="${OUTPUT_ROOT}/${VERSION}/${STUDY}"
+    mkdir -p "$OUT"
+    echo "==== ${STUDY} (n=${N}) -> ${OUT} ===="
+
+    CMD=(gen3-metadata-simulator generate
+         --schema "$SCHEMA"
+         --output-dir "$OUT"
+         --project-code "$STUDY"
+         --num-records "$N"
+         --provider "$PROVIDER")
+    [[ -n "$SEED" ]] && CMD+=(--seed "$SEED")
+    # Point the LLM provider at the user-level env file regardless of the caller's CWD.
+    if [[ "$PROVIDER" == "llm" && -f "${ENV_FILE}" ]]; then
+        CMD+=(--env-file "${ENV_FILE}")
+    fi
+    "${CMD[@]}"
+done
+
+echo "Done. Output under ${OUTPUT_ROOT}/${VERSION}/"

g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py

@@ -0,0 +1,165 @@
+import os
+import argparse
+import logging
+from g3dt.upload.metadata_submitter import (
+    find_data_import_order_file,
+    list_metadata_jsons,
+    create_boto3_session,
+    get_gen3_api_key_aws_secret,
+    MetadataSubmitter,
+)
+
+DEFAULT_REGION = "ap-southeast-2"
+DEFAULT_PROFILE = None  # ambient credentials unless --aws-profile is given
+DEFAULT_SUBMISSION_SIZE_KB = 50
+
+def submit_synthetic_metadata(
+    base_dir,
+    project_id,
+    aws_secret_name,
+    aws_region=DEFAULT_REGION,
+    aws_profile=DEFAULT_PROFILE,
+    max_submission_size_kb=DEFAULT_SUBMISSION_SIZE_KB,
+):
+    """
+    Main function to orchestrate metadata submission for multiple projects.
+    """
+    logger = logging.getLogger(__name__)
+    script_path = os.path.dirname(os.path.abspath(__file__))
+    os.chdir(script_path)
+    logger.info("Changed working directory to %s", script_path)
+
+    logger.info("Using base_dir=%s", base_dir)
+
+    session = create_boto3_session(aws_profile)
+    api_key = get_gen3_api_key_aws_secret(aws_secret_name, aws_region, session)
+
+    logger.info(
+        "Preparing to submit metadata for project_id=%s from %s",
+        project_id,
+        base_dir,
+    )
+
+    try:
+        data_import_order_path = find_data_import_order_file(base_dir)
+        file_list = list_metadata_jsons(base_dir)
+
+        logger.info("Found %d metadata files for submission.", len(file_list))
+
+        submitter = MetadataSubmitter(
+            metadata_file_list=file_list,
+            api_key=api_key,
+            project_id=project_id,
+            data_import_order_path=data_import_order_path,
+            max_size_kb=max_submission_size_kb,
+            aws_profile=aws_profile,
+            upload_to_database=False,
+            dataset_root=None,
+            database=None,
+            table=None
+        )
+        submitter.submit_metadata()
+        logger.info("Finished submitting for project_id=%s", project_id)
+
+    except FileNotFoundError as e:
+        logger.error(
+            "Could not find required files for project %s in %s. Error: %s",
+            project_id,
+            base_dir,
+            e,
+        )
+        raise e
+    except Exception as e:
+        logger.error(
+            "An unexpected error occurred during submission for project %s: %s",
+            project_id,
+            e,
+        )
+        raise e
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Upload synthetic Gen3 metadata for all projects in a directory."
+    )
+    parser.add_argument(
+        "--base-dir",
+        required=True,
+        help="Base directory where metadata project folders are found."
+    )
+    parser.add_argument(
+        "--aws-secret-name",
+        required=True,
+        help="AWS secrets manager secret name containing the Gen3 API key."
+    )
+    parser.add_argument(
+        "--aws-region",
+        default=DEFAULT_REGION,
+        help=f"AWS region for secrets manager (default: {DEFAULT_REGION})."
+    )
+    parser.add_argument(
+        "--aws-profile",
+        default=DEFAULT_PROFILE,
+        help="AWS profile to use (default: None)."
+    )
+    parser.add_argument(
+        "--submission-size-kb",
+        type=int,
+        default=DEFAULT_SUBMISSION_SIZE_KB,
+        help="Maximum submission size in KB per chunk (default: 50)."
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)s:%(name)s:%(message)s", level=logging.INFO
+    )
+    logger = logging.getLogger(__name__)
+
+    logger.info("Starting metadata submission script.")
+
+    # Validate base_dir
+    if not os.path.isdir(args.base_dir):
+        logger.error(
+            "The base directory %s does not exist or is not a directory.",
+            args.base_dir
+        )
+        raise FileNotFoundError(
+            "The base directory %s does not exist or is not a directory."
+            % args.base_dir
+        )
+
+    # Find all projects: folders inside base_dir
+    logger.info(
+        "Looking for project subdirectories in base directory: %s",
+        args.base_dir
+    )
+    project_id_list = os.listdir(args.base_dir)
+    logger.info(
+        "Found %d project subdirectories: %s",
+        len(project_id_list),
+        project_id_list
+    )
+    if not project_id_list:
+        logger.error(
+            "No project subdirectories found in base directory: %s",
+            args.base_dir
+        )
+        raise FileNotFoundError(
+            "No project subdirectories found in base directory: %s"
+            % args.base_dir
+        )
+
+    for project_id in project_id_list:
+        project_base_dir = os.path.join(args.base_dir, project_id)
+        submit_synthetic_metadata(
+            base_dir=project_base_dir,
+            project_id=project_id,
+            aws_secret_name=args.aws_secret_name,
+            aws_region=args.aws_region,
+            aws_profile=args.aws_profile,
+            max_submission_size_kb=args.submission_size_kb,
+        )
+
+    logger.info("Script finished.")
+
+if __name__ == "__main__":
+    main()