gen3-dataops-toolkit 3.6.0__tar.gz → 3.7.0__tar.gz

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.6.0
+Version: 3.7.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -224,6 +224,14 @@ g3dt synth generate synthetic_dataset_1 --llm -n 100 -e test
 g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1.2.0
 ```
 
+`deploy --skip-dict` runs the synthetic-data-only flow (delete previous
+batch, generate, upload, ETL) without re-uploading the dictionary or
+restarting the schema microservices — use it when the deployed dictionary is
+already current. Deleting the previous batch is the flow's one destructive
+step, so it asks for confirmation (decline to skip it, or pass
+`--skip-delete` to skip without prompting); on a first deploy, when no
+previous batch exists locally, deletion is skipped automatically.
+
 `deploy` without `--studies` falls back to the original ACDC demo set
 (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
 at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.7.0}/README.md

@@ -190,6 +190,14 @@ g3dt synth generate synthetic_dataset_1 --llm -n 100 -e test
 g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1.2.0
 ```
 
+`deploy --skip-dict` runs the synthetic-data-only flow (delete previous
+batch, generate, upload, ETL) without re-uploading the dictionary or
+restarting the schema microservices — use it when the deployed dictionary is
+already current. Deleting the previous batch is the flow's one destructive
+step, so it asks for confirmation (decline to skip it, or pass
+`--skip-delete` to skip without prompting); on a first deploy, when no
+previous batch exists locally, deletion is skipped automatically.
+
 `deploy` without `--studies` falls back to the original ACDC demo set
 (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
 at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.6.0"
+version = "3.7.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.7.0}/src/g3dt/cli/synth.py

@@ -226,6 +226,18 @@ def deploy(
         help="Dictionary version whose previously-uploaded synthetic batch is "
         f"deleted before uploading the new one. Default: {DEPLOY_DEFAULT_PREV_VERSION}.",
     ),
+    skip_dict: bool = typer.Option(
+        False, "--skip-dict",
+        help="Skip the dictionary upload and schema restarts (steps 1-2) — "
+        "synthetic data only: delete previous batch, generate, upload, run "
+        "ETL. Use when the deployed dictionary is already current.",
+    ),
+    skip_delete: bool = typer.Option(
+        False, "--skip-delete",
+        help="Skip step 3 (deleting the previous synthetic batch) without "
+        "prompting. Without this flag, deletion asks for confirmation; "
+        "declining skips it and the flow continues.",
+    ),
 ) -> None:
     """Full end-to-end synthetic deploy: the whole cycle in one command.
 
@@ -240,6 +252,16 @@ def deploy(
       5. upload the new batch to Gen3
       6. run the ETL cronjob (env's SSM etl_cronjob)
 
+    With --skip-dict, steps 1-2 are skipped (the schema is still fetched
+    locally if missing — generation validates against it) and the flow is
+    synthetic-data only. Equivalent by hand: synth delete + synth generate +
+    synth upload + k8s restart-etl.
+
+    Step 3 is destructive, so it asks for confirmation (skip it outright with
+    --skip-delete, or decline the prompt — the flow continues either way).
+    On a first deploy, when no previous batch exists locally, deletion is
+    skipped automatically.
+
     Provider/model, restart targets, and the ETL cronjob come from the env's
     SSM tree unless overridden; the API key path comes from
     --llm-api-key-file or the marker. Studies and record counts are batch
@@ -247,12 +269,22 @@ def deploy(
 
     Examples:
       g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100
+      g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --skip-dict
       g3dt synth deploy -e test --studies "s1,s2" -n "100,50" --prev-version v1.2.0
     """
     e = env_of(env)
     safety.confirm_prod_strict("synthetic full deploy", env)
     effective_studies = studies or DEPLOY_DEFAULT_STUDIES
     _check_per_study_counts(effective_studies, num_records)
+    if not skip_delete:
+        # Deleting the previous batch is the one destructive step in the
+        # flow; declining just skips it — everything else still runs.
+        effective_prev = prev_version or DEPLOY_DEFAULT_PREV_VERSION
+        skip_delete = not typer.confirm(
+            f"Delete the previous synthetic batch ({effective_studies} @ "
+            f"{effective_prev}) from the commons before uploading the new one?",
+            default=True,
+        )
     env_vars = script_env(e)
     env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
     if restart_services:
@@ -265,6 +297,10 @@ def deploy(
         env_vars["G3DT_SYNTH_NUM_RECORDS"] = num_records
     if prev_version:
         env_vars["G3DT_SYNTH_PREV_VERSION"] = prev_version
+    if skip_dict:
+        env_vars["G3DT_SYNTH_SKIP_DICT"] = "1"
+    if skip_delete:
+        env_vars["G3DT_SYNTH_SKIP_DELETE"] = "1"
     runner.run(
         runner.bash_script(
             "services/synthetic_data/full_deploy_dd_and_synth.sh", env

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.7.0}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh

@@ -84,33 +84,53 @@ if ! aws eks update-kubeconfig \
     exit 1
 fi
 
-echo "==== [1] Pulling dictionary for version ${VERSION} ===="
-bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "${DICT_URL}" "${DICT_FILENAME}"
-
-echo "==== [2] Uploading dictionary to S3: s3://${SCHEMA_S3_URI} ===="
-UPLOAD_DICT_ARGS=("${SCHEMA_DIR}/${DICT_FILENAME}" "s3://${SCHEMA_S3_URI}")
-# The optional trailing positional is the AWS profile; omit it for ambient credentials.
-if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
-    UPLOAD_DICT_ARGS+=("${G3DT_AWS_PROFILE}")
+if [ -z "${G3DT_SYNTH_SKIP_DICT:-}" ]; then
+    echo "==== [1] Pulling dictionary for version ${VERSION} ===="
+    bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "${DICT_URL}" "${DICT_FILENAME}"
+
+    echo "==== [2] Uploading dictionary to S3: s3://${SCHEMA_S3_URI} ===="
+    UPLOAD_DICT_ARGS=("${SCHEMA_DIR}/${DICT_FILENAME}" "s3://${SCHEMA_S3_URI}")
+    # The optional trailing positional is the AWS profile; omit it for ambient credentials.
+    if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+        UPLOAD_DICT_ARGS+=("${G3DT_AWS_PROFILE}")
+    fi
+    python3 "${SERVICE_DIR}/dictionary/upload_dictionary.py" "${UPLOAD_DICT_ARGS[@]}"
+
+    echo "==== [3] Restarting microservices (schema) ===="
+    bash "${ARGO_SCRIPT_DIR}/argocd_restart_schema.sh" \
+        -d "${DOMAIN}" \
+        -a "${APP_NAME}" \
+        -n "${NAMESPACE}"
+else
+    echo "==== [1-3] Skipped dictionary upload + schema restarts (--skip-dict) ===="
+    # Generation still validates against the local schema copy — fetch it if
+    # it is missing (a local download only; the commons is not touched).
+    if [ ! -f "${SCHEMA_DIR}/${DICT_FILENAME}" ]; then
+        echo "Local schema missing; pulling ${VERSION}..."
+        bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "${DICT_URL}" "${DICT_FILENAME}"
+    fi
 fi
-python3 "${SERVICE_DIR}/dictionary/upload_dictionary.py" "${UPLOAD_DICT_ARGS[@]}"
 
-echo "==== [3] Restarting microservices (schema) ===="
-bash "${ARGO_SCRIPT_DIR}/argocd_restart_schema.sh" \
-    -d "${DOMAIN}" \
-    -a "${APP_NAME}" \
-    -n "${NAMESPACE}"
-
-echo "==== [4] Deleting old synthetic data (${STUDIES}) for version ${PREV_VERSION} ===="
-DELETE_SYNTH_ARGS=(
-    -p "${STUDIES}"
-    -s "${AWS_SECRET_NAME}"
-    -i "${DATA_IMPORT_ORDER_FILE}"
-)
-if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
-    DELETE_SYNTH_ARGS+=(-profile "${G3DT_AWS_PROFILE}")
+if [ -n "${G3DT_SYNTH_SKIP_DELETE:-}" ]; then
+    echo "==== [4] Skipping deletion of the previous synthetic batch (per operator choice) ===="
+elif [ ! -f "${DATA_IMPORT_ORDER_FILE}" ]; then
+    # First deploy for these studies (or a different --prev-version): there is
+    # no previous local batch to walk, so there is nothing to delete.
+    echo "==== [4] No previous batch found — skipping deletion ===="
+    echo "     (missing ${DATA_IMPORT_ORDER_FILE};"
+    echo "      first deploy for these studies, or pass --prev-version)"
+else
+    echo "==== [4] Deleting old synthetic data (${STUDIES}) for version ${PREV_VERSION} ===="
+    DELETE_SYNTH_ARGS=(
+        -p "${STUDIES}"
+        -s "${AWS_SECRET_NAME}"
+        -i "${DATA_IMPORT_ORDER_FILE}"
+    )
+    if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+        DELETE_SYNTH_ARGS+=(-profile "${G3DT_AWS_PROFILE}")
+    fi
+    python3 "${SERVICE_DIR}/synthetic_data/delete_synth_metadata_sheepdog.py" "${DELETE_SYNTH_ARGS[@]}"
 fi
-python3 "${SERVICE_DIR}/synthetic_data/delete_synth_metadata_sheepdog.py" "${DELETE_SYNTH_ARGS[@]}"
 
 echo "==== [5] Generating new synthetic data (${STUDIES}) for version ${VERSION} (LLM-realistic) ===="
 bash "${SERVICE_DIR}/synthetic_data/generate_synth_metadata.sh" \