gen3-dataops-toolkit 3.6.1__tar.gz → 3.7.0__tar.gz

{gen3_dataops_toolkit-3.6.1 → gen3_dataops_toolkit-3.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.6.1
+Version: 3.7.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -227,7 +227,10 @@ g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1
 `deploy --skip-dict` runs the synthetic-data-only flow (delete previous
 batch, generate, upload, ETL) without re-uploading the dictionary or
 restarting the schema microservices — use it when the deployed dictionary is
-already current.
+already current. Deleting the previous batch is the flow's one destructive
+step, so it asks for confirmation (decline to skip it, or pass
+`--skip-delete` to skip without prompting); on a first deploy, when no
+previous batch exists locally, deletion is skipped automatically.
 
 `deploy` without `--studies` falls back to the original ACDC demo set
 (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`

{gen3_dataops_toolkit-3.6.1 → gen3_dataops_toolkit-3.7.0}/README.md

@@ -193,7 +193,10 @@ g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1
 `deploy --skip-dict` runs the synthetic-data-only flow (delete previous
 batch, generate, upload, ETL) without re-uploading the dictionary or
 restarting the schema microservices — use it when the deployed dictionary is
-already current.
+already current. Deleting the previous batch is the flow's one destructive
+step, so it asks for confirmation (decline to skip it, or pass
+`--skip-delete` to skip without prompting); on a first deploy, when no
+previous batch exists locally, deletion is skipped automatically.
 
 `deploy` without `--studies` falls back to the original ACDC demo set
 (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`

{gen3_dataops_toolkit-3.6.1 → gen3_dataops_toolkit-3.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.6.1"
+version = "3.7.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-3.6.1 → gen3_dataops_toolkit-3.7.0}/src/g3dt/cli/synth.py

@@ -232,6 +232,12 @@ def deploy(
         "synthetic data only: delete previous batch, generate, upload, run "
         "ETL. Use when the deployed dictionary is already current.",
     ),
+    skip_delete: bool = typer.Option(
+        False, "--skip-delete",
+        help="Skip step 3 (deleting the previous synthetic batch) without "
+        "prompting. Without this flag, deletion asks for confirmation; "
+        "declining skips it and the flow continues.",
+    ),
 ) -> None:
     """Full end-to-end synthetic deploy: the whole cycle in one command.
 
@@ -251,6 +257,11 @@ def deploy(
     synthetic-data only. Equivalent by hand: synth delete + synth generate +
     synth upload + k8s restart-etl.
 
+    Step 3 is destructive, so it asks for confirmation (skip it outright with
+    --skip-delete, or decline the prompt — the flow continues either way).
+    On a first deploy, when no previous batch exists locally, deletion is
+    skipped automatically.
+
     Provider/model, restart targets, and the ETL cronjob come from the env's
     SSM tree unless overridden; the API key path comes from
     --llm-api-key-file or the marker. Studies and record counts are batch
@@ -265,6 +276,15 @@ def deploy(
     safety.confirm_prod_strict("synthetic full deploy", env)
     effective_studies = studies or DEPLOY_DEFAULT_STUDIES
     _check_per_study_counts(effective_studies, num_records)
+    if not skip_delete:
+        # Deleting the previous batch is the one destructive step in the
+        # flow; declining just skips it — everything else still runs.
+        effective_prev = prev_version or DEPLOY_DEFAULT_PREV_VERSION
+        skip_delete = not typer.confirm(
+            f"Delete the previous synthetic batch ({effective_studies} @ "
+            f"{effective_prev}) from the commons before uploading the new one?",
+            default=True,
+        )
     env_vars = script_env(e)
     env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
     if restart_services:
@@ -279,6 +299,8 @@ def deploy(
         env_vars["G3DT_SYNTH_PREV_VERSION"] = prev_version
     if skip_dict:
         env_vars["G3DT_SYNTH_SKIP_DICT"] = "1"
+    if skip_delete:
+        env_vars["G3DT_SYNTH_SKIP_DELETE"] = "1"
     runner.run(
         runner.bash_script(
             "services/synthetic_data/full_deploy_dd_and_synth.sh", env

{gen3_dataops_toolkit-3.6.1 → gen3_dataops_toolkit-3.7.0}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh

@@ -111,16 +111,26 @@ else
     fi
 fi
 
-echo "==== [4] Deleting old synthetic data (${STUDIES}) for version ${PREV_VERSION} ===="
-DELETE_SYNTH_ARGS=(
-    -p "${STUDIES}"
-    -s "${AWS_SECRET_NAME}"
-    -i "${DATA_IMPORT_ORDER_FILE}"
-)
-if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
-    DELETE_SYNTH_ARGS+=(-profile "${G3DT_AWS_PROFILE}")
+if [ -n "${G3DT_SYNTH_SKIP_DELETE:-}" ]; then
+    echo "==== [4] Skipping deletion of the previous synthetic batch (per operator choice) ===="
+elif [ ! -f "${DATA_IMPORT_ORDER_FILE}" ]; then
+    # First deploy for these studies (or a different --prev-version): there is
+    # no previous local batch to walk, so there is nothing to delete.
+    echo "==== [4] No previous batch found — skipping deletion ===="
+    echo "     (missing ${DATA_IMPORT_ORDER_FILE};"
+    echo "      first deploy for these studies, or pass --prev-version)"
+else
+    echo "==== [4] Deleting old synthetic data (${STUDIES}) for version ${PREV_VERSION} ===="
+    DELETE_SYNTH_ARGS=(
+        -p "${STUDIES}"
+        -s "${AWS_SECRET_NAME}"
+        -i "${DATA_IMPORT_ORDER_FILE}"
+    )
+    if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+        DELETE_SYNTH_ARGS+=(-profile "${G3DT_AWS_PROFILE}")
+    fi
+    python3 "${SERVICE_DIR}/synthetic_data/delete_synth_metadata_sheepdog.py" "${DELETE_SYNTH_ARGS[@]}"
 fi
-python3 "${SERVICE_DIR}/synthetic_data/delete_synth_metadata_sheepdog.py" "${DELETE_SYNTH_ARGS[@]}"
 
 echo "==== [5] Generating new synthetic data (${STUDIES}) for version ${VERSION} (LLM-realistic) ===="
 bash "${SERVICE_DIR}/synthetic_data/generate_synth_metadata.sh" \