gen3-dataops-toolkit 3.6.0__tar.gz → 3.6.1__tar.gz

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.6.0
+Version: 3.6.1
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -224,6 +224,11 @@ g3dt synth generate synthetic_dataset_1 --llm -n 100 -e test
 g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1.2.0
 ```
 
+`deploy --skip-dict` runs the synthetic-data-only flow (delete previous
+batch, generate, upload, ETL) without re-uploading the dictionary or
+restarting the schema microservices — use it when the deployed dictionary is
+already current.
+
 `deploy` without `--studies` falls back to the original ACDC demo set
 (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
 at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.6.1}/README.md

@@ -190,6 +190,11 @@ g3dt synth generate synthetic_dataset_1 --llm -n 100 -e test
 g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1.2.0
 ```
 
+`deploy --skip-dict` runs the synthetic-data-only flow (delete previous
+batch, generate, upload, ETL) without re-uploading the dictionary or
+restarting the schema microservices — use it when the deployed dictionary is
+already current.
+
 `deploy` without `--studies` falls back to the original ACDC demo set
 (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
 at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.6.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.6.0"
+version = "3.6.1"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.6.1}/src/g3dt/cli/synth.py

@@ -226,6 +226,12 @@ def deploy(
         help="Dictionary version whose previously-uploaded synthetic batch is "
         f"deleted before uploading the new one. Default: {DEPLOY_DEFAULT_PREV_VERSION}.",
     ),
+    skip_dict: bool = typer.Option(
+        False, "--skip-dict",
+        help="Skip the dictionary upload and schema restarts (steps 1-2) — "
+        "synthetic data only: delete previous batch, generate, upload, run "
+        "ETL. Use when the deployed dictionary is already current.",
+    ),
 ) -> None:
     """Full end-to-end synthetic deploy: the whole cycle in one command.
 
@@ -240,6 +246,11 @@ def deploy(
       5. upload the new batch to Gen3
       6. run the ETL cronjob (env's SSM etl_cronjob)
 
+    With --skip-dict, steps 1-2 are skipped (the schema is still fetched
+    locally if missing — generation validates against it) and the flow is
+    synthetic-data only. Equivalent by hand: synth delete + synth generate +
+    synth upload + k8s restart-etl.
+
     Provider/model, restart targets, and the ETL cronjob come from the env's
     SSM tree unless overridden; the API key path comes from
     --llm-api-key-file or the marker. Studies and record counts are batch
@@ -247,6 +258,7 @@ def deploy(
 
     Examples:
       g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100
+      g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --skip-dict
       g3dt synth deploy -e test --studies "s1,s2" -n "100,50" --prev-version v1.2.0
     """
     e = env_of(env)
@@ -265,6 +277,8 @@ def deploy(
         env_vars["G3DT_SYNTH_NUM_RECORDS"] = num_records
     if prev_version:
         env_vars["G3DT_SYNTH_PREV_VERSION"] = prev_version
+    if skip_dict:
+        env_vars["G3DT_SYNTH_SKIP_DICT"] = "1"
     runner.run(
         runner.bash_script(
             "services/synthetic_data/full_deploy_dd_and_synth.sh", env

{gen3_dataops_toolkit-3.6.0 → gen3_dataops_toolkit-3.6.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh

@@ -84,22 +84,32 @@ if ! aws eks update-kubeconfig \
     exit 1
 fi
 
-echo "==== [1] Pulling dictionary for version ${VERSION} ===="
-bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "${DICT_URL}" "${DICT_FILENAME}"
-
-echo "==== [2] Uploading dictionary to S3: s3://${SCHEMA_S3_URI} ===="
-UPLOAD_DICT_ARGS=("${SCHEMA_DIR}/${DICT_FILENAME}" "s3://${SCHEMA_S3_URI}")
-# The optional trailing positional is the AWS profile; omit it for ambient credentials.
-if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
-    UPLOAD_DICT_ARGS+=("${G3DT_AWS_PROFILE}")
+if [ -z "${G3DT_SYNTH_SKIP_DICT:-}" ]; then
+    echo "==== [1] Pulling dictionary for version ${VERSION} ===="
+    bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "${DICT_URL}" "${DICT_FILENAME}"
+
+    echo "==== [2] Uploading dictionary to S3: s3://${SCHEMA_S3_URI} ===="
+    UPLOAD_DICT_ARGS=("${SCHEMA_DIR}/${DICT_FILENAME}" "s3://${SCHEMA_S3_URI}")
+    # The optional trailing positional is the AWS profile; omit it for ambient credentials.
+    if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
+        UPLOAD_DICT_ARGS+=("${G3DT_AWS_PROFILE}")
+    fi
+    python3 "${SERVICE_DIR}/dictionary/upload_dictionary.py" "${UPLOAD_DICT_ARGS[@]}"
+
+    echo "==== [3] Restarting microservices (schema) ===="
+    bash "${ARGO_SCRIPT_DIR}/argocd_restart_schema.sh" \
+        -d "${DOMAIN}" \
+        -a "${APP_NAME}" \
+        -n "${NAMESPACE}"
+else
+    echo "==== [1-3] Skipped dictionary upload + schema restarts (--skip-dict) ===="
+    # Generation still validates against the local schema copy — fetch it if
+    # it is missing (a local download only; the commons is not touched).
+    if [ ! -f "${SCHEMA_DIR}/${DICT_FILENAME}" ]; then
+        echo "Local schema missing; pulling ${VERSION}..."
+        bash "${SERVICE_DIR}/dictionary/pull_dict.sh" "${DICT_URL}" "${DICT_FILENAME}"
+    fi
 fi
-python3 "${SERVICE_DIR}/dictionary/upload_dictionary.py" "${UPLOAD_DICT_ARGS[@]}"
-
-echo "==== [3] Restarting microservices (schema) ===="
-bash "${ARGO_SCRIPT_DIR}/argocd_restart_schema.sh" \
-    -d "${DOMAIN}" \
-    -a "${APP_NAME}" \
-    -n "${NAMESPACE}"
 
 echo "==== [4] Deleting old synthetic data (${STUDIES}) for version ${PREV_VERSION} ===="
 DELETE_SYNTH_ARGS=(