gen3-dataops-toolkit 3.5.0__tar.gz → 3.6.0__tar.gz

{gen3_dataops_toolkit-3.5.0 → gen3_dataops_toolkit-3.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.5.0
+Version: 3.6.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -208,6 +208,28 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
 `g3dt config show --env <env>` prints the resolved provider, model, and key
 path.
 
+### The synthetic data lifecycle
+
+`synth generate` builds a batch locally (keyless `random` by default, `--llm`
+for realistic values); `synth upload` pushes a batch to the commons; `synth
+delete` removes one; and `synth deploy` runs the whole cycle in one command —
+dictionary pull + upload, schema restarts, delete of the previous batch,
+LLM-generate, upload, and the ETL run.
+
+**Studies and record counts are batch inputs, not environment facts** — they
+are passed per command and never come from SSM:
+
+```bash
+g3dt synth generate synthetic_dataset_1 --llm -n 100 -e test
+g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1.2.0
+```
+
+`deploy` without `--studies` falls back to the original ACDC demo set
+(`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
+at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any
+other project should always pass its own `--studies`. The deploy's delete step
+targets exactly the studies being regenerated, at `--prev-version`.
+
 ### Kubernetes restart targets
 
 `k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`

{gen3_dataops_toolkit-3.5.0 → gen3_dataops_toolkit-3.6.0}/README.md

@@ -174,6 +174,28 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
 `g3dt config show --env <env>` prints the resolved provider, model, and key
 path.
 
+### The synthetic data lifecycle
+
+`synth generate` builds a batch locally (keyless `random` by default, `--llm`
+for realistic values); `synth upload` pushes a batch to the commons; `synth
+delete` removes one; and `synth deploy` runs the whole cycle in one command —
+dictionary pull + upload, schema restarts, delete of the previous batch,
+LLM-generate, upload, and the ETL run.
+
+**Studies and record counts are batch inputs, not environment facts** — they
+are passed per command and never come from SSM:
+
+```bash
+g3dt synth generate synthetic_dataset_1 --llm -n 100 -e test
+g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100 --prev-version v1.2.0
+```
+
+`deploy` without `--studies` falls back to the original ACDC demo set
+(`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
+at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any
+other project should always pass its own `--studies`. The deploy's delete step
+targets exactly the studies being regenerated, at `--prev-version`.
+
 ### Kubernetes restart targets
 
 `k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`

{gen3_dataops_toolkit-3.5.0 → gen3_dataops_toolkit-3.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.5.0"
+version = "3.6.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-3.5.0 → gen3_dataops_toolkit-3.6.0}/src/g3dt/cli/synth.py

@@ -9,6 +9,12 @@ Every command accepts ``--env``; targeting a **production** environment (any env
 whose name contains ``prod``) shows a warning and requires typing the env name to
 confirm — it cannot be bypassed.
 
+Studies and record counts are **batch inputs**, passed per command
+(``generate STUDIES -n N``, ``deploy --studies ... -n ...``) — they are not
+environment facts, so they never come from SSM. The ``deploy`` defaults are
+the original ACDC demo set, kept for continuity; other projects pass their
+own.
+
 Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm``
 for LLM-realistic values. The LLM provider and model come from the
 environment's SSM tree (the CDK config's optional ``llm`` block, published as
@@ -44,6 +50,32 @@ app = typer.Typer(
 
 SYNTH_DIR = Path("~/.g3dt/synth_metadata").expanduser()
 
+#: Defaults for the full-deploy flow, kept for continuity with the original
+#: ACDC demo environment. Any other project should pass --studies (and
+#: --num-records) — there is no SSM fact for these because simulated study
+#: sets are batch inputs, not environment facts.
+DEPLOY_DEFAULT_STUDIES = (
+    "AusDiab_Simulated,Baker-Biobank_Simulated,"
+    "BioHeart-CT_Simulated,CAUGHT-CAD_Simulated"
+)
+DEPLOY_DEFAULT_NUM_RECORDS = "30,60,20,55"
+DEPLOY_DEFAULT_PREV_VERSION = "v1.0.0"
+
+
+def _check_per_study_counts(studies: str, num_records: Optional[str]) -> None:
+    """Exit 1 when a per-study count list does not line up with the studies."""
+    if num_records and "," in num_records:
+        n_counts = len(num_records.split(","))
+        n_studies = len(studies.split(","))
+        if n_counts != n_studies:
+            typer.secho(
+                f"--num-records has {n_counts} values but {n_studies} studies "
+                f"were given (pass one count, or one per study).",
+                fg=typer.colors.RED,
+                err=True,
+            )
+            raise typer.Exit(1)
+
 
 def _llm_env_overrides(
     e,
@@ -177,22 +209,62 @@ def deploy(
         None, "--etl-cronjob",
         help="ETL cronjob name; default: the env's SSM app/etl_cronjob.",
     ),
+    studies: Optional[str] = typer.Option(
+        None, "--studies",
+        help="Simulated study id(s), comma-separated. These are (re)generated, "
+        "uploaded, and their previous batch deleted. Default: the original "
+        f"demo set ({DEPLOY_DEFAULT_STUDIES}).",
+    ),
+    num_records: Optional[str] = typer.Option(
+        None, "--num-records", "-n",
+        help="Records per study: one number for all, or a comma list (one per "
+        "study). Default: 30 per study (or the classic 30,60,20,55 when "
+        "--studies is not given).",
+    ),
+    prev_version: Optional[str] = typer.Option(
+        None, "--prev-version",
+        help="Dictionary version whose previously-uploaded synthetic batch is "
+        f"deleted before uploading the new one. Default: {DEPLOY_DEFAULT_PREV_VERSION}.",
+    ),
 ) -> None:
-    """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts).
+    """Full end-to-end synthetic deploy: the whole cycle in one command.
+
+    Wraps services/synthetic_data/full_deploy_dd_and_synth.sh, which runs:
+
+    \b
+      1. pull the dictionary at the env's version and upload it to S3
+      2. restart the schema microservices (env's SSM restart_services order)
+      3. delete the PREVIOUS synthetic batch for the given studies
+         (--prev-version, so stale records don't linger in the commons)
+      4. LLM-generate a new batch for the studies (provider/model from SSM)
+      5. upload the new batch to Gen3
+      6. run the ETL cronjob (env's SSM etl_cronjob)
 
-    Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed
-    generation). Provider/model and the restart targets come from the env's
+    Provider/model, restart targets, and the ETL cronjob come from the env's
     SSM tree unless overridden; the API key path comes from
-    --llm-api-key-file or the marker.
+    --llm-api-key-file or the marker. Studies and record counts are batch
+    inputs with ACDC-era defaults — any other project should pass --studies.
+
+    Examples:
+      g3dt synth deploy -e test --studies synthetic_dataset_1 -n 100
+      g3dt synth deploy -e test --studies "s1,s2" -n "100,50" --prev-version v1.2.0
     """
     e = env_of(env)
     safety.confirm_prod_strict("synthetic full deploy", env)
+    effective_studies = studies or DEPLOY_DEFAULT_STUDIES
+    _check_per_study_counts(effective_studies, num_records)
     env_vars = script_env(e)
     env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
     if restart_services:
         env_vars["G3DT_RESTART_SERVICES"] = restart_services
     if etl_cronjob:
         env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob
+    if studies:
+        env_vars["G3DT_SYNTH_STUDIES"] = studies
+    if num_records:
+        env_vars["G3DT_SYNTH_NUM_RECORDS"] = num_records
+    if prev_version:
+        env_vars["G3DT_SYNTH_PREV_VERSION"] = prev_version
     runner.run(
         runner.bash_script(
             "services/synthetic_data/full_deploy_dd_and_synth.sh", env
@@ -254,25 +326,15 @@ def generate(
     generate LLM-realistic values instead.
 
     Examples:
-      g3dt synth generate AusDiab_Simulated -n 5 --seed 1
-      g3dt synth generate AusDiab_Simulated --llm -n 5
-      g3dt synth generate "AusDiab_Simulated,Baker-Biobank_Simulated" -n "30,60"
+      g3dt synth generate synthetic_dataset_1 -n 5 --seed 1
+      g3dt synth generate synthetic_dataset_1 --llm -n 100
+      g3dt synth generate "dataset_a,dataset_b" -n "30,60"
+      g3dt synth generate dataset_a --llm --llm-model gpt-4o-mini \
+          --llm-api-key-file ~/keys/openai_api_key.txt
     """
     e = env_of(env)
     safety.confirm_prod_strict("synthetic generation", env)
-
-    # A comma list of per-study counts must line up with the studies given.
-    if num_records and "," in num_records:
-        n_counts = len(num_records.split(","))
-        n_studies = len(studies.split(","))
-        if n_counts != n_studies:
-            typer.secho(
-                f"--num-records has {n_counts} values but {n_studies} studies "
-                f"were given (pass one count, or one per study).",
-                fg=typer.colors.RED,
-                err=True,
-            )
-            raise typer.Exit(1)
+    _check_per_study_counts(studies, num_records)
 
     ver = version or e.dictionary_version
     schema_path = schema or str(SCHEMA_DIR / dictionary_filename(e, ver))

{gen3_dataops_toolkit-3.5.0 → gen3_dataops_toolkit-3.6.0}/src/g3dt/config.py

@@ -428,8 +428,24 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
     scripts compose a URL keeps Python the single source of truth: the scripts
     read ``G3DT_DICT_URL``/``G3DT_DICT_FILENAME`` and never build either
     themselves, so there is only one implementation to keep correct.
+
+    The interpreter's own bin directory is prepended to PATH: console scripts
+    installed next to g3dt — gen3-metadata-simulator via
+    ``g3dt synth install-simulator`` — land there, but a pipx install exposes
+    only g3dt's own entry points on the caller's PATH, so without this the
+    wrapped scripts' ``command -v gen3-metadata-simulator`` check fails even
+    though the tool is installed.
     """
+    import sys
+
     env = dict(os.environ)
+    # No .resolve(): a venv's python is a symlink to the base interpreter, and
+    # resolving it would point at the base install's bin instead of the venv
+    # bin where console scripts are actually created.
+    venv_bin = str(Path(sys.executable).parent)
+    path_entries = env.get("PATH", "").split(os.pathsep)
+    if venv_bin not in path_entries:
+        env["PATH"] = venv_bin + os.pathsep + env.get("PATH", "")
     values = {
         "G3DT_ENV": e.name,
         "G3DT_REGION": e.region,

{gen3_dataops_toolkit-3.5.0 → gen3_dataops_toolkit-3.6.0}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh

@@ -43,10 +43,24 @@ ARGO_SCRIPT_DIR="${SERVICE_DIR}/k8s_ops"
 SCHEMA_DIR="${G3DT_SCHEMA_DIR:-$HOME/.g3dt/schemas}"
 SYNTH_BASE="${G3DT_SYNTH_DIR:-$HOME/.g3dt/synth_metadata}"
 
+# Batch inputs (set by `g3dt synth deploy --studies/--num-records/--prev-version`;
+# the fallbacks are the original ACDC demo values, kept for continuity).
+STUDIES="${G3DT_SYNTH_STUDIES:-AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated}"
+if [ -n "${G3DT_SYNTH_NUM_RECORDS:-}" ]; then
+    NUM_RECORDS="${G3DT_SYNTH_NUM_RECORDS}"
+elif [ -n "${G3DT_SYNTH_STUDIES:-}" ]; then
+    NUM_RECORDS="30"          # custom studies, no counts given: 30 per study
+else
+    NUM_RECORDS="30,60,20,55" # the classic per-study counts for the demo set
+fi
+PREV_VERSION="${G3DT_SYNTH_PREV_VERSION:-v1.0.0}"
+
 # Derived variables
-PREV_VERSION="v1.0.0"
 SYNTH_META_DIR="${SYNTH_BASE}/${VERSION}/"
-DATA_IMPORT_ORDER_FILE="${SYNTH_BASE}/${PREV_VERSION}/AusDiab_Simulated/DataImportOrder.txt"
+# The delete step walks the previous batch's import order; every study in a
+# batch shares one, so the first study's copy serves.
+FIRST_STUDY="${STUDIES%%,*}"
+DATA_IMPORT_ORDER_FILE="${SYNTH_BASE}/${PREV_VERSION}/${FIRST_STUDY}/DataImportOrder.txt"
 
 # Never export an empty AWS_PROFILE (empty means ambient credentials).
 if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
@@ -87,9 +101,9 @@ bash "${ARGO_SCRIPT_DIR}/argocd_restart_schema.sh" \
     -a "${APP_NAME}" \
     -n "${NAMESPACE}"
 
-echo "==== [4] Deleting old synthetic data for version ${PREV_VERSION} ===="
+echo "==== [4] Deleting old synthetic data (${STUDIES}) for version ${PREV_VERSION} ===="
 DELETE_SYNTH_ARGS=(
-    -p "AusDiab_Simulated,EDCAD-PMS_Simulated,PREDICT_Simulated,Baker-Biobank_Simulated,CAUGHT-CAD_Simulated,BioHeart-CT_Simulated"
+    -p "${STUDIES}"
     -s "${AWS_SECRET_NAME}"
     -i "${DATA_IMPORT_ORDER_FILE}"
 )
@@ -98,12 +112,13 @@ if [ -n "${G3DT_AWS_PROFILE:-}" ]; then
 fi
 python3 "${SERVICE_DIR}/synthetic_data/delete_synth_metadata_sheepdog.py" "${DELETE_SYNTH_ARGS[@]}"
 
-echo "==== [5] Generating new synthetic data for version ${VERSION} (LLM-realistic) ===="
+echo "==== [5] Generating new synthetic data (${STUDIES}) for version ${VERSION} (LLM-realistic) ===="
 bash "${SERVICE_DIR}/synthetic_data/generate_synth_metadata.sh" \
     --schema "${SCHEMA_DIR}/${DICT_FILENAME}" \
     --version "${VERSION}" \
     --provider llm \
-    --num-records "30,60,20,55" \
+    --studies "${STUDIES}" \
+    --num-records "${NUM_RECORDS}" \
     --output-root "${SYNTH_BASE}"
 
 echo "==== [6] Uploading new synthetic data for version ${VERSION} ===="