gen3-dataops-toolkit 3.3.0__tar.gz → 3.4.0__tar.gz

{gen3_dataops_toolkit-3.3.0 → gen3_dataops_toolkit-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.3.0
+Version: 3.4.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -182,6 +182,32 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
 refuses a batch that doesn't match the version being uploaded (override with
 `--allow-version-mismatch`).
 
+### Synthetic data: LLM configuration
+
+`synth generate --llm` and `synth deploy` generate LLM-realistic values with
+[gen3-metadata-simulator](https://github.com/AustralianBioCommons/gen3-metadata-simulator).
+The provider and model resolve with precedence **CLI flags > SSM > default**:
+the CDK config's optional `llm` block publishes `app/llm_provider` /
+`app/llm_model` to SSM, so every operator gets the deployment's values, and
+`--llm-provider` / `--llm-model` override them for one run (e.g. to try a
+model before adding it to the CDK config). Environments deployed without the
+block fall back to provider `anthropic`, and the `--llm` path errors with
+guidance when no model is configured anywhere.
+
+Only the API key stays local — as a *path* to the file holding it, never the
+key itself, set once per operator:
+
+```bash
+g3dt config set llm_api_key_file ~/.g3dt/anthropic_api_key
+g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
+```
+
+(or per run with `--llm-api-key-file`; the vendor env var `ANTHROPIC_API_KEY`
+/ `OPENAI_API_KEY` also works as a fallback.) The old `~/.g3dt/.env`
+(`LLM_PROVIDER`/`LLM_MODEL`/`LLM_API_KEY_FILE`) is **no longer read**.
+`g3dt config show --env <env>` prints the resolved provider, model, and key
+path.
+
 ## Verifying download access (check-download)
 
 Registration alone does not prove a file can be downloaded. Two failure modes

{gen3_dataops_toolkit-3.3.0 → gen3_dataops_toolkit-3.4.0}/README.md

@@ -148,6 +148,32 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
 refuses a batch that doesn't match the version being uploaded (override with
 `--allow-version-mismatch`).
 
+### Synthetic data: LLM configuration
+
+`synth generate --llm` and `synth deploy` generate LLM-realistic values with
+[gen3-metadata-simulator](https://github.com/AustralianBioCommons/gen3-metadata-simulator).
+The provider and model resolve with precedence **CLI flags > SSM > default**:
+the CDK config's optional `llm` block publishes `app/llm_provider` /
+`app/llm_model` to SSM, so every operator gets the deployment's values, and
+`--llm-provider` / `--llm-model` override them for one run (e.g. to try a
+model before adding it to the CDK config). Environments deployed without the
+block fall back to provider `anthropic`, and the `--llm` path errors with
+guidance when no model is configured anywhere.
+
+Only the API key stays local — as a *path* to the file holding it, never the
+key itself, set once per operator:
+
+```bash
+g3dt config set llm_api_key_file ~/.g3dt/anthropic_api_key
+g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
+```
+
+(or per run with `--llm-api-key-file`; the vendor env var `ANTHROPIC_API_KEY`
+/ `OPENAI_API_KEY` also works as a fallback.) The old `~/.g3dt/.env`
+(`LLM_PROVIDER`/`LLM_MODEL`/`LLM_API_KEY_FILE`) is **no longer read**.
+`g3dt config show --env <env>` prints the resolved provider, model, and key
+path.
+
 ## Verifying download access (check-download)
 
 Registration alone does not prove a file can be downloaded. Two failure modes

{gen3_dataops_toolkit-3.3.0 → gen3_dataops_toolkit-3.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.3.0"
+version = "3.4.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-3.3.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/cli/config_cmds.py

@@ -118,6 +118,11 @@ def show(
     typer.echo(f"  namespace          : {e.namespace}")
     typer.echo(f"  cluster_name       : {e.cluster_name}")
     typer.echo(f"  ec2_instance_id    : {e.ec2_instance_id}")
+    # Synthetic-data LLM facts: provider/model from SSM (the CDK's optional
+    # llm block); only the key *path* is local, from the marker.
+    typer.echo(f"  llm_provider       : {e.llm_provider}")
+    typer.echo(f"  llm_model          : {e.llm_model or '(not set — pass --llm-model or add the llm block to the CDK config)'}")
+    typer.echo(f"  llm_api_key_file   : {config.llm_api_key_file() or '(not set — g3dt config set llm_api_key_file <path>)'}")
     if study:
         s = study_of(study, env)
         typer.secho(f"Study: {study} -> {s.key}", bold=True)
@@ -199,6 +204,20 @@ def diff(
         check(f"gen3.{camel}", gen3.get(camel), rc.get(f"app/{leaf}"))
     check("toolkitVersion", inputs.get("toolkitVersion"), rc.get("meta/toolkitVersion"))
 
+    # Optional inputs: compared only when the file defines them — an absent
+    # input legitimately publishes no SSM parameter (or, for the dictionary
+    # fields, predates their addition), so absence on both sides is not drift.
+    for camel, leaf in {
+        "dictionaryBaseUrl": "dictionary_base_url",
+        "dictionaryPath": "dictionary_path",
+    }.items():
+        if camel in gen3:
+            check(f"gen3.{camel}", gen3.get(camel), rc.get(f"app/{leaf}"))
+    llm = inputs.get("llm") or {}
+    for camel, leaf in {"provider": "llm_provider", "model": "llm_model"}.items():
+        if camel in llm:
+            check(f"llm.{camel}", llm.get(camel), rc.get(f"app/{leaf}"))
+
     if not drift:
         typer.secho(
             f"No drift: SSM /{project}/{config.env_base(env)} matches {file}.",

{gen3_dataops_toolkit-3.3.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/cli/synth.py

@@ -9,9 +9,14 @@ Every command accepts ``--env``; targeting a **production** environment (any env
 whose name contains ``prod``) shows a warning and requires typing the env name to
 confirm — it cannot be bypassed.
 
-Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm`` for
-LLM-realistic values, which needs an API key configured in a ``.env`` in the
-working directory (``LLM_PROVIDER`` / ``LLM_MODEL`` / ``LLM_API_KEY_FILE``).
+Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm``
+for LLM-realistic values. The LLM provider and model come from the
+environment's SSM tree (the CDK config's optional ``llm`` block, published as
+``app/llm_provider`` / ``app/llm_model``) and can be overridden per run with
+``--llm-provider`` / ``--llm-model``. Only the API key stays local: point at
+its file once with ``g3dt config set llm_api_key_file <path>`` (or per run
+with ``--llm-api-key-file``); the vendor env var ``ANTHROPIC_API_KEY`` /
+``OPENAI_API_KEY`` also works. The old ``~/.g3dt/.env`` is no longer read.
 """
 from __future__ import annotations
 
@@ -21,6 +26,7 @@ from typing import Optional
 
 import typer
 
+from g3dt import config
 from g3dt.config import (
     dictionary_filename,
     dictionary_url,
@@ -38,6 +44,45 @@ app = typer.Typer(
 
 SYNTH_DIR = Path("~/.g3dt/synth_metadata").expanduser()
 
+
+def _llm_env_overrides(
+    e,
+    llm_provider: Optional[str],
+    llm_model: Optional[str],
+    llm_api_key_file: Optional[Path],
+) -> dict:
+    """Resolve the effective LLM settings for a run: flags > SSM > default.
+
+    Returns env-var overrides for the generator script, which forwards
+    provider/model to gen3-metadata-simulator as CLI flags (the simulator's
+    own precedence puts flags first). Exits with guidance when no model is
+    configured anywhere; the key-file path is optional — the simulator falls
+    back to the vendor env var and raises its own error if neither exists.
+    """
+    effective_provider = llm_provider or e.llm_provider
+    effective_model = llm_model or e.llm_model
+    if not effective_model:
+        typer.secho(
+            "No LLM model configured. Set the llm block in the CDK config "
+            "(published to SSM as app/llm_model) and redeploy, or pass "
+            "--llm-model for this run.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(1)
+    overrides = {
+        "G3DT_LLM_PROVIDER": effective_provider,
+        "G3DT_LLM_MODEL": effective_model,
+    }
+    key_file = (
+        str(Path(llm_api_key_file).expanduser())
+        if llm_api_key_file
+        else config.llm_api_key_file()
+    )
+    if key_file:
+        overrides["LLM_API_KEY_FILE"] = key_file
+    return overrides
+
 #: Written into each generated batch so `synth upload` can tell which dictionary
 #: produced it. A batch is only valid against that dictionary, and the directory
 #: name alone cannot be trusted: --schema and --version are separate options, so
@@ -110,19 +155,35 @@ def deploy(
     env: str = typer.Option(
         "test", "--env", "-e", help="Target environment (prod requires typed confirmation)."
     ),
+    llm_provider: Optional[str] = typer.Option(
+        None, "--llm-provider",
+        help="LLM vendor override (anthropic|openai); default: the env's SSM app/llm_provider.",
+    ),
+    llm_model: Optional[str] = typer.Option(
+        None, "--llm-model",
+        help="LLM model override; default: the env's SSM app/llm_model.",
+    ),
+    llm_api_key_file: Optional[Path] = typer.Option(
+        None, "--llm-api-key-file", exists=True, dir_okay=False,
+        help="Path to the file holding the LLM API key; default: the marker's "
+        "llm_api_key_file (set once: g3dt config set llm_api_key_file <path>).",
+    ),
 ) -> None:
     """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts).
 
     Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed
-    generation). Requires an LLM key configured in .env.
+    generation). Provider/model come from the env's SSM tree unless
+    overridden; the API key path comes from --llm-api-key-file or the marker.
     """
     e = env_of(env)
     safety.confirm_prod_strict("synthetic full deploy", env)
+    env_vars = script_env(e)
+    env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
     runner.run(
         runner.bash_script(
             "services/synthetic_data/full_deploy_dd_and_synth.sh", env
         ),
-        env=script_env(e),
+        env=env_vars,
     )
 
 
@@ -147,8 +208,22 @@ def generate(
     llm: bool = typer.Option(
         False,
         "--llm",
-        help="Generate LLM-realistic values; reads LLM config from a .env in the "
-        "working directory. Default is keyless random data (no API key, no API calls).",
+        help="Generate LLM-realistic values; provider/model resolve from the "
+        "env's SSM tree (override with --llm-provider/--llm-model). Default is "
+        "keyless random data (no API key, no API calls).",
+    ),
+    llm_provider: Optional[str] = typer.Option(
+        None, "--llm-provider",
+        help="LLM vendor override (anthropic|openai); default: the env's SSM app/llm_provider.",
+    ),
+    llm_model: Optional[str] = typer.Option(
+        None, "--llm-model",
+        help="LLM model override; default: the env's SSM app/llm_model.",
+    ),
+    llm_api_key_file: Optional[Path] = typer.Option(
+        None, "--llm-api-key-file", exists=True, dir_okay=False,
+        help="Path to the file holding the LLM API key; default: the marker's "
+        "llm_api_key_file (set once: g3dt config set llm_api_key_file <path>).",
     ),
     seed: int = typer.Option(None, "--seed", help="RNG seed for reproducible output."),
     schema: str = typer.Option(
@@ -227,11 +302,16 @@ def generate(
         args += ["--num-records", num_records]
     if seed is not None:
         args += ["--seed", str(seed)]
+    env_vars = script_env(e, ver)
+    if effective_provider == "llm":
+        env_vars.update(
+            _llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file)
+        )
     runner.run(
         runner.bash_script(
             "services/synthetic_data/generate_synth_metadata.sh", *args
         ),
-        env=script_env(e, ver),
+        env=env_vars,
     )
     # Only after a successful generate: runner.run raises on failure, so a
     # half-written batch is never stamped as valid.

{gen3_dataops_toolkit-3.3.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/config.py

@@ -50,11 +50,21 @@ DEFAULT_REGION = "ap-southeast-2"
 DEFAULT_DICT_BASE_URL = "https://raw.githubusercontent.com"
 DEFAULT_DICT_PATH = "dictionary/prod_dict/acdc_schema.json"
 
+#: Synthetic-data LLM facts, published by the CDK's OPTIONAL ``llm`` config
+#: block as ``app/llm_provider`` / ``app/llm_model`` and consumed by
+#: gen3-metadata-simulator through ``g3dt synth``. Optional app inputs:
+#: environments deployed without the block fall back to this provider default.
+#: The model deliberately has no default — ``g3dt synth`` errors with guidance
+#: when the ``--llm`` path is used and no model is configured anywhere.
+DEFAULT_LLM_PROVIDER = "anthropic"
+
 #: Marker locations, most specific first.
 MARKER_PATHS = ("g3dt.yaml", "~/.g3dt/g3dt.yaml", "/etc/g3dt/g3dt.yaml")
 
-#: Marker keys `g3dt config set` may write.
-SETTABLE_MARKER_KEYS = ("project", "region", "default_env")
+#: Marker keys `g3dt config set` may write. ``llm_api_key_file`` is the path
+#: to the file holding the synth LLM API key — the one LLM setting that stays
+#: local (the provider and model come from SSM; the key never leaves the box).
+SETTABLE_MARKER_KEYS = ("project", "region", "default_env", "llm_api_key_file")
 
 #: Gen3 app facts mirrored to SSM /{project}/{env}/app/* by the CDK.
 REQUIRED_APP_KEYS = (
@@ -150,6 +160,18 @@ def require_project(marker: Optional[dict] = None) -> str:
     return project
 
 
+def llm_api_key_file(marker: Optional[dict] = None) -> Optional[str]:
+    """Path to the file holding the synth LLM API key, from the marker.
+
+    Set once per operator with ``g3dt config set llm_api_key_file <path>``.
+    Returns ``None`` when unset — gen3-metadata-simulator then falls back to
+    the vendor's standard env var (``ANTHROPIC_API_KEY`` / ``OPENAI_API_KEY``).
+    """
+    m = marker if marker is not None else load_marker()
+    value = m.get("llm_api_key_file")
+    return str(Path(str(value)).expanduser()) if value else None
+
+
 def set_marker_value(key: str, value: str) -> Tuple[Optional[str], str, Path]:
     """Set one bootstrap key in the user's marker file and write it back.
 
@@ -230,6 +252,11 @@ class EnvConfig:
     # hand-built EnvConfig still composes a valid URL.
     dictionary_base_url: str = DEFAULT_DICT_BASE_URL
     dictionary_path: str = DEFAULT_DICT_PATH
+    # Optional synthetic-data LLM inputs (SSM app/llm_provider, app/llm_model).
+    # The model has no default on purpose: synth's --llm path checks and errors
+    # with guidance rather than silently picking a model.
+    llm_provider: str = DEFAULT_LLM_PROVIDER
+    llm_model: Optional[str] = None
 
 
 def _app_or_default(rc, leaf: str, default: str) -> str:
@@ -289,6 +316,10 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
             rc, "dictionary_base_url", DEFAULT_DICT_BASE_URL
         ),
         dictionary_path=_app_or_default(rc, "dictionary_path", DEFAULT_DICT_PATH),
+        # Same optional-app-fact contract: the CDK publishes these only when
+        # the config has an llm block, so absence means "use the defaults".
+        llm_provider=_app_or_default(rc, "llm_provider", DEFAULT_LLM_PROVIDER),
+        llm_model=(_app_or_default(rc, "llm_model", "") or None),
     )
 
 
@@ -395,6 +426,8 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
         "G3DT_NAMESPACE": e.namespace,
         "G3DT_CLUSTER_NAME": e.cluster_name,
         "G3DT_SCHEMA_REPO": e.schema_repo,
+        "G3DT_LLM_PROVIDER": e.llm_provider,
+        "G3DT_LLM_MODEL": e.llm_model,
     }
     env.update({k: v for k, v in values.items() if v is not None})
     return env

{gen3_dataops_toolkit-3.3.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh

@@ -8,14 +8,13 @@
 # The tool takes a LOCAL bundled Gen3 schema file (pulled by pull_dict.sh into
 # ~/.g3dt/schemas/acdc_schema_<version>.json, or $G3DT_SCHEMA_DIR if set). The
 # default provider is keyless 'random'; pass --provider llm for LLM-realistic
-# values, in which case LLM config is read from ~/.g3dt/.env (or $G3DT_ENV_FILE
-# if set): LLM_PROVIDER / LLM_MODEL / LLM_API_KEY_FILE.
+# values. The LLM vendor/model arrive as $G3DT_LLM_PROVIDER / $G3DT_LLM_MODEL
+# (resolved by g3dt: CLI flags > SSM app/llm_* > default) and are forwarded to
+# the simulator as flags; the API key file path arrives as $LLM_API_KEY_FILE.
+# The old ~/.g3dt/.env is no longer read.
 
 set -euo pipefail
 
-# LLM provider config file (lives outside the installed package).
-ENV_FILE="${G3DT_ENV_FILE:-$HOME/.g3dt/.env}"
-
 usage() {
     cat <<EOF
 Usage: $(basename "$0") --schema <path> --version <ver> [options]
@@ -32,8 +31,8 @@ Options:
   --num-records N|n1,n2  Records per study: one number for all, or a comma list
                          (one per study). Default: ${DEFAULT_NUM_RECORDS}
   --provider random|llm  Value strategy. Default: ${DEFAULT_PROVIDER}
-                         'random' needs no key; 'llm' reads LLM config from
-                         ${ENV_FILE}.
+                         'random' needs no key; 'llm' uses \$G3DT_LLM_PROVIDER /
+                         \$G3DT_LLM_MODEL / \$LLM_API_KEY_FILE (set by g3dt).
   --seed N               RNG seed for reproducible output.
   --output-root DIR      Root output dir. Default: ${DEFAULT_OUTPUT_ROOT}
   -h, --help             Show this help and exit.
@@ -123,9 +122,16 @@ for i in "${!STUDY_ARRAY[@]}"; do
          --num-records "$N"
          --provider "$PROVIDER")
     [[ -n "$SEED" ]] && CMD+=(--seed "$SEED")
-    # Point the LLM provider at the user-level env file regardless of the caller's CWD.
-    if [[ "$PROVIDER" == "llm" && -f "${ENV_FILE}" ]]; then
-        CMD+=(--env-file "${ENV_FILE}")
+    if [[ "$PROVIDER" == "llm" ]]; then
+        # Vendor/model resolved by g3dt (flags > SSM > default) and forwarded
+        # as simulator flags, because the simulator's own precedence puts
+        # flags above any .env or environment variable.
+        [[ -n "${G3DT_LLM_PROVIDER:-}" ]] && CMD+=(--llm-provider "$G3DT_LLM_PROVIDER")
+        [[ -n "${G3DT_LLM_MODEL:-}" ]] && CMD+=(--llm-model "$G3DT_LLM_MODEL")
+        # Neutralize any .env in the caller's CWD: /dev/null exists (satisfies
+        # the simulator's exists=True check) and dotenv-parses to empty, so
+        # only the flags above and the inherited $LLM_API_KEY_FILE apply.
+        CMD+=(--env-file /dev/null)
     fi
     "${CMD[@]}"
 done