gen3-dataops-toolkit 3.2.0__tar.gz → 3.4.0__tar.gz

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.2.0
+Version: 3.4.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -17,7 +17,7 @@ Requires-Dist: awswrangler (>=3.14.0,<4.0.0)
 Requires-Dist: boto3
 Requires-Dist: gen3 (>=4.27.4,<5.0.0)
 Requires-Dist: gen3-metadata (>=1.4.0,<2.0.0)
-Requires-Dist: gen3_validator (>=2.0.0,<3.0.0)
+Requires-Dist: gen3_validator (>=2.3.0,<3.0.0)
 Requires-Dist: numpy (<2.0.0)
 Requires-Dist: openpyxl (>=3.1.0)
 Requires-Dist: pyarrow (>=14.0.0,<19.0.0)
@@ -182,6 +182,32 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
 refuses a batch that doesn't match the version being uploaded (override with
 `--allow-version-mismatch`).
 
+### Synthetic data: LLM configuration
+
+`synth generate --llm` and `synth deploy` generate LLM-realistic values with
+[gen3-metadata-simulator](https://github.com/AustralianBioCommons/gen3-metadata-simulator).
+The provider and model resolve with precedence **CLI flags > SSM > default**:
+the CDK config's optional `llm` block publishes `app/llm_provider` /
+`app/llm_model` to SSM, so every operator gets the deployment's values, and
+`--llm-provider` / `--llm-model` override them for one run (e.g. to try a
+model before adding it to the CDK config). Environments deployed without the
+block fall back to provider `anthropic`, and the `--llm` path errors with
+guidance when no model is configured anywhere.
+
+Only the API key stays local — as a *path* to the file holding it, never the
+key itself, set once per operator:
+
+```bash
+g3dt config set llm_api_key_file ~/.g3dt/anthropic_api_key
+g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
+```
+
+(or per run with `--llm-api-key-file`; the vendor env var `ANTHROPIC_API_KEY`
+/ `OPENAI_API_KEY` also works as a fallback.) The old `~/.g3dt/.env`
+(`LLM_PROVIDER`/`LLM_MODEL`/`LLM_API_KEY_FILE`) is **no longer read**.
+`g3dt config show --env <env>` prints the resolved provider, model, and key
+path.
+
 ## Verifying download access (check-download)
 
 Registration alone does not prove a file can be downloaded. Two failure modes

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/README.md

@@ -148,6 +148,32 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
 refuses a batch that doesn't match the version being uploaded (override with
 `--allow-version-mismatch`).
 
+### Synthetic data: LLM configuration
+
+`synth generate --llm` and `synth deploy` generate LLM-realistic values with
+[gen3-metadata-simulator](https://github.com/AustralianBioCommons/gen3-metadata-simulator).
+The provider and model resolve with precedence **CLI flags > SSM > default**:
+the CDK config's optional `llm` block publishes `app/llm_provider` /
+`app/llm_model` to SSM, so every operator gets the deployment's values, and
+`--llm-provider` / `--llm-model` override them for one run (e.g. to try a
+model before adding it to the CDK config). Environments deployed without the
+block fall back to provider `anthropic`, and the `--llm` path errors with
+guidance when no model is configured anywhere.
+
+Only the API key stays local — as a *path* to the file holding it, never the
+key itself, set once per operator:
+
+```bash
+g3dt config set llm_api_key_file ~/.g3dt/anthropic_api_key
+g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
+```
+
+(or per run with `--llm-api-key-file`; the vendor env var `ANTHROPIC_API_KEY`
+/ `OPENAI_API_KEY` also works as a fallback.) The old `~/.g3dt/.env`
+(`LLM_PROVIDER`/`LLM_MODEL`/`LLM_API_KEY_FILE`) is **no longer read**.
+`g3dt config show --env <env>` prints the resolved provider, model, and key
+path.
+
 ## Verifying download access (check-download)
 
 Registration alone does not prove a file can be downloaded. Two failure modes

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.2.0"
+version = "3.4.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"
@@ -19,7 +19,7 @@ numpy = "<2.0.0"
 pyyaml = ">=6.0.2,<7.0.0"
 tzlocal = ">=5.3.1,<6.0.0"
 gen3 = ">=4.27.4, <5.0.0"
-gen3_validator = ">=2.0.0,<3.0.0"
+gen3_validator = ">=2.3.0,<3.0.0"
 pytz = ">=2025.2,<2026.0"
 s3fs = "2025.10.0"
 pyjwt = ">=2.10.1,<3.0.0"

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/cli/config_cmds.py

@@ -118,6 +118,11 @@ def show(
     typer.echo(f"  namespace          : {e.namespace}")
     typer.echo(f"  cluster_name       : {e.cluster_name}")
     typer.echo(f"  ec2_instance_id    : {e.ec2_instance_id}")
+    # Synthetic-data LLM facts: provider/model from SSM (the CDK's optional
+    # llm block); only the key *path* is local, from the marker.
+    typer.echo(f"  llm_provider       : {e.llm_provider}")
+    typer.echo(f"  llm_model          : {e.llm_model or '(not set — pass --llm-model or add the llm block to the CDK config)'}")
+    typer.echo(f"  llm_api_key_file   : {config.llm_api_key_file() or '(not set — g3dt config set llm_api_key_file <path>)'}")
     if study:
         s = study_of(study, env)
         typer.secho(f"Study: {study} -> {s.key}", bold=True)
@@ -199,6 +204,20 @@ def diff(
         check(f"gen3.{camel}", gen3.get(camel), rc.get(f"app/{leaf}"))
     check("toolkitVersion", inputs.get("toolkitVersion"), rc.get("meta/toolkitVersion"))
 
+    # Optional inputs: compared only when the file defines them — an absent
+    # input legitimately publishes no SSM parameter (or, for the dictionary
+    # fields, predates their addition), so absence on both sides is not drift.
+    for camel, leaf in {
+        "dictionaryBaseUrl": "dictionary_base_url",
+        "dictionaryPath": "dictionary_path",
+    }.items():
+        if camel in gen3:
+            check(f"gen3.{camel}", gen3.get(camel), rc.get(f"app/{leaf}"))
+    llm = inputs.get("llm") or {}
+    for camel, leaf in {"provider": "llm_provider", "model": "llm_model"}.items():
+        if camel in llm:
+            check(f"llm.{camel}", llm.get(camel), rc.get(f"app/{leaf}"))
+
     if not drift:
         typer.secho(
             f"No drift: SSM /{project}/{config.env_base(env)} matches {file}.",

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/cli/synth.py

@@ -9,9 +9,14 @@ Every command accepts ``--env``; targeting a **production** environment (any env
 whose name contains ``prod``) shows a warning and requires typing the env name to
 confirm — it cannot be bypassed.
 
-Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm`` for
-LLM-realistic values, which needs an API key configured in a ``.env`` in the
-working directory (``LLM_PROVIDER`` / ``LLM_MODEL`` / ``LLM_API_KEY_FILE``).
+Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm``
+for LLM-realistic values. The LLM provider and model come from the
+environment's SSM tree (the CDK config's optional ``llm`` block, published as
+``app/llm_provider`` / ``app/llm_model``) and can be overridden per run with
+``--llm-provider`` / ``--llm-model``. Only the API key stays local: point at
+its file once with ``g3dt config set llm_api_key_file <path>`` (or per run
+with ``--llm-api-key-file``); the vendor env var ``ANTHROPIC_API_KEY`` /
+``OPENAI_API_KEY`` also works. The old ``~/.g3dt/.env`` is no longer read.
 """
 from __future__ import annotations
 
@@ -21,6 +26,7 @@ from typing import Optional
 
 import typer
 
+from g3dt import config
 from g3dt.config import (
     dictionary_filename,
     dictionary_url,
@@ -38,6 +44,45 @@ app = typer.Typer(
 
 SYNTH_DIR = Path("~/.g3dt/synth_metadata").expanduser()
 
+
+def _llm_env_overrides(
+    e,
+    llm_provider: Optional[str],
+    llm_model: Optional[str],
+    llm_api_key_file: Optional[Path],
+) -> dict:
+    """Resolve the effective LLM settings for a run: flags > SSM > default.
+
+    Returns env-var overrides for the generator script, which forwards
+    provider/model to gen3-metadata-simulator as CLI flags (the simulator's
+    own precedence puts flags first). Exits with guidance when no model is
+    configured anywhere; the key-file path is optional — the simulator falls
+    back to the vendor env var and raises its own error if neither exists.
+    """
+    effective_provider = llm_provider or e.llm_provider
+    effective_model = llm_model or e.llm_model
+    if not effective_model:
+        typer.secho(
+            "No LLM model configured. Set the llm block in the CDK config "
+            "(published to SSM as app/llm_model) and redeploy, or pass "
+            "--llm-model for this run.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(1)
+    overrides = {
+        "G3DT_LLM_PROVIDER": effective_provider,
+        "G3DT_LLM_MODEL": effective_model,
+    }
+    key_file = (
+        str(Path(llm_api_key_file).expanduser())
+        if llm_api_key_file
+        else config.llm_api_key_file()
+    )
+    if key_file:
+        overrides["LLM_API_KEY_FILE"] = key_file
+    return overrides
+
 #: Written into each generated batch so `synth upload` can tell which dictionary
 #: produced it. A batch is only valid against that dictionary, and the directory
 #: name alone cannot be trusted: --schema and --version are separate options, so
@@ -110,19 +155,35 @@ def deploy(
     env: str = typer.Option(
         "test", "--env", "-e", help="Target environment (prod requires typed confirmation)."
     ),
+    llm_provider: Optional[str] = typer.Option(
+        None, "--llm-provider",
+        help="LLM vendor override (anthropic|openai); default: the env's SSM app/llm_provider.",
+    ),
+    llm_model: Optional[str] = typer.Option(
+        None, "--llm-model",
+        help="LLM model override; default: the env's SSM app/llm_model.",
+    ),
+    llm_api_key_file: Optional[Path] = typer.Option(
+        None, "--llm-api-key-file", exists=True, dir_okay=False,
+        help="Path to the file holding the LLM API key; default: the marker's "
+        "llm_api_key_file (set once: g3dt config set llm_api_key_file <path>).",
+    ),
 ) -> None:
     """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts).
 
     Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed
-    generation). Requires an LLM key configured in .env.
+    generation). Provider/model come from the env's SSM tree unless
+    overridden; the API key path comes from --llm-api-key-file or the marker.
     """
     e = env_of(env)
     safety.confirm_prod_strict("synthetic full deploy", env)
+    env_vars = script_env(e)
+    env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
     runner.run(
         runner.bash_script(
             "services/synthetic_data/full_deploy_dd_and_synth.sh", env
         ),
-        env=script_env(e),
+        env=env_vars,
     )
 
 
@@ -147,8 +208,22 @@ def generate(
     llm: bool = typer.Option(
         False,
         "--llm",
-        help="Generate LLM-realistic values; reads LLM config from a .env in the "
-        "working directory. Default is keyless random data (no API key, no API calls).",
+        help="Generate LLM-realistic values; provider/model resolve from the "
+        "env's SSM tree (override with --llm-provider/--llm-model). Default is "
+        "keyless random data (no API key, no API calls).",
+    ),
+    llm_provider: Optional[str] = typer.Option(
+        None, "--llm-provider",
+        help="LLM vendor override (anthropic|openai); default: the env's SSM app/llm_provider.",
+    ),
+    llm_model: Optional[str] = typer.Option(
+        None, "--llm-model",
+        help="LLM model override; default: the env's SSM app/llm_model.",
+    ),
+    llm_api_key_file: Optional[Path] = typer.Option(
+        None, "--llm-api-key-file", exists=True, dir_okay=False,
+        help="Path to the file holding the LLM API key; default: the marker's "
+        "llm_api_key_file (set once: g3dt config set llm_api_key_file <path>).",
     ),
     seed: int = typer.Option(None, "--seed", help="RNG seed for reproducible output."),
     schema: str = typer.Option(
@@ -227,11 +302,16 @@ def generate(
         args += ["--num-records", num_records]
     if seed is not None:
         args += ["--seed", str(seed)]
+    env_vars = script_env(e, ver)
+    if effective_provider == "llm":
+        env_vars.update(
+            _llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file)
+        )
     runner.run(
         runner.bash_script(
             "services/synthetic_data/generate_synth_metadata.sh", *args
         ),
-        env=script_env(e, ver),
+        env=env_vars,
     )
     # Only after a successful generate: runner.run raises on failure, so a
     # half-written batch is never stamped as valid.

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/config.py

@@ -50,11 +50,21 @@ DEFAULT_REGION = "ap-southeast-2"
 DEFAULT_DICT_BASE_URL = "https://raw.githubusercontent.com"
 DEFAULT_DICT_PATH = "dictionary/prod_dict/acdc_schema.json"
 
+#: Synthetic-data LLM facts, published by the CDK's OPTIONAL ``llm`` config
+#: block as ``app/llm_provider`` / ``app/llm_model`` and consumed by
+#: gen3-metadata-simulator through ``g3dt synth``. Optional app inputs:
+#: environments deployed without the block fall back to this provider default.
+#: The model deliberately has no default — ``g3dt synth`` errors with guidance
+#: when the ``--llm`` path is used and no model is configured anywhere.
+DEFAULT_LLM_PROVIDER = "anthropic"
+
 #: Marker locations, most specific first.
 MARKER_PATHS = ("g3dt.yaml", "~/.g3dt/g3dt.yaml", "/etc/g3dt/g3dt.yaml")
 
-#: Marker keys `g3dt config set` may write.
-SETTABLE_MARKER_KEYS = ("project", "region", "default_env")
+#: Marker keys `g3dt config set` may write. ``llm_api_key_file`` is the path
+#: to the file holding the synth LLM API key — the one LLM setting that stays
+#: local (the provider and model come from SSM; the key never leaves the box).
+SETTABLE_MARKER_KEYS = ("project", "region", "default_env", "llm_api_key_file")
 
 #: Gen3 app facts mirrored to SSM /{project}/{env}/app/* by the CDK.
 REQUIRED_APP_KEYS = (
@@ -150,6 +160,18 @@ def require_project(marker: Optional[dict] = None) -> str:
     return project
 
 
+def llm_api_key_file(marker: Optional[dict] = None) -> Optional[str]:
+    """Path to the file holding the synth LLM API key, from the marker.
+
+    Set once per operator with ``g3dt config set llm_api_key_file <path>``.
+    Returns ``None`` when unset — gen3-metadata-simulator then falls back to
+    the vendor's standard env var (``ANTHROPIC_API_KEY`` / ``OPENAI_API_KEY``).
+    """
+    m = marker if marker is not None else load_marker()
+    value = m.get("llm_api_key_file")
+    return str(Path(str(value)).expanduser()) if value else None
+
+
 def set_marker_value(key: str, value: str) -> Tuple[Optional[str], str, Path]:
     """Set one bootstrap key in the user's marker file and write it back.
 
@@ -230,6 +252,11 @@ class EnvConfig:
     # hand-built EnvConfig still composes a valid URL.
     dictionary_base_url: str = DEFAULT_DICT_BASE_URL
     dictionary_path: str = DEFAULT_DICT_PATH
+    # Optional synthetic-data LLM inputs (SSM app/llm_provider, app/llm_model).
+    # The model has no default on purpose: synth's --llm path checks and errors
+    # with guidance rather than silently picking a model.
+    llm_provider: str = DEFAULT_LLM_PROVIDER
+    llm_model: Optional[str] = None
 
 
 def _app_or_default(rc, leaf: str, default: str) -> str:
@@ -289,6 +316,10 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
             rc, "dictionary_base_url", DEFAULT_DICT_BASE_URL
         ),
         dictionary_path=_app_or_default(rc, "dictionary_path", DEFAULT_DICT_PATH),
+        # Same optional-app-fact contract: the CDK publishes these only when
+        # the config has an llm block, so absence means "use the defaults".
+        llm_provider=_app_or_default(rc, "llm_provider", DEFAULT_LLM_PROVIDER),
+        llm_model=(_app_or_default(rc, "llm_model", "") or None),
     )
 
 
@@ -395,6 +426,8 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
         "G3DT_NAMESPACE": e.namespace,
         "G3DT_CLUSTER_NAME": e.cluster_name,
         "G3DT_SCHEMA_REPO": e.schema_repo,
+        "G3DT_LLM_PROVIDER": e.llm_provider,
+        "G3DT_LLM_MODEL": e.llm_model,
     }
     env.update({k: v for k, v in values.items() if v is not None})
     return env

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh

@@ -8,14 +8,13 @@
 # The tool takes a LOCAL bundled Gen3 schema file (pulled by pull_dict.sh into
 # ~/.g3dt/schemas/acdc_schema_<version>.json, or $G3DT_SCHEMA_DIR if set). The
 # default provider is keyless 'random'; pass --provider llm for LLM-realistic
-# values, in which case LLM config is read from ~/.g3dt/.env (or $G3DT_ENV_FILE
-# if set): LLM_PROVIDER / LLM_MODEL / LLM_API_KEY_FILE.
+# values. The LLM vendor/model arrive as $G3DT_LLM_PROVIDER / $G3DT_LLM_MODEL
+# (resolved by g3dt: CLI flags > SSM app/llm_* > default) and are forwarded to
+# the simulator as flags; the API key file path arrives as $LLM_API_KEY_FILE.
+# The old ~/.g3dt/.env is no longer read.
 
 set -euo pipefail
 
-# LLM provider config file (lives outside the installed package).
-ENV_FILE="${G3DT_ENV_FILE:-$HOME/.g3dt/.env}"
-
 usage() {
     cat <<EOF
 Usage: $(basename "$0") --schema <path> --version <ver> [options]
@@ -32,8 +31,8 @@ Options:
   --num-records N|n1,n2  Records per study: one number for all, or a comma list
                          (one per study). Default: ${DEFAULT_NUM_RECORDS}
   --provider random|llm  Value strategy. Default: ${DEFAULT_PROVIDER}
-                         'random' needs no key; 'llm' reads LLM config from
-                         ${ENV_FILE}.
+                         'random' needs no key; 'llm' uses \$G3DT_LLM_PROVIDER /
+                         \$G3DT_LLM_MODEL / \$LLM_API_KEY_FILE (set by g3dt).
   --seed N               RNG seed for reproducible output.
   --output-root DIR      Root output dir. Default: ${DEFAULT_OUTPUT_ROOT}
   -h, --help             Show this help and exit.
@@ -123,9 +122,16 @@ for i in "${!STUDY_ARRAY[@]}"; do
          --num-records "$N"
          --provider "$PROVIDER")
     [[ -n "$SEED" ]] && CMD+=(--seed "$SEED")
-    # Point the LLM provider at the user-level env file regardless of the caller's CWD.
-    if [[ "$PROVIDER" == "llm" && -f "${ENV_FILE}" ]]; then
-        CMD+=(--env-file "${ENV_FILE}")
+    if [[ "$PROVIDER" == "llm" ]]; then
+        # Vendor/model resolved by g3dt (flags > SSM > default) and forwarded
+        # as simulator flags, because the simulator's own precedence puts
+        # flags above any .env or environment variable.
+        [[ -n "${G3DT_LLM_PROVIDER:-}" ]] && CMD+=(--llm-provider "$G3DT_LLM_PROVIDER")
+        [[ -n "${G3DT_LLM_MODEL:-}" ]] && CMD+=(--llm-model "$G3DT_LLM_MODEL")
+        # Neutralize any .env in the caller's CWD: /dev/null exists (satisfies
+        # the simulator's exists=True check) and dotenv-parses to empty, so
+        # only the flags above and the inherited $LLM_API_KEY_FILE apply.
+        CMD+=(--env-file /dev/null)
     fi
     "${CMD[@]}"
 done

{gen3_dataops_toolkit-3.2.0 → gen3_dataops_toolkit-3.4.0}/src/g3dt/validate/validate.py

@@ -479,6 +479,49 @@ def truncate_linkage_results(linkage_results_dict: dict) -> dict:
     return linkage_results_dict
 
 
+#: The eight fields every result row carries, as emitted by
+#: ``gen3_validator.validate.validate_object`` (FAIL rows) and
+#: ``gen3_validator.validate.error_record`` (ERROR rows).
+VALIDATION_RESULT_BASE_COLUMNS = [
+    "node",
+    "index",
+    "validation_result",
+    "invalid_key",
+    "schema_path",
+    "validator",
+    "validator_value",
+    "validation_error",
+]
+
+#: Full column contract of the results table, in write order.
+VALIDATION_RESULT_COLUMNS = [
+    "validation_id",
+    "index",
+    "node",
+    "study_id",
+    "validation_result",
+    "invalid_key",
+    "schema_path",
+    "validator",
+    "validator_value",
+    "validation_error",
+    "schema_version",
+]
+
+#: Written when a study validates clean, so that every run leaves exactly one
+#: gradeable row. Excluded by the validation gate — see
+#: :func:`run_validation_gate`.
+PASS_MARKER_ROW = {
+    "node": None,
+    "index": None,
+    "validation_result": "PASS",
+    "invalid_key": None,
+    "schema_path": None,
+    "validator": None,
+    "validator_value": None,
+    "validation_error": None,
+}
+
 #: validation_error patterns treated as known noise by the validation gate —
 #: structural artefacts of the export (null-typed fields, extra export-side
 #: properties, the Gen3 'programs' root requirement), not data quality issues.
@@ -497,9 +540,14 @@ def run_validation_gate(glue_database: str, athena_s3_output: str,
 
     The gate looks at the most recent ``validation_id`` in the results table,
     filters out the known-noise error patterns
-    (:data:`VALIDATION_GATE_IGNORED_ERRORS`) and synthetic studies, and
-    aggregates what remains. An empty result means the latest validation run
-    is clean; any rows are genuine schema failures an operator must fix.
+    (:data:`VALIDATION_GATE_IGNORED_ERRORS`), PASS markers and synthetic
+    studies, and aggregates what remains. An empty result means the latest
+    validation run is clean; any rows are genuine schema failures an operator
+    must fix.
+
+    ERROR rows are deliberately NOT filtered out. A record whose node is absent
+    from the dictionary could not be checked at all, which is a failure to
+    validate rather than a clean result, and must hold the gate closed.
 
     Callers (the validator Glue job) FAIL the run when rows come back, so the
     validation Step Function goes red until the data is fixed and validation
@@ -523,6 +571,7 @@ WHERE validation_id = (
     FROM "{glue_database}"."{results_table}"
 )
 {ignore_clauses}
+  AND validation_result <> 'PASS'
   AND study_id NOT LIKE '%synthetic%'
 GROUP BY node, study_id, invalid_key, validator_value, validation_error
 ORDER BY node, study_id
@@ -567,6 +616,7 @@ def validate_pipeline(
     resolver=None,
     metadata_table: pd.DataFrame = None,
     write_iceberg: bool = True,
+    results_table: str = "full_validation_results",
 ) -> pd.DataFrame:
     """
     Orchestrate the validation workflow for a study: load + resolve schema, find the latest
@@ -592,9 +642,14 @@ def validate_pipeline(
         write_iceberg: When False, skip the per-study Iceberg write and leave it
             to the caller (who can concatenate several studies' results into a
             single write). The CSV write-back always happens.
+        results_table: Table to write into when ``write_iceberg`` is True. Callers
+            validating an isolated warehouse (e.g. the CI databases) MUST pass
+            their own — the real and CI results must never share a table, since
+            the gate grades whichever run has the greatest validation_id.
 
     Returns:
-        The study's validation results DataFrame.
+        The study's validation results DataFrame. Never empty: a study with no
+        failures yields a single PASS marker row.
 
     Raises:
         RuntimeError: When expected inputs are missing or a pipeline step fails.
@@ -663,24 +718,34 @@ def validate_pipeline(
         logger.info("Getting Schema Version")
         schema_version = resolver.get_schema_version(schema = schema)
         
-        full_validation_results_df = pd.DataFrame(results)
+        if not results:
+            # No failures. Record the run anyway: the validation gate grades
+            # the greatest validation_id in the results table, so a clean run
+            # that writes nothing would leave the previous FAILING run as the
+            # latest and the gate could never go green no matter how many
+            # times the data was fixed. A PASS marker also guarantees the
+            # table exists after the first run, which is what stops a fresh
+            # environment reporting TABLE_NOT_FOUND.
+            logger.info("No validation failures for study '%s'; writing PASS marker.", study_id)
+            results = [PASS_MARKER_ROW.copy()]
+
+        # Build with an explicit column list rather than inferring it from the
+        # rows: pd.DataFrame([]) has NO columns, so the reindex below used to
+        # raise KeyError and surface as "Validation failed." — a clean study
+        # failing with the same message as a dirty one.
+        full_validation_results_df = pd.DataFrame(results, columns=VALIDATION_RESULT_BASE_COLUMNS)
         full_validation_results_df["validation_id"] = latest_validation_id
         full_validation_results_df["study_id"] = study_id
         full_validation_results_df["schema_version"] = schema_version
-        
-        full_validation_results_df = full_validation_results_df[[
-            "validation_id",
-            "index",
-            "node",
-            "study_id",
-            "validation_result",
-            "invalid_key",
-            "schema_path",
-            "validator",
-            "validator_value",
-            "validation_error",
-            "schema_version"
-        ]]
+
+        full_validation_results_df = full_validation_results_df[VALIDATION_RESULT_COLUMNS]
+
+        # 'index' is the only numeric column. A PASS-only frame leaves it
+        # entirely null, which would infer a different Iceberg column type
+        # than a frame containing failures — and full_validation_results
+        # already exists in deployed environments. Pin it so both shapes
+        # write the same schema.
+        full_validation_results_df = full_validation_results_df.astype({"index": "Int64"})
 
         logger.info("Validation completed (%s rows).", len(full_validation_results_df))
     except Exception as e:
@@ -703,12 +768,12 @@ def validate_pipeline(
             logger.info(
                 "Writing Parquet to Glue DB '%s', table '%s'.",
                 glue_database,
-                "full_validation_results",
+                results_table,
             )
             write_iceberg_to_db(
                 df=full_validation_results_df,
                 database=glue_database,
-                table="full_validation_results",
+                table=results_table,
                 athena_s3_output=athena_s3_output,
                 workgroup=workgroup,
             )