gen3-dataops-toolkit 2.3.0__tar.gz → 3.1.0__tar.gz

{gen3_dataops_toolkit-2.3.0 → gen3_dataops_toolkit-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 2.3.0
+Version: 3.1.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -113,9 +113,15 @@ structurally impossible.
 
 **Only the dbt template's `ci` target is prefixed.** `g3dt config dbt-env`
 emits, alongside the real names, the CI-isolation variants the template's
-`ci` target consumes: `G3DT_DB_RAW_SILVER_CI` / `G3DT_DB_RAW_GOLD_CI`
-(`ci_` + the real database name) and `G3DT_S3_SILVER_DATA_DIR_CI` /
-`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Commit-
+`ci` target consumes: `G3DT_DB_BRONZE_CI` / `G3DT_DB_SILVER_CI` /
+`G3DT_DB_GOLD_CI` (`ci_` + the real database name) and
+`G3DT_S3_BRONZE_DATA_DIR_CI` / `G3DT_S3_SILVER_DATA_DIR_CI` /
+`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Bronze can be
+dbt-managed from gen3-dbt-template's synthetic-data revision onwards; a
+pipeline deployment >= v2.1.0 provides the matching `ci_..._bronze_db` Glue
+database. Toolkit
+releases >= 3 read the raw-free medallion SSM keys and therefore require a
+pipeline deployment >= v2.0.0, which publishes them. Commit-
 triggered CI builds land there; every other target (default, local) and the
 release build keep the real, unprefixed names — so CI can never advance the
 warehouse's Iceberg snapshots that releases pin. The library enforces the

{gen3_dataops_toolkit-2.3.0 → gen3_dataops_toolkit-3.1.0}/README.md

@@ -79,9 +79,15 @@ structurally impossible.
 
 **Only the dbt template's `ci` target is prefixed.** `g3dt config dbt-env`
 emits, alongside the real names, the CI-isolation variants the template's
-`ci` target consumes: `G3DT_DB_RAW_SILVER_CI` / `G3DT_DB_RAW_GOLD_CI`
-(`ci_` + the real database name) and `G3DT_S3_SILVER_DATA_DIR_CI` /
-`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Commit-
+`ci` target consumes: `G3DT_DB_BRONZE_CI` / `G3DT_DB_SILVER_CI` /
+`G3DT_DB_GOLD_CI` (`ci_` + the real database name) and
+`G3DT_S3_BRONZE_DATA_DIR_CI` / `G3DT_S3_SILVER_DATA_DIR_CI` /
+`G3DT_S3_GOLD_DATA_DIR_CI` (`dbt_ci/` under the same buckets). Bronze can be
+dbt-managed from gen3-dbt-template's synthetic-data revision onwards; a
+pipeline deployment >= v2.1.0 provides the matching `ci_..._bronze_db` Glue
+database. Toolkit
+releases >= 3 read the raw-free medallion SSM keys and therefore require a
+pipeline deployment >= v2.0.0, which publishes them. Commit-
 triggered CI builds land there; every other target (default, local) and the
 release build keep the real, unprefixed names — so CI can never advance the
 warehouse's Iceberg snapshots that releases pin. The library enforces the

{gen3_dataops_toolkit-2.3.0 → gen3_dataops_toolkit-3.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "2.3.0"
+version = "3.1.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-2.3.0 → gen3_dataops_toolkit-3.1.0}/src/g3dt/cli/config_cmds.py

@@ -25,6 +25,27 @@ app = typer.Typer(
 )
 
 
+def _req_key(rc, key: str) -> str:
+    """Return the SSM leaf ``key`` from ``rc``, failing loudly if absent.
+
+    The medallion names (``buckets/silver|gold``, ``glue/db/bronze|silver|gold``)
+    are published under these raw-free keys by pipeline deployments >= v2.0.0.
+    Before this guard, a missing key propagated as ``None``: dbt-env silently
+    dropped the G3DT_DB_* vars and emitted ``s3://None/dbt/`` data dirs, and
+    the release search silently fell back to an account-wide catalog walk.
+    """
+    value = rc.get(key)
+    if value is None:
+        raise config.ConfigError(
+            f"SSM parameter /{rc.project}/{rc.env}/{key} is missing. "
+            f"gen3-dataops-toolkit >= 3 reads the raw-free medallion keys "
+            f"published by gen3-aws-data-pipeline >= v2.0.0; a pipeline "
+            f"deployment older than v2.0.0 still publishes raw-prefixed keys. "
+            f"Upgrade the pipeline deployment (or pin gen3-dataops-toolkit<3)."
+        )
+    return value
+
+
 @app.command()
 def envs() -> None:
     """List the environments with a deployed SSM tree for this project."""
@@ -208,28 +229,35 @@ def dbt_env(
     profile = None if env.endswith("_ec2") else config.aws_profile_for(base, marker)
     try:
         rc = resolver.resolve(project, base, profile=profile)
+        silver_db = _req_key(rc, "glue/db/silver")
+        gold_db = _req_key(rc, "glue/db/gold")
+        silver_bucket = _req_key(rc, "buckets/silver")
+        gold_bucket = _req_key(rc, "buckets/gold")
+        bronze_bucket = _req_key(rc, "buckets/bronze")
+        bronze_db = _req_key(rc, "glue/db/bronze")
     except config.ConfigError as exc:
         typer.secho(str(exc), fg=typer.colors.RED, err=True)
         raise typer.Exit(1)
 
-    raw_silver_db = rc.get("glue/db/rawSilver")
-    raw_gold_db = rc.get("glue/db/rawGold")
     values = {
         "G3DT_REGION": rc.region,
         "G3DT_ATHENA_WORKGROUP": rc.athena_workgroup,
         "G3DT_ATHENA_OUTPUT": rc.athena_output_location,
-        "G3DT_DB_RAW_BRONZE": rc.get("glue/db/rawBronze"),
-        "G3DT_DB_RAW_SILVER": raw_silver_db,
-        "G3DT_DB_RAW_GOLD": raw_gold_db,
-        "G3DT_S3_SILVER_DATA_DIR": f"s3://{rc.get('buckets/rawSilver')}/dbt/",
-        "G3DT_S3_GOLD_DATA_DIR": f"s3://{rc.get('buckets/rawGold')}/dbt/",
+        "G3DT_DB_BRONZE": bronze_db,
+        "G3DT_DB_SILVER": silver_db,
+        "G3DT_DB_GOLD": gold_db,
+        "G3DT_S3_BRONZE_DATA_DIR": f"s3://{bronze_bucket}/dbt/",
+        "G3DT_S3_SILVER_DATA_DIR": f"s3://{silver_bucket}/dbt/",
+        "G3DT_S3_GOLD_DATA_DIR": f"s3://{gold_bucket}/dbt/",
         # CI isolation: the dbt template's `ci` target builds into these
         # instead — same grammar as the CDK's ci_ databases, same buckets
         # under a dbt_ci/ prefix. Real names above are never prefixed.
-        "G3DT_DB_RAW_SILVER_CI": f"ci_{raw_silver_db}" if raw_silver_db else None,
-        "G3DT_DB_RAW_GOLD_CI": f"ci_{raw_gold_db}" if raw_gold_db else None,
-        "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{rc.get('buckets/rawSilver')}/dbt_ci/",
-        "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{rc.get('buckets/rawGold')}/dbt_ci/",
+        "G3DT_DB_BRONZE_CI": f"ci_{bronze_db}",
+        "G3DT_DB_SILVER_CI": f"ci_{silver_db}",
+        "G3DT_DB_GOLD_CI": f"ci_{gold_db}",
+        "G3DT_S3_BRONZE_DATA_DIR_CI": f"s3://{bronze_bucket}/dbt_ci/",
+        "G3DT_S3_SILVER_DATA_DIR_CI": f"s3://{silver_bucket}/dbt_ci/",
+        "G3DT_S3_GOLD_DATA_DIR_CI": f"s3://{gold_bucket}/dbt_ci/",
     }
     if profile:
         # A named profile means a laptop run: select the dbt target that

{gen3_dataops_toolkit-2.3.0 → gen3_dataops_toolkit-3.1.0}/src/g3dt/cli/release_cmds.py

@@ -57,10 +57,14 @@ def write(
         )
         # The env's own silver/gold DBs (from SSM) scope the model->DB search:
         # deterministic in shared accounts, no account-wide Glue perms needed.
+        # _req_key fails loudly if the raw-free keys are absent (pre-v2.0.0
+        # pipeline) instead of silently widening to an account-wide walk.
+        from g3dt.cli.config_cmds import _req_key
+
         search_databases = [
-            db for db in (rc.get("glue/db/rawSilver"), rc.get("glue/db/rawGold"))
-            if db
-        ] or None
+            _req_key(rc, "glue/db/silver"),
+            _req_key(rc, "glue/db/gold"),
+        ]
         release_writer.run(
             dbt_schema_path=dbt_schema_path,
             release_db=rc.release_db,