gen3-dataops-toolkit 2.0.0__tar.gz → 2.1.0__tar.gz

{gen3_dataops_toolkit-2.0.0 → gen3_dataops_toolkit-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 2.0.0
+Version: 2.1.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391

{gen3_dataops_toolkit-2.0.0 → gen3_dataops_toolkit-2.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "2.0.0"
+version = "2.1.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-2.0.0 → gen3_dataops_toolkit-2.1.0}/src/g3dt/cli/config_cmds.py

@@ -182,6 +182,52 @@ def diff(
     raise typer.Exit(1 if drift else 0)
 
 
+@app.command("dbt-env")
+def dbt_env(
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+) -> None:
+    """Emit `export` lines for the env's dbt settings (resolved from SSM).
+
+    The dbt template's profiles.yml / dbt_project.yml read their derived names
+    from env_var(); this command is the one source of those values, for both
+    CodeBuild and a laptop:
+
+        eval "$(g3dt config dbt-env --env test)" && dbt build
+    """
+    import shlex
+
+    from g3dt import resolver
+
+    marker = config.load_marker()
+    project = config.require_project(marker)
+    base = config.env_base(env)
+    profile = None if env.endswith("_ec2") else config.aws_profile_for(base, marker)
+    try:
+        rc = resolver.resolve(project, base, profile=profile)
+    except config.ConfigError as exc:
+        typer.secho(str(exc), fg=typer.colors.RED, err=True)
+        raise typer.Exit(1)
+
+    values = {
+        "G3DT_REGION": rc.region,
+        "G3DT_ATHENA_WORKGROUP": rc.athena_workgroup,
+        "G3DT_ATHENA_OUTPUT": rc.athena_output_location,
+        "G3DT_DB_RAW_BRONZE": rc.get("glue/db/rawBronze"),
+        "G3DT_DB_RAW_SILVER": rc.get("glue/db/rawSilver"),
+        "G3DT_DB_RAW_GOLD": rc.get("glue/db/rawGold"),
+        "G3DT_S3_SILVER_DATA_DIR": f"s3://{rc.get('buckets/rawSilver')}/dbt/",
+        "G3DT_S3_GOLD_DATA_DIR": f"s3://{rc.get('buckets/rawGold')}/dbt/",
+    }
+    if profile:
+        # A named profile means a laptop run: select the dbt target that
+        # carries aws_profile_name (CodeBuild/EC2 stay on `default`, ambient).
+        values["G3DT_AWS_PROFILE"] = profile
+        values["G3DT_DBT_TARGET"] = "local"
+    for key, value in values.items():
+        if value is not None:
+            typer.echo(f"export {key}={shlex.quote(str(value))}")
+
+
 @app.command("set")
 def set_value(
     key: str = typer.Argument(..., help="Bootstrap key: project, region, default_env."),

{gen3_dataops_toolkit-2.0.0 → gen3_dataops_toolkit-2.1.0}/src/g3dt/cli/main.py

@@ -16,6 +16,8 @@ from g3dt.cli import (
     jobs,
     k8s,
     metadata,
+    pipeline_cmds,
+    release_cmds,
     synth,
 )
 
@@ -34,6 +36,8 @@ app.add_typer(indexd_cmds.app, name="indexd")
 app.add_typer(ec2_cmds.app, name="ec2")
 app.add_typer(jobs.app, name="jobs")
 app.add_typer(config_cmds.app, name="config")
+app.add_typer(release_cmds.app, name="release")
+app.add_typer(pipeline_cmds.app, name="pipeline")
 
 
 _DOCS = """\
@@ -69,6 +73,12 @@ Typical release runbook (staging shown; repeat for prod with care)
   3. g3dt jobs logs <run-id> --follow
   4. g3dt k8s restart-etl --env staging
 
+Data releases (the dbt pipeline; see the project's dbt repo)
+  git tag data-v1.4.0 && git push origin data-v1.4.0
+  g3dt pipeline status --env staging           which stage is running/failed
+  g3dt pipeline logs   --env staging --follow  live dbt + release-writer output
+  (the pipeline itself runs `g3dt release write` — no names needed anywhere)
+
 Synthetic data (test only, all local)
   g3dt synth deploy --env test
 
@@ -78,9 +88,9 @@ EC2 / SSM prerequisites
   - Local profile needs: ssm:SendCommand / ssm:GetCommandInvocation,
     s3:GetObject on the log prefix, ec2:Start/Stop/DescribeInstances.
 
-NOT run by this CLI: the Glue jobs (validation, release-JSON) and the CodeBuild
-dbt pipelines. Those are deployed and triggered via the CDK repo and the
-project's dbt repo.
+NOT run by this CLI: the Glue jobs (validation, release-JSON). The CodeBuild
+dbt pipelines are triggered from the project's dbt repo (branch push = CI,
+data-v* tag = release) and watched with `g3dt pipeline status|logs`.
 """
 
 

gen3_dataops_toolkit-2.1.0/src/g3dt/cli/pipeline_cmds.py

@@ -0,0 +1,155 @@
+"""`g3dt pipeline` — watch the dbt CodePipeline / CodeBuild, mirroring `g3dt jobs`.
+
+One way to watch any long thing: `g3dt jobs` tails EC2-dispatched runs;
+`g3dt pipeline` does the same for the dbt pipelines. Pipeline and CodeBuild
+project names resolve from the env's SSM tree, so the operator never types an
+ARN or a console name.
+"""
+from __future__ import annotations
+
+import time
+from typing import Optional
+
+import typer
+
+from g3dt import config
+from g3dt.upload.metadata_submitter import create_boto3_session
+
+app = typer.Typer(no_args_is_help=True, help="Watch the dbt CodePipeline/CodeBuild.")
+
+#: CodeBuild build statuses that mean the build is over.
+_TERMINAL_BUILD_STATUSES = ("SUCCEEDED", "FAILED", "FAULT", "STOPPED", "TIMED_OUT")
+
+
+def _resolved(env: str):
+    """Return ``(rc, session)`` for the env (names from SSM, auth from marker)."""
+    from g3dt import resolver
+
+    marker = config.load_marker()
+    project = config.require_project(marker)
+    base = config.env_base(env)
+    profile = None if env.endswith("_ec2") else config.aws_profile_for(base, marker)
+    rc = resolver.resolve(project, base, profile=profile)
+    session = create_boto3_session(aws_profile=profile, aws_region=rc.region)
+    return rc, session
+
+
+@app.command()
+def status(
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    which: str = typer.Option(
+        "writeReleaseInfo",
+        "--which",
+        help="writeReleaseInfo | dbtTestAndRun",
+    ),
+) -> None:
+    """Show the latest execution state per stage of the pipeline."""
+    try:
+        rc, session = _resolved(env)
+    except config.ConfigError as exc:
+        typer.secho(str(exc), fg=typer.colors.RED, err=True)
+        raise typer.Exit(1)
+    pipeline_name = rc.get(f"codepipeline/{which}")
+    if not pipeline_name:
+        typer.secho(
+            f"No SSM parameter codepipeline/{which} — valid values: "
+            f"writeReleaseInfo, dbtTestAndRun.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+    state = session.client("codepipeline").get_pipeline_state(name=pipeline_name)
+    typer.secho(f"{pipeline_name}", bold=True)
+    for stage in state.get("stageStates", []):
+        latest = stage.get("latestExecution", {})
+        typer.echo(f"  {stage['stageName']:<32} {latest.get('status', 'n/a')}")
+
+
+@app.command()
+def logs(
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    which: str = typer.Option(
+        "dbtReleaseBuilder",
+        "--which",
+        help="dbtReleaseBuilder | dbtTestAndRun",
+    ),
+    follow: bool = typer.Option(False, "--follow", "-f", help="Tail until the build ends."),
+    poll_seconds: int = typer.Option(5, "--poll-seconds", help="Follow poll interval."),
+) -> None:
+    """Print (and optionally follow) the latest build's CloudWatch output.
+
+    The log group is CodeBuild's default ``/aws/codebuild/<project>``; the
+    project name comes from SSM ``codebuild/*``. The follow loop is the same
+    filter_log_events + de-dup pattern as `g3dt jobs logs`.
+    """
+    try:
+        rc, session = _resolved(env)
+    except config.ConfigError as exc:
+        typer.secho(str(exc), fg=typer.colors.RED, err=True)
+        raise typer.Exit(1)
+    project_name = rc.get(f"codebuild/{which}")
+    if not project_name:
+        typer.secho(
+            f"No SSM parameter codebuild/{which} — valid values: "
+            f"dbtReleaseBuilder, dbtTestAndRun.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+
+    cb = session.client("codebuild")
+    build_ids = cb.list_builds_for_project(
+        projectName=project_name, sortOrder="DESCENDING"
+    ).get("ids", [])
+    if not build_ids:
+        typer.secho(f"No builds yet for {project_name}.", fg=typer.colors.YELLOW)
+        return
+    build_id = build_ids[0]
+    # CodeBuild names the stream <project>/<build-uuid> in /aws/codebuild/<project>.
+    stream_prefix = build_id.split(":", 1)[1] if ":" in build_id else build_id
+    group = f"/aws/codebuild/{project_name}"
+    logs_client = session.client("logs")
+
+    def build_status() -> Optional[str]:
+        builds = cb.batch_get_builds(ids=[build_id]).get("builds", [])
+        return builds[0].get("buildStatus") if builds else None
+
+    typer.secho(f"{build_id}  ({group})", bold=True)
+    seen_ids: set = set()
+    last_ts = 0
+
+    def drain() -> None:
+        nonlocal last_ts
+        kwargs = {
+            "logGroupName": group,
+            "logStreamNamePrefix": stream_prefix,
+            "startTime": last_ts,
+        }
+        events = []
+        try:
+            while True:
+                resp = logs_client.filter_log_events(**kwargs)
+                events.extend(resp.get("events", []))
+                token = resp.get("nextToken")
+                if not token:
+                    break
+                kwargs["nextToken"] = token
+        except logs_client.exceptions.ResourceNotFoundException:
+            return
+        for ev in sorted(events, key=lambda e: e["timestamp"]):
+            if ev["eventId"] in seen_ids:
+                continue
+            seen_ids.add(ev["eventId"])
+            last_ts = max(last_ts, ev["timestamp"])
+            typer.echo(ev["message"].rstrip("\n"))
+
+    while True:
+        drain()
+        if not follow:
+            break
+        current = build_status()
+        if current in _TERMINAL_BUILD_STATUSES:
+            drain()  # catch events ingested between the last drain and completion
+            typer.secho(f"\n[build finished: {current}]", fg=typer.colors.BRIGHT_BLACK)
+            break
+        time.sleep(poll_seconds)

gen3_dataops_toolkit-2.1.0/src/g3dt/cli/release_cmds.py

@@ -0,0 +1,74 @@
+"""`g3dt release` — write the dbt data-release manifest to Athena.
+
+The data half of the release decoupling: a `data-v*` tag on a project's dbt
+repo drives CodePipeline → CodeBuild → `g3dt release write`, which records one
+idempotent row per dbt model in the env's `releases` Iceberg table. Every name
+it needs — release DB/table, the metadata bucket the table lives under, the
+Athena workgroup output, the region — resolves from the env's SSM tree, so the
+buildspec passes only `--env`, the version, and the commit SHA.
+"""
+from __future__ import annotations
+
+import typer
+
+from g3dt import config
+from g3dt.utils import release_writer
+
+app = typer.Typer(no_args_is_help=True, help="Write/inspect dbt data releases.")
+
+
+@app.command()
+def write(
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    data_release_version: str = typer.Option(
+        ...,
+        "--data-release-version",
+        help="Data version, e.g. 1.4.0 (from the data-v* tag, prefix stripped).",
+    ),
+    commit_id: str = typer.Option("", "--commit-id", help="Git SHA for auditing."),
+    dbt_schema_path: str = typer.Option(
+        "models/schema.yml",
+        "--dbt-schema-path",
+        help="dbt schema file listing the models to track (relative to the dbt project root).",
+    ),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", help="Print the resolved DB/table/SQL and write nothing."
+    ),
+) -> None:
+    """Resolve release/db, release/table, athena/* from SSM, then write the rows.
+
+    Idempotent: a (release_tag, model, db) row that already exists is skipped,
+    so re-running the same tag is safe.
+    """
+    from g3dt import resolver
+
+    marker = config.load_marker()
+    project = config.require_project(marker)
+    base = config.env_base(env)
+    profile = None if env.endswith("_ec2") else config.aws_profile_for(base, marker)
+
+    try:
+        rc = resolver.resolve(project, base, profile=profile)
+        typer.secho(
+            f"Release target (from SSM /{project}/{base}): "
+            f"{rc.release_db}.{rc.release_table} "
+            f"(workgroup output {rc.athena_output_location})",
+            bold=True,
+        )
+        release_writer.run(
+            dbt_schema_path=dbt_schema_path,
+            release_db=rc.release_db,
+            release_table=rc.release_table,
+            release_s3_location=f"s3://{rc.metadata_bucket}/",
+            data_release_version=data_release_version,
+            commit_id=commit_id,
+            aws_region=rc.region,
+            athena_s3_output=rc.athena_output_location,
+            aws_profile=profile,
+            dry_run=dry_run,
+        )
+    except config.ConfigError as exc:
+        typer.secho(str(exc), fg=typer.colors.RED, err=True)
+        raise typer.Exit(1)
+    if dry_run:
+        typer.secho("Dry run complete — nothing was written.", fg=typer.colors.GREEN)

{gen3_dataops_toolkit-2.0.0 → gen3_dataops_toolkit-2.1.0}/src/g3dt/utils/release_writer.py

@@ -38,7 +38,8 @@ def insert_release_row(
     release_db: str,
     release_table: str,
     release_tag: str,
-    github_sha: str
+    github_sha: str,
+    dry_run: bool = False,
 ) -> None:
     """
     Insert a new row into the release tracking table for a given model and snapshot.
@@ -46,35 +47,40 @@ def insert_release_row(
     This function checks if a row with the same release_tag, model_name, and db_name already exists
     in the release table to prevent duplicate entries. If no such row exists, it inserts a new row
     with the provided snapshot_id, committed_at timestamp, and other metadata.
+
+    With ``dry_run`` the INSERT is built and logged but never executed (and the
+    existence check is skipped), so an operator can confirm the exact target
+    table and SQL before a real release.
     """
     athena_query = AthenaQuery(athena_config)
     if not db_name:
         logger.warning(f"Skipping model '{model_name}': No database found for this model.")
         return
 
-    logger.debug(
-        f"Checking for existing release row: release_tag={release_tag!r}, model_name={model_name!r}, db_name={db_name!r}"
-    )
-    check_sql = f"""
-        SELECT COUNT(*) AS cnt
-        FROM "{release_db}"."{release_table}"
-        WHERE release_tag = {safe_sql_string(release_tag)}
-          AND model_name  = {safe_sql_string(model_name)}
-          AND db_name     = {safe_sql_string(db_name)}
-    """
-    try:
-        cnt_df = athena_query.query_athena(check_sql, release_db)
-        cnt = cnt_df.iloc[0]['cnt'] if not cnt_df.empty else 0
-    except Exception as e:
-        logger.error(
-            f"Could not query existing release rows for release_tag='{release_tag}', model='{model_name}', db='{db_name}': {e}",
-            exc_info=True
+    if not dry_run:
+        logger.debug(
+            f"Checking for existing release row: release_tag={release_tag!r}, model_name={model_name!r}, db_name={db_name!r}"
         )
-        raise
-
-    if cnt > 0:
-        logger.info(f"[SKIP] Release row already exists: {release_tag} / {db_name}.{model_name}")
-        return
+        check_sql = f"""
+            SELECT COUNT(*) AS cnt
+            FROM "{release_db}"."{release_table}"
+            WHERE release_tag = {safe_sql_string(release_tag)}
+              AND model_name  = {safe_sql_string(model_name)}
+              AND db_name     = {safe_sql_string(db_name)}
+        """
+        try:
+            cnt_df = athena_query.query_athena(check_sql, release_db)
+            cnt = cnt_df.iloc[0]['cnt'] if not cnt_df.empty else 0
+        except Exception as e:
+            logger.error(
+                f"Could not query existing release rows for release_tag='{release_tag}', model='{model_name}', db='{db_name}': {e}",
+                exc_info=True
+            )
+            raise
+
+        if cnt > 0:
+            logger.info(f"[SKIP] Release row already exists: {release_tag} / {db_name}.{model_name}")
+            return
 
     snap_val = "NULL" if snapshot_id is None else str(snapshot_id)
     commit_val = f"TIMESTAMP '{committed_at}'" if committed_at else "NULL"
@@ -93,6 +99,13 @@ def insert_release_row(
             {sha_val}
         )
     """
+    if dry_run:
+        logger.info(
+            f"[DRY-RUN] would insert into {release_db}.{release_table} for "
+            f"{db_name}.{model_name} [release_tag={release_tag}]:\n{insert_sql}"
+        )
+        return
+
     logger.info(
         f"Inserting new release row for {db_name}.{model_name} [release_tag={release_tag}, snapshot_id={snapshot_id}, committed_at={committed_at}]"
     )
@@ -130,31 +143,55 @@ def parse_args():
                         help="S3 URI for Athena query results (e.g., 's3://athena-results-bucket/').")
     parser.add_argument("--release-s3-location", type=str, required=True,
                         help="S3 location for the release table (e.g., 's3://<metadata-bucket>/').")
+    parser.add_argument("--dry-run", action="store_true", default=False,
+                        help="Resolve everything and log the SQL; write nothing.")
     parser.add_argument("-v", "--verbose", action="store_true", default=False,
                         help="Enable debug logging.")
     return parser.parse_args()
 
-def main():
-    args = parse_args()
-    log_level = logging.DEBUG if args.verbose else logging.INFO
-    logging.getLogger().setLevel(log_level)
 
+def run(
+    *,
+    dbt_schema_path: str,
+    release_db: str,
+    release_table: str,
+    release_s3_location: str,
+    data_release_version: str,
+    commit_id: str,
+    aws_region: str,
+    athena_s3_output: str,
+    aws_profile: Optional[str] = None,
+    dry_run: bool = False,
+) -> None:
+    """Write one release row per dbt model (idempotent; see insert_release_row).
+
+    The callable core behind both entry points: the argv-driven ``main()``
+    (``python -m g3dt.utils.release_writer``) and the SSM-resolving CLI wrapper
+    (``g3dt release write``). With ``dry_run`` the resolved target and the
+    INSERT SQL are logged but nothing is written (model/snapshot lookups are
+    read-only and still run).
+    """
     logger.info("==== DBT release snapshot info writer started ====")
-    logger.debug(f"Parsed arguments: {args}")
 
     athena_config = AthenaConfig(
-        aws_region=args.aws_region,
-        aws_profile=args.aws_profile,
-        athena_s3_output=args.athena_s3_output
+        aws_region=aws_region,
+        aws_profile=aws_profile,
+        athena_s3_output=athena_s3_output
     )
 
-    dbt_models = get_model_names(args.dbt_schema_path)
+    dbt_models = get_model_names(dbt_schema_path)
     athena_query = AthenaQuery(athena_config)
-    
-    logger.info(f"Creating release table: {args.release_db}.{args.release_table}")
-    athena_query.create_release_table(
-        args.release_db, args.release_table, args.release_s3_location
-    )
+
+    if dry_run:
+        logger.info(
+            f"[DRY-RUN] would ensure release table {release_db}.{release_table} "
+            f"at {release_s3_location}"
+        )
+    else:
+        logger.info(f"Creating release table: {release_db}.{release_table}")
+        athena_query.create_release_table(
+            release_db, release_table, release_s3_location
+        )
 
     logger.info(f"Processing DBT models from schema: {dbt_models}")
 
@@ -175,14 +212,35 @@ def main():
             db_name=db_name,
             snapshot_id=snapshot_id,
             committed_at=commit_datetime,
-            release_db=args.release_db,
-            release_table=args.release_table,
-            release_tag=args.data_release_version,
-            github_sha=args.commit_id
+            release_db=release_db,
+            release_table=release_table,
+            release_tag=data_release_version,
+            github_sha=commit_id,
+            dry_run=dry_run,
         )
         logger.info(f"[SUCCESS] Release info recorded for {db_name}.{model_name}")
 
     logger.info(f"Finished release process for DBT models: {dbt_models}")
 
+
+def main():
+    args = parse_args()
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    logging.getLogger().setLevel(log_level)
+    logger.debug(f"Parsed arguments: {args}")
+
+    run(
+        dbt_schema_path=args.dbt_schema_path,
+        release_db=args.release_db,
+        release_table=args.release_table,
+        release_s3_location=args.release_s3_location,
+        data_release_version=args.data_release_version,
+        commit_id=args.commit_id,
+        aws_region=args.aws_region,
+        athena_s3_output=args.athena_s3_output,
+        aws_profile=args.aws_profile,
+        dry_run=args.dry_run,
+    )
+
 if __name__ == "__main__":
     main()