gen3-dataops-toolkit 2.0.0__py3-none-any.whl

g3dt/cli/jobs.py

@@ -0,0 +1,83 @@
+"""`acdc jobs` — track EC2-dispatched runs by their friendly run id."""
+from __future__ import annotations
+
+from typing import Optional
+
+import typer
+
+from g3dt.cli._internal import dispatch, registry
+
+app = typer.Typer(no_args_is_help=True, help="Track EC2-dispatched jobs.")
+
+
+def _status_color(state: str) -> str:
+    """Map an SSM/derived status to a terminal colour for the all-runs listing."""
+    if state == "Success":
+        return typer.colors.GREEN
+    if state in ("Failed", "Cancelled", "TimedOut"):
+        return typer.colors.RED
+    if state in ("Pending", "InProgress", "Delayed", "pending"):
+        return typer.colors.YELLOW
+    return typer.colors.BRIGHT_BLACK  # n/a (ssh) / unknown
+
+
+@app.command(name="list")
+def list_runs() -> None:
+    """List recorded EC2 dispatches (most recent first)."""
+    runs = registry.all_runs()
+    if not runs:
+        typer.echo("No dispatched runs recorded.")
+        return
+    for run_id in sorted(runs, reverse=True):
+        rec = runs[run_id]
+        typer.echo(
+            f"{run_id}  env={rec.get('env')}  "
+            f"via={rec.get('mechanism')}  instance={rec.get('instance_id')}"
+        )
+
+
+@app.command()
+def status(
+    run_id: Optional[str] = typer.Argument(
+        None, help="Run id from dispatch. Omit to show all runs with their status."
+    ),
+) -> None:
+    """Show SSM status for one run, or all recorded runs if no run id is given."""
+    if run_id is None:
+        runs = registry.all_runs()
+        if not runs:
+            typer.echo("No dispatched runs recorded.")
+            return
+        for rid in sorted(runs, reverse=True):
+            rec = runs[rid]
+            state = dispatch.status_label(rec)
+            line = f"{rid}  env={rec.get('env')}  via={rec.get('mechanism')}  "
+            typer.echo(line, nl=False)
+            typer.secho(state, fg=_status_color(state))
+        return
+
+    inv = dispatch.status(run_id)
+    state = inv.get("Status", "unknown") if isinstance(inv, dict) else "unknown"
+    typer.secho(f"{run_id}: {state}", bold=True)
+    if isinstance(inv, dict):
+        out = inv.get("StandardOutputContent")
+        err = inv.get("StandardErrorContent")
+        if out:
+            typer.echo(out)
+        if err:
+            typer.secho(err, fg=typer.colors.RED)
+
+
+@app.command()
+def stop(run_id: str = typer.Argument(..., help="Run id from dispatch.")) -> None:
+    """Stop (cancel) a running EC2-dispatched job."""
+    dispatch.stop(run_id)
+
+
+@app.command()
+def logs(
+    run_id: str = typer.Argument(..., help="Run id from dispatch."),
+    follow: bool = typer.Option(False, "--follow", "-f", help="Stream new output as it arrives."),
+) -> None:
+    """Print (and optionally follow) the CloudWatch logs for a dispatched run."""
+    dispatch.logs(run_id, follow=follow)

g3dt/cli/k8s.py

@@ -0,0 +1,54 @@
+"""`g3dt k8s` — restart Gen3 microservices / ETL via ArgoCD (LOCAL only).
+
+These use ``argocd login --sso`` (a browser flow), so they cannot run headless
+on EC2. The wrapped scripts receive their settings as ``G3DT_*`` environment
+variables resolved from SSM — they read no config files.
+"""
+from __future__ import annotations
+
+import typer
+
+from g3dt.config import script_env
+from g3dt.cli._internal import runner
+from g3dt.cli._internal.resolve import env_of
+
+app = typer.Typer(no_args_is_help=True, help="ArgoCD / Kubernetes restarts (local).")
+
+_SCHEMA = "services/k8s_ops/argocd_restart_schema.sh"
+_ETL = "services/k8s_ops/argocd_restart_etl.sh"
+_ETL_AND_MS = "services/k8s_ops/restart_etl_and_ms.sh"
+
+
+@app.command(name="restart-schema")
+def restart_schema(
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
+) -> None:
+    """Restart sheepdog/peregrine/guppy/portal (schema microservices)."""
+    e = env_of(env)
+    args = ["-d", e.domain, "-a", e.app_name, "-n", e.namespace]
+    if sync:
+        args.append("-s")
+    runner.run(runner.bash_script(_SCHEMA, *args), env=script_env(e))
+
+
+@app.command(name="restart-etl")
+def restart_etl(
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
+) -> None:
+    """Create + run the ETL cronjob and wait for completion."""
+    e = env_of(env)
+    args = ["-e", env]
+    if sync:
+        args.append("-s")
+    runner.run(runner.bash_script(_ETL, *args), env=script_env(e))
+
+
+@app.command(name="restart-ms")
+def restart_ms(
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+) -> None:
+    """Restart both ETL and schema microservices (wraps restart_etl_and_ms.sh)."""
+    e = env_of(env)
+    runner.run(runner.bash_script(_ETL_AND_MS, env), env=script_env(e))

g3dt/cli/main.py

@@ -0,0 +1,110 @@
+"""Root ``g3dt`` Typer application.
+
+Assembles every command group and the top-level ``version`` / ``docs`` helpers.
+The console-script entry point in pyproject.toml points at :func:`main`.
+"""
+from __future__ import annotations
+
+import typer
+
+from g3dt.cli import (
+    config_cmds,
+    delete_cmds,
+    dict_cmds,
+    ec2_cmds,
+    indexd_cmds,
+    jobs,
+    k8s,
+    metadata,
+    synth,
+)
+
+app = typer.Typer(
+    no_args_is_help=True,
+    rich_markup_mode="rich",
+    help="Gen3 DataOps toolkit. Run [bold]g3dt docs[/bold] for an overview.",
+)
+
+app.add_typer(dict_cmds.app, name="dict")
+app.add_typer(synth.app, name="synth")
+app.add_typer(metadata.app, name="metadata")
+app.add_typer(delete_cmds.app, name="delete")
+app.add_typer(k8s.app, name="k8s")
+app.add_typer(indexd_cmds.app, name="indexd")
+app.add_typer(ec2_cmds.app, name="ec2")
+app.add_typer(jobs.app, name="jobs")
+app.add_typer(config_cmds.app, name="config")
+
+
+_DOCS = """\
+Gen3 DataOps toolkit (g3dt) — operations overview
+=================================================
+
+Configuration: two kinds, nothing else
+  - INPUTS live in the CDK repo (gen3-aws-data-pipeline) as
+    config/<project>.<env>.json, read only by `cdk deploy`.
+  - Everything else is resolved live from SSM (/{project}/{env}/...), which
+    `cdk deploy` publishes. The only local file is the g3dt.yaml marker
+    (project/region/default_env, optional profiles:/studies: maps), searched
+    at ./g3dt.yaml, ~/.g3dt/g3dt.yaml, /etc/g3dt/g3dt.yaml.
+
+Mental model: two execution planes
+  - Control plane (LOCAL): dict deploy, k8s restarts. These use the interactive
+    `argocd login --sso` browser flow and AWS named profiles, so they run on
+    your laptop only.
+  - Data plane (LONG jobs): metadata upload/delete, indexd register. Add
+    `--on ec2` to run them on the env's job box via SSM (disconnect-safe);
+    watch with `g3dt jobs status|logs <run-id> --follow`.
+
+Discover everything
+  g3dt --help                      list all command groups
+  g3dt <group> --help              commands + options for a group
+  g3dt config envs                 environments with a deployed SSM tree
+  g3dt config studies              studies from your g3dt.yaml marker
+  g3dt config show --env test      resolved settings (safe, read-only)
+
+Typical release runbook (staging shown; repeat for prod with care)
+  1. g3dt dict deploy   --env staging
+  2. g3dt metadata upload --study <study> --env staging --on ec2
+  3. g3dt jobs logs <run-id> --follow
+  4. g3dt k8s restart-etl --env staging
+
+Synthetic data (test only, all local)
+  g3dt synth deploy --env test
+
+EC2 / SSM prerequisites
+  - The env's job box is created by the CDK (ec2-job-runner stack): SSM-managed,
+    toolkit pre-installed by user-data, instance id published to SSM.
+  - Local profile needs: ssm:SendCommand / ssm:GetCommandInvocation,
+    s3:GetObject on the log prefix, ec2:Start/Stop/DescribeInstances.
+
+NOT run by this CLI: the Glue jobs (validation, release-JSON) and the CodeBuild
+dbt pipelines. Those are deployed and triggered via the CDK repo and the
+project's dbt repo.
+"""
+
+
+@app.command()
+def docs() -> None:
+    """Print the operations overview (mental model, runbook, prerequisites)."""
+    typer.echo(_DOCS)
+
+
+@app.command()
+def version() -> None:
+    """Print the installed gen3-dataops-toolkit version."""
+    try:
+        from importlib.metadata import version as _v
+
+        typer.echo(_v("gen3-dataops-toolkit"))
+    except Exception:  # pragma: no cover - fallback when not installed
+        typer.echo("unknown")
+
+
+def main() -> None:
+    """Console-script entry point."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

g3dt/cli/metadata.py

@@ -0,0 +1,76 @@
+"""`g3dt metadata` — upload real study metadata to Gen3 (data-plane).
+
+These are the multi-hour jobs, so they support ``--on ec2`` to run on the
+env's EC2 job box via SSM Run Command (disconnect-safe) instead of the laptop.
+"""
+from __future__ import annotations
+
+import typer
+
+from g3dt.cli._internal import dispatch
+from g3dt.cli._internal.dispatch import Target
+from g3dt.cli._internal.resolve import study_of
+
+app = typer.Typer(no_args_is_help=True, help="Upload study metadata to Gen3.")
+
+_UPLOAD = "services/upload/metadata/upload_metadata.py"
+_UPLOAD_ALL = "services/upload/metadata/upload_all_studies.sh"
+
+
+@app.command()
+def upload(
+    study: str = typer.Option(..., "--study", "-s", help="Study, e.g. ausdiab."),
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    node: str = typer.Option(None, "--node", help="Submit only this node type."),
+    on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
+) -> None:
+    """Upload a study's release metadata to Gen3 sheepdog.
+
+    Examples:
+      g3dt metadata upload --study ausdiab --env staging
+      g3dt metadata upload --study ausdiab --env staging --on ec2
+    """
+    s = study_of(study, env)
+
+    def build_args(env_name):
+        a = ["--study", s.key, "--env", env_name]
+        if node:
+            a += ["--specific-node", node]
+        return a
+
+    def remote_cli(env_name):
+        a = ["metadata", "upload", "--study", study, "--env", env_name]
+        if node:
+            a += ["--node", node]
+        return a
+
+    dispatch.run_or_dispatch(
+        on, env, _UPLOAD, build_args, "metadata-upload", remote_cli=remote_cli,
+    )
+
+
+@app.command(name="upload-all")
+def upload_all(
+    studies: str = typer.Option(
+        ..., "--studies", help="Comma-separated studies, e.g. ausdiab,caughtcad."
+    ),
+    env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
+    on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
+) -> None:
+    """Upload several studies sequentially (wraps upload_all_studies.sh).
+
+    The wrapped script aborts on any 'prod' environment.
+    """
+    names = [s.strip() for s in studies.split(",") if s.strip()]
+    keys = [study_of(name, env).key for name in names]
+
+    def build_args(env_name):
+        return ["--studies", ",".join(keys), "--env", env_name]
+
+    def remote_cli(env_name):
+        return ["metadata", "upload-all", "--studies", studies, "--env", env_name]
+
+    dispatch.run_or_dispatch(
+        on, env, _UPLOAD_ALL, build_args, "metadata-upload-all",
+        interpreter="bash", remote_cli=remote_cli,
+    )

g3dt/cli/synth.py

@@ -0,0 +1,206 @@
+"""`g3dt synth` — synthetic data lifecycle for any configured environment.
+
+Generation uses **gen3-metadata-simulator** (schema-valid). It runs locally:
+synthetic metadata is generated on the laptop (writing under
+``~/.g3dt/synth_metadata/<version>/<study>/``) and uploaded/deleted from
+there, so there is nothing to run on EC2.
+
+Every command accepts ``--env``; targeting a **production** environment (any env
+whose name contains ``prod``) shows a warning and requires typing the env name to
+confirm — it cannot be bypassed.
+
+Generation defaults to keyless ``random`` data (no API calls). Pass ``--llm`` for
+LLM-realistic values, which needs an API key configured in a ``.env`` in the
+working directory (``LLM_PROVIDER`` / ``LLM_MODEL`` / ``LLM_API_KEY_FILE``).
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from g3dt.config import script_env
+from g3dt.cli._internal import runner, safety
+from g3dt.cli._internal.resolve import env_of
+from g3dt.cli.dict_cmds import SCHEMA_DIR, dict_url
+
+app = typer.Typer(
+    no_args_is_help=True,
+    help="Synthetic data lifecycle (local generation; prod requires typed confirmation).",
+)
+
+SYNTH_DIR = Path("~/.g3dt/synth_metadata").expanduser()
+
+
+@app.command()
+def deploy(
+    env: str = typer.Option(
+        "test", "--env", "-e", help="Target environment (prod requires typed confirmation)."
+    ),
+) -> None:
+    """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts).
+
+    Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed
+    generation). Requires an LLM key configured in .env.
+    """
+    e = env_of(env)
+    safety.confirm_prod_strict("synthetic full deploy", env)
+    runner.run(
+        runner.bash_script(
+            "services/synthetic_data/full_deploy_dd_and_synth.sh", env
+        ),
+        env=script_env(e),
+    )
+
+
+@app.command()
+def generate(
+    studies: str = typer.Argument(
+        ...,
+        help="Simulated study id(s); comma-separated for many, e.g. AusDiab_Simulated.",
+    ),
+    env: str = typer.Option(
+        "test", "--env", "-e", help="Target environment (prod requires typed confirmation)."
+    ),
+    num_records: str = typer.Option(
+        None,
+        "--num-records",
+        "-n",
+        help="Records per study: one number for all, or a comma list (one per study).",
+    ),
+    provider: str = typer.Option(
+        "random", "--provider", help="Value strategy: 'random' (default, keyless) or 'llm'."
+    ),
+    llm: bool = typer.Option(
+        False,
+        "--llm",
+        help="Generate LLM-realistic values; reads LLM config from a .env in the "
+        "working directory. Default is keyless random data (no API key, no API calls).",
+    ),
+    seed: int = typer.Option(None, "--seed", help="RNG seed for reproducible output."),
+    schema: str = typer.Option(
+        None, "--schema", help="Gen3 schema path (default: pulled for the version)."
+    ),
+    version: str = typer.Option(
+        None, "--version", help="Version label for output dir (default: env dictionary_version)."
+    ),
+) -> None:
+    """Generate synthetic metadata locally with gen3-metadata-simulator.
+
+    STUDIES is one simulated study id, or several comma-separated. Defaults to
+    keyless, schema-valid random data (no API key, no API calls). Pass --llm to
+    generate LLM-realistic values instead.
+
+    Examples:
+      g3dt synth generate AusDiab_Simulated -n 5 --seed 1
+      g3dt synth generate AusDiab_Simulated --llm -n 5
+      g3dt synth generate "AusDiab_Simulated,Baker-Biobank_Simulated" -n "30,60"
+    """
+    e = env_of(env)
+    safety.confirm_prod_strict("synthetic generation", env)
+
+    # A comma list of per-study counts must line up with the studies given.
+    if num_records and "," in num_records:
+        n_counts = len(num_records.split(","))
+        n_studies = len(studies.split(","))
+        if n_counts != n_studies:
+            typer.secho(
+                f"--num-records has {n_counts} values but {n_studies} studies "
+                f"were given (pass one count, or one per study).",
+                fg=typer.colors.RED,
+                err=True,
+            )
+            raise typer.Exit(1)
+
+    ver = version or e.dictionary_version
+    schema_path = schema or str(SCHEMA_DIR / f"acdc_schema_{ver}.json")
+
+    # Ensure the schema is available locally; pull it if missing.
+    if not Path(schema_path).exists():
+        typer.secho(f"Schema not found locally; pulling {ver}...", fg=typer.colors.YELLOW)
+        runner.run(
+            runner.bash_script("services/dictionary/pull_dict.sh", dict_url(e, ver)),
+            env=script_env(e),
+        )
+
+    effective_provider = "llm" if llm else provider
+    args = [
+        "--schema", schema_path,
+        "--version", ver,
+        "--provider", effective_provider,
+        "--studies", studies,
+    ]
+    if num_records:
+        args += ["--num-records", num_records]
+    if seed is not None:
+        args += ["--seed", str(seed)]
+    runner.run(
+        runner.bash_script(
+            "services/synthetic_data/generate_synth_metadata.sh", *args
+        ),
+        env=script_env(e),
+    )
+
+
+@app.command()
+def upload(
+    env: str = typer.Option(
+        "test", "--env", "-e", help="Target environment (prod requires typed confirmation)."
+    ),
+    version: str = typer.Option(
+        None, "--version", help="Dictionary version dir (default: the env's version)."
+    ),
+) -> None:
+    """Upload generated synthetic metadata to Gen3 (reads local files)."""
+    e = env_of(env)
+    safety.confirm_prod_strict("synthetic metadata upload", env)
+    v = version or e.dictionary_version
+    base_dir = str(SYNTH_DIR / v) + "/"
+    args = ["--base-dir", base_dir, "--aws-secret-name", e.aws_secret_name]
+    if e.aws_profile:
+        args += ["--aws-profile", e.aws_profile]
+    runner.run(
+        runner.python_script(
+            "services/synthetic_data/upload_synth_metadata_sheepdog.py", *args
+        ),
+        env=script_env(e),
+    )
+
+
+@app.command()
+def delete(
+    env: str = typer.Option(
+        "test", "--env", "-e", help="Target environment (prod requires typed confirmation)."
+    ),
+    projects: str = typer.Option(
+        None, "--projects", "-p", help="Comma-separated simulated project ids."
+    ),
+    import_order: str = typer.Option(
+        None,
+        "--import-order",
+        help="DataImportOrder.txt path (default: DataImportOrder.txt in the cwd).",
+    ),
+) -> None:
+    """Delete previously-uploaded synthetic metadata from Gen3."""
+    e = env_of(env)
+    safety.confirm_prod_strict("synthetic metadata deletion", env)
+    order = import_order or "DataImportOrder.txt"
+    args = ["-i", order, "-s", e.aws_secret_name]
+    if e.aws_profile:
+        args += ["-profile", e.aws_profile]
+    if projects:
+        args += ["-p", projects]
+    runner.run(
+        runner.python_script(
+            "services/synthetic_data/delete_synth_metadata_sheepdog.py", *args
+        ),
+        env=script_env(e),
+    )
+
+
+@app.command(name="install-simulator")
+def install_simulator() -> None:
+    """Install the gen3-metadata-simulator generator (the 'synth' extra)."""
+    import sys
+
+    runner.run([sys.executable, "-m", "pip", "install", "gen3-metadata-simulator"])