gen3-dataops-toolkit 4.0.1__tar.gz → 4.2.0__tar.gz

{gen3_dataops_toolkit-4.0.1 → gen3_dataops_toolkit-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 4.0.1
+Version: 4.2.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -80,8 +80,11 @@ confirmation, and destructive actions on them require typing the context
 name — `--yes` never bypasses that.
 
 The marker can also be written by hand (design doc:
-`docs/design/contexts.md`); the study registry lives either in a top-level
-`studies:` block or per env at `s3://<metadata-bucket>/config/studies.yaml`.
+`docs/design/contexts.md`). The study registry lives in the env's SSM tree
+(`/{project}/{env}/studies/<name>`, one JSON parameter per study — manage
+it with `g3dt study`; design doc `docs/design/studies.md`). The legacy
+per-env `s3://<metadata-bucket>/config/studies.yaml` is a deprecation-
+warned read fallback until 5.0 — import it with `g3dt study migrate`.
 Search order: `./g3dt.yaml` → `~/.g3dt/g3dt.yaml` → `/etc/g3dt/g3dt.yaml`
 (the EC2 job box's copy, written by CDK user-data). Legacy markers
 (`project`/`default_env`/`profiles:` keys) keep working unchanged, as do the
@@ -92,6 +95,7 @@ the file-less CodeBuild/EC2 path.
 
 ```bash
 g3dt config show                 # every resolved name — the safety check
+g3dt study repoint --latest      # point the registry at the newest release
 g3dt ec2 up                      # start the context's job box (SSM-managed)
 g3dt metadata upload --study mystudy --on ec2
 g3dt jobs logs <run-id> --follow # live logs; laptop can sleep, job keeps going

{gen3_dataops_toolkit-4.0.1 → gen3_dataops_toolkit-4.2.0}/README.md

@@ -46,8 +46,11 @@ confirmation, and destructive actions on them require typing the context
 name — `--yes` never bypasses that.
 
 The marker can also be written by hand (design doc:
-`docs/design/contexts.md`); the study registry lives either in a top-level
-`studies:` block or per env at `s3://<metadata-bucket>/config/studies.yaml`.
+`docs/design/contexts.md`). The study registry lives in the env's SSM tree
+(`/{project}/{env}/studies/<name>`, one JSON parameter per study — manage
+it with `g3dt study`; design doc `docs/design/studies.md`). The legacy
+per-env `s3://<metadata-bucket>/config/studies.yaml` is a deprecation-
+warned read fallback until 5.0 — import it with `g3dt study migrate`.
 Search order: `./g3dt.yaml` → `~/.g3dt/g3dt.yaml` → `/etc/g3dt/g3dt.yaml`
 (the EC2 job box's copy, written by CDK user-data). Legacy markers
 (`project`/`default_env`/`profiles:` keys) keep working unchanged, as do the
@@ -58,6 +61,7 @@ the file-less CodeBuild/EC2 path.
 
 ```bash
 g3dt config show                 # every resolved name — the safety check
+g3dt study repoint --latest      # point the registry at the newest release
 g3dt ec2 up                      # start the context's job box (SSM-managed)
 g3dt metadata upload --study mystudy --on ec2
 g3dt jobs logs <run-id> --follow # live logs; laptop can sleep, job keeps going

{gen3_dataops_toolkit-4.0.1 → gen3_dataops_toolkit-4.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "4.0.1"
+version = "4.2.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"
@@ -44,7 +44,7 @@ moto = ">=5.0"
 optional = true
 
 [tool.poetry.group.synth.dependencies]
-gen3-metadata-simulator = "^0.3.0"
+gen3-metadata-simulator = "^0.5.3"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

{gen3_dataops_toolkit-4.0.1 → gen3_dataops_toolkit-4.2.0}/src/g3dt/cli/_internal/resolve.py

@@ -30,6 +30,28 @@ def study_of(study: str, env: str) -> config.StudyConfig:
     except config.ConfigError as exc:
         typer.secho(str(exc), fg=typer.colors.RED, err=True)
         raise typer.Exit(1)
+    except Exception as exc:  # botocore auth/permission failures, guided
+        typer.secho(_aws_error_message(exc, env), fg=typer.colors.RED, err=True)
+        raise typer.Exit(1)
+
+
+def _aws_error_message(exc: Exception, env: str) -> str:
+    """A guided rendering of a raw AWS/botocore failure.
+
+    An expired SSO session used to surface as either a traceback or —
+    worse — a silent "(none)" study registry. Name the likely fix instead.
+    """
+    profile = None
+    try:
+        profile = config.aws_profile_for(env, config.load_marker())
+    except Exception:
+        pass
+    hint = (
+        f" If your SSO session expired: aws sso login --profile {profile}"
+        if profile
+        else ""
+    )
+    return f"AWS error while resolving env '{env}': {exc}.{hint}"
 
 
 def active_env(env: Optional[str]) -> str:
@@ -83,3 +105,22 @@ def rc_of(env: str):
     except config.ConfigError as exc:
         typer.secho(str(exc), fg=typer.colors.RED, err=True)
         raise typer.Exit(1)
+    except Exception as exc:  # botocore auth/permission failures, guided
+        typer.secho(_aws_error_message(exc, env), fg=typer.colors.RED, err=True)
+        raise typer.Exit(1)
+
+
+def rc_session_of(env: str):
+    """``(rc, session)`` for ``env`` — resolved names plus an authed session.
+
+    The single idiom for commands that both read SSM names and make further
+    AWS calls (Athena, S3, SSM writes). Same credential rule as :func:`rc_of`;
+    the session's region is the resolved env's own ``meta/region``.
+    """
+    import boto3
+
+    rc = rc_of(env)
+    marker = config.load_marker()
+    profile = None if env.endswith("_ec2") else config.aws_profile_for(env, marker)
+    session = boto3.Session(profile_name=profile, region_name=rc.region)
+    return rc, session

{gen3_dataops_toolkit-4.0.1 → gen3_dataops_toolkit-4.2.0}/src/g3dt/cli/config_cmds.py

@@ -528,36 +528,12 @@ def envs() -> None:
 
 @app.command()
 def studies(
-    env: Optional[str] = typer.Option(
-        None,
-        "--env",
-        "-e",
-        help="Read the study registry from this env's S3 bucket "
-             "(s3://<metadata-bucket>/config/studies.yaml) instead of the "
-             "local marker — this changes the data source, not just the "
-             "target.",
-    ),
+    env: Optional[str] = typer.Option(None, "--env", "-e", help=ENV_OPT),
 ) -> None:
-    """List the configured studies (bare names).
+    """List the env's registered studies (alias of `g3dt study list`)."""
+    from g3dt.cli import study_cmds
 
-    The registry comes from the marker's studies: block, or — pass --env —
-    from the env's S3 registry, which is what the EC2 job box uses.
-    """
-    if env is not None:
-        env = resolve.active_env(env)
-    else:
-        resolve.announce_context()
-    names = config.list_studies(env=env)
-    if not names:
-        typer.secho(
-            "No studies configured. Add a studies: block to your g3dt.yaml "
-            "marker, or upload config/studies.yaml to the env's metadata "
-            "bucket (and pass --env).",
-            fg=typer.colors.YELLOW,
-        )
-        return
-    for name in names:
-        typer.echo(name)
+    study_cmds.list_impl(env)
 
 
 @app.command()

{gen3_dataops_toolkit-4.0.1 → gen3_dataops_toolkit-4.2.0}/src/g3dt/cli/delete_cmds.py

@@ -7,6 +7,13 @@ the bare names (a specific version like ``0.9.8``, resolved via an Athena GUID
 lookup, or ``all`` for every version). A bare study with no version anywhere
 is refused (exit 2).
 
+``--synthetic`` switches to registry-free mode for synthetic data: each
+``--studies`` name is the Gen3 project code itself (no SSM study registry —
+synthetic projects are never registered), bare names default to version
+``all``, and a specific version is matched verbatim against the records'
+``data_version`` property via GraphQL rather than Athena receipts (synthetic
+uploads write none).
+
 Every command confirms before acting. Production always requires typing the
 target id, even with ``--yes``. Deleting ALL versions always prompts, even with
 ``--yes``. Confirmation happens locally before any EC2 dispatch (SSM has no
@@ -61,12 +68,30 @@ def _normalise_version(raw: str, where: str) -> str:
     return match.group(1)
 
 
-def _parse_study_specs(studies: str, fallback, env: str):
+def _synthetic_version(raw: str) -> str:
+    """Canonicalise a synthetic version token: ``all`` (any case) or verbatim.
+
+    Synthetic versions are matched exactly against the records' ``data_version``
+    property, and the natural label there is the dictionary version WITH its
+    leading ``v`` (batch dirs are ``~/.g3dt/synth_metadata/v1.3.0/...``) — so
+    unlike ``_normalise_version`` nothing is stripped. A version that matches
+    no records is reported by the worker (skip + hint), not silently absorbed.
+    """
+    token = raw.strip()
+    return "all" if token.lower() == "all" else token
+
+
+def _parse_study_specs(studies: str, fallback, env: str, synthetic: bool = False):
     """Turn ``--studies`` into ``[(resolved_study_key, version), ...]``.
 
     Each comma-separated entry is ``name`` or ``name:version``. A bare name
     takes *fallback* (the ``--version`` default); *fallback* is ``None`` when
-    ``--version`` was not given, which makes a bare name a usage error.
+    ``--version`` was not given, which makes a bare name a usage error —
+    except with *synthetic*, where a bare name defaults to ``all`` (the whole
+    point of the flag is "wipe the synthetic project").
+
+    With *synthetic* the raw name IS the Gen3 project code: no study-registry
+    lookup, and version tokens pass through :func:`_synthetic_version`.
 
     Every entry is validated before anything is dispatched, so a typo in the
     last study cannot leave the earlier ones already deleted.
@@ -102,9 +127,15 @@ def _parse_study_specs(studies: str, fallback, env: str):
             raise typer.Exit(2)
 
         if sep:
-            version = _normalise_version(raw_version, f"for study '{name}'")
+            version = (
+                _synthetic_version(raw_version)
+                if synthetic
+                else _normalise_version(raw_version, f"for study '{name}'")
+            )
         elif fallback is not None:
             version = fallback
+        elif synthetic:
+            version = "all"
         else:
             typer.secho(
                 f"No version for study '{name}': add ':<version>' to it "
@@ -115,7 +146,7 @@ def _parse_study_specs(studies: str, fallback, env: str):
             )
             raise typer.Exit(2)
 
-        specs.append((study_of(name, env).key, version))
+        specs.append((name if synthetic else study_of(name, env).key, version))
 
     if not specs:
         typer.secho("--studies is empty.", fg=typer.colors.RED, err=True)
@@ -141,6 +172,20 @@ def metadata(
              "':version', e.g. 0.9.8, or 'all' for every version.",
     ),
     node: Optional[str] = typer.Option(None, "--node", help="Delete only this node type."),
+    synthetic: bool = typer.Option(
+        False,
+        "--synthetic",
+        help="Registry-free synthetic-data mode: each --studies name is the "
+             "Gen3 project id itself (no SSM study registry). Bare names "
+             "default to version 'all'; a specific version matches records' "
+             "data_version property verbatim.",
+    ),
+    program_id: Optional[str] = typer.Option(
+        None,
+        "--program-id",
+        help="Gen3 program for --synthetic (default: program1). "
+             "Invalid without --synthetic.",
+    ),
     yes: bool = typer.Option(
         False, "--yes", "-y", help="Skip the non-prod prompt (specific-version only)."
     ),
@@ -156,12 +201,24 @@ def metadata(
 
       g3dt delete metadata --studies "ausdiab:0.7.5,cdah:0.8.1" --env staging
       g3dt delete metadata --studies "ausdiab:all,cdah" --version 0.9.8 --env staging
+      g3dt delete metadata --studies "synthetic_dataset_1,synthetic_dataset_2" --env test --synthetic
     """
     env = resolve.active_env(env)
-    fallback = (
-        _normalise_version(version, "for --version") if version is not None else None
-    )
-    specs = _parse_study_specs(studies, fallback, env)
+    if program_id is not None and not synthetic:
+        typer.secho(
+            "--program-id is only valid with --synthetic (registered studies "
+            "carry their program in the study registry).",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+    if version is None:
+        fallback = None
+    elif synthetic:
+        fallback = _synthetic_version(version)
+    else:
+        fallback = _normalise_version(version, "for --version")
+    specs = _parse_study_specs(studies, fallback, env, synthetic=synthetic)
     versions = [v for _, v in specs]
 
     # The typed production confirmation stays the study keys alone: short
@@ -171,13 +228,17 @@ def metadata(
     uniform = len(set(versions)) == 1
     any_all = "all" in versions
 
+    prefix = "synthetic " if synthetic else ""
     if uniform and versions[0] == "all":
-        action = "deletion of ALL VERSIONS"
+        action = f"{prefix}deletion of ALL VERSIONS"
     elif uniform:
-        action = f"deletion of v{versions[0]}"
+        # Synthetic versions are verbatim data_version values (often already
+        # v-prefixed); Athena versions are canonical x.y.z, displayed with v.
+        shown = versions[0] if synthetic else f"v{versions[0]}"
+        action = f"{prefix}deletion of {shown}"
     else:
         plan = ", ".join(f"{key}:{v}" for key, v in specs)
-        action = f"deletion of per-study versions [{plan}]"
+        action = f"{prefix}deletion of per-study versions [{plan}]"
 
     # Deleting every version is the most destructive path: always prompt (pass
     # assume_yes=False so --yes can't bypass it; prod still types the target).
@@ -200,6 +261,11 @@ def metadata(
             ]
         if node:
             a += ["--node", node]
+        if synthetic:
+            # Program is always passed explicitly: the shell default makes it
+            # optional on the wire, but an explicit value keeps the contract
+            # visible in logs and SSM command history.
+            a += ["--synthetic", "--program-id", program_id or "program1"]
         return a
 
     def remote_cli(env_name):
@@ -212,6 +278,12 @@ def metadata(
         a.append("--yes")
         if node:
             a += ["--node", node]
+        if synthetic:
+            # Without this the remote re-entry would re-parse --studies
+            # against the study registry on the box and exit 2.
+            a.append("--synthetic")
+            if program_id is not None:
+                a += ["--program-id", program_id]
         return a
 
     dispatch.run_or_dispatch(

{gen3_dataops_toolkit-4.0.1 → gen3_dataops_toolkit-4.2.0}/src/g3dt/cli/main.py

@@ -20,6 +20,7 @@ from g3dt.cli import (
     metadata,
     pipeline_cmds,
     release_cmds,
+    study_cmds,
     synth,
 )
 
@@ -49,6 +50,7 @@ def _root(
 
 app.add_typer(dict_cmds.app, name="dict")
 app.add_typer(synth.app, name="synth")
+app.add_typer(study_cmds.app, name="study")
 app.add_typer(metadata.app, name="metadata")
 app.add_typer(delete_cmds.app, name="delete")
 app.add_typer(k8s.app, name="k8s")
@@ -64,12 +66,15 @@ _DOCS = """\
 Gen3 DataOps toolkit (g3dt) — operations overview
 =================================================
 
-Configuration: two kinds, nothing else
+Configuration: three kinds, nothing else
   - INPUTS live in your deployment wrapper repo as config/<project>.<env>.json,
     read only by `cdk deploy` (via the aws-gen3-pipeline template).
-  - Everything else is resolved live from SSM (/{project}/{env}/...), which
+  - OUTPUTS are resolved live from SSM (/{project}/{env}/...), which
     `cdk deploy` publishes. The only local file is the g3dt.yaml marker,
     searched at ./g3dt.yaml, ~/.g3dt/g3dt.yaml, /etc/g3dt/g3dt.yaml.
+  - OPERATIONAL state is written by the toolkit itself: the study registry
+    (SSM /{project}/{env}/studies/*, managed with `g3dt study`) and the
+    Iceberg ledgers (releases, upload receipts, indexd registry).
 
 Contexts: what am I pointed at?
   A context is a named (project, env, profile, region) tuple. Every command
@@ -97,14 +102,18 @@ Discover everything
   g3dt config contexts             your contexts (current marked *; add
                                    --verify to check what is deployed)
   g3dt config envs                 environments with a deployed SSM tree
-  g3dt config studies              studies from your g3dt.yaml marker
+  g3dt study list                  the env's study registry (config studies
+                                   is an alias)
   g3dt config show                 resolved settings for the current context
 
 Typical release runbook (staging shown; repeat for prod with care)
   1. g3dt dict deploy   --env staging
-  2. g3dt metadata upload --study <study> --env staging --on ec2
-  3. g3dt jobs logs <run-id> --follow
-  4. g3dt k8s restart-etl --env staging
+  2. g3dt study repoint --latest --env staging   point the registry at the
+                                                 newest release (validates
+                                                 every target first)
+  3. g3dt metadata upload --study <study> --env staging --on ec2
+  4. g3dt jobs logs <run-id> --follow
+  5. g3dt k8s restart-etl --env staging
 
 Promoting one dictionary across environments
   The source repo/path are env inputs; usually only the tag changes, and it