gen3-dataops-toolkit 3.8.0__tar.gz → 4.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/PKG-INFO +20 -13
  2. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/README.md +19 -12
  3. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/pyproject.toml +1 -1
  4. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/__init__.py +1 -1
  5. gen3_dataops_toolkit-4.0.1/src/g3dt/cli/_internal/aws_quiet.py +36 -0
  6. gen3_dataops_toolkit-4.0.1/src/g3dt/cli/_internal/helptext.py +15 -0
  7. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/config_cmds.py +210 -72
  8. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/delete_cmds.py +14 -7
  9. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/dict_cmds.py +23 -11
  10. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/ec2_cmds.py +6 -3
  11. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/indexd_cmds.py +14 -8
  12. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/jobs.py +1 -1
  13. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/k8s.py +23 -9
  14. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/main.py +14 -8
  15. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/metadata.py +10 -6
  16. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/pipeline_cmds.py +26 -8
  17. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/release_cmds.py +18 -8
  18. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/synth.py +107 -105
  19. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/config.py +2 -2
  20. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/contexts.py +2 -2
  21. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/delete/delete_metadata.sh +2 -2
  22. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/dictionary/deploy_dd.sh +8 -8
  23. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +7 -7
  24. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +18 -20
  25. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +59 -7
  26. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/__init__.py +0 -0
  27. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/__init__.py +0 -0
  28. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/dispatch.py +0 -0
  29. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/registry.py +0 -0
  30. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/resolve.py +0 -0
  31. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/runner.py +0 -0
  32. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/safety.py +0 -0
  33. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/indexd/__init__.py +0 -0
  34. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/indexd/file_access.py +0 -0
  35. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/indexd/indexd_registrar.py +0 -0
  36. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/ingest/ingest.py +0 -0
  37. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/resolver.py +0 -0
  38. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
  39. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
  40. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
  41. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
  42. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/indexd/register_indexd.py +0 -0
  43. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/indexd/verify_file_access.py +0 -0
  44. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +0 -0
  45. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +0 -0
  46. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +0 -0
  47. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
  48. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
  49. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
  50. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/upload/metadata/upload_all_studies.sh +0 -0
  51. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/upload/metadata/upload_metadata.py +0 -0
  52. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/__init__.py +0 -0
  53. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/metadata_deleter.py +0 -0
  54. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/metadata_submitter.py +0 -0
  55. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
  56. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/utils/athena_utils.py +0 -0
  57. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/utils/dbt_utils.py +0 -0
  58. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/utils/release_writer.py +0 -0
  59. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/validate/validate.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gen3-dataops-toolkit
3
- Version: 3.8.0
3
+ Version: 4.0.1
4
4
  Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
5
5
  License: Apache-2.0
6
6
  Author: JoshuaHarris391
@@ -59,10 +59,12 @@ pip install gen3-dataops-toolkit
59
59
  ## Bootstrap: discover, then use
60
60
 
61
61
  ```bash
62
- g3dt config discover --all-profiles --add # find every deployed env your
63
- # AWS profiles can see; register
64
- # them as contexts
65
- g3dt config contexts # list them (current marked *)
62
+ g3dt config discover <aws-profile> --add # logs the profile in if needed,
63
+ # lists every deployed env its
64
+ # account holds, registers them
65
+ g3dt config contexts # list them (current marked *);
66
+ # offline — add --verify to check
67
+ # what each account has deployed
66
68
  g3dt config use myproj/test # point g3dt at one
67
69
  ```
68
70
 
@@ -184,8 +186,9 @@ g3dt dict deploy --env staging --version v1.1.7 # same tag, no cdk deploy
184
186
  ```
185
187
 
186
188
  An override does not persist, so `config show` keeps reporting the declared
187
- version until the CDK config catches up — `g3dt config diff --env <env>` reports
188
- exactly that gap and exits 1, so it can gate CI.
189
+ version until the CDK config catches up —
190
+ `g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
191
+ reports exactly that gap and exits 1, so it can gate CI.
189
192
 
190
193
  Synthetic data is only schema-valid against the dictionary that generated it, so
191
194
  `synth generate` records the dictionary version in each batch and `synth upload`
@@ -208,7 +211,7 @@ Only the API key stays local — as a *path* to the file holding it, never the
208
211
  key itself, set once per operator:
209
212
 
210
213
  ```bash
211
- g3dt config set llm_api_key_file ~/.g3dt/anthropic_api_key
214
+ g3dt synth set-key ~/.g3dt/anthropic_api_key
212
215
  g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
213
216
  ```
214
217
 
@@ -242,14 +245,18 @@ step, so it asks for confirmation (decline to skip it, or pass
242
245
  `--skip-delete` to skip without prompting); on a first deploy, when no
243
246
  previous batch exists locally, deletion is skipped automatically.
244
247
 
245
- `deploy` without `--studies` falls back to the original ACDC demo set
246
- (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
247
- at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any
248
- other project should always pass its own `--studies`. The deploy's delete step
249
- targets exactly the studies being regenerated, at `--prev-version`.
248
+ `deploy` requires `--studies`: the list scopes every step — the previous
249
+ batch deleted (at `--prev-version`), the batch generated, and the batch
250
+ uploaded. Stale study directories from an earlier run that happen to sit in
251
+ the same version directory are skipped with a log line, never submitted.
252
+ (Before 4.0.0, omitting `--studies` silently deployed a hardcoded ACDC demo
253
+ set — wrong for every other project.)
250
254
 
251
255
  ### Kubernetes restart targets
252
256
 
257
+ Restarts and dictionary uploads against a production context require typing
258
+ the context name to confirm, the same gate the synth and delete commands use.
259
+
253
260
  `k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
254
261
  restart the commons' schema microservices **serially, in a configured order**,
255
262
  waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
@@ -25,10 +25,12 @@ pip install gen3-dataops-toolkit
25
25
  ## Bootstrap: discover, then use
26
26
 
27
27
  ```bash
28
- g3dt config discover --all-profiles --add # find every deployed env your
29
- # AWS profiles can see; register
30
- # them as contexts
31
- g3dt config contexts # list them (current marked *)
28
+ g3dt config discover <aws-profile> --add # logs the profile in if needed,
29
+ # lists every deployed env its
30
+ # account holds, registers them
31
+ g3dt config contexts # list them (current marked *);
32
+ # offline — add --verify to check
33
+ # what each account has deployed
32
34
  g3dt config use myproj/test # point g3dt at one
33
35
  ```
34
36
 
@@ -150,8 +152,9 @@ g3dt dict deploy --env staging --version v1.1.7 # same tag, no cdk deploy
150
152
  ```
151
153
 
152
154
  An override does not persist, so `config show` keeps reporting the declared
153
- version until the CDK config catches up — `g3dt config diff --env <env>` reports
154
- exactly that gap and exits 1, so it can gate CI.
155
+ version until the CDK config catches up —
156
+ `g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
157
+ reports exactly that gap and exits 1, so it can gate CI.
155
158
 
156
159
  Synthetic data is only schema-valid against the dictionary that generated it, so
157
160
  `synth generate` records the dictionary version in each batch and `synth upload`
@@ -174,7 +177,7 @@ Only the API key stays local — as a *path* to the file holding it, never the
174
177
  key itself, set once per operator:
175
178
 
176
179
  ```bash
177
- g3dt config set llm_api_key_file ~/.g3dt/anthropic_api_key
180
+ g3dt synth set-key ~/.g3dt/anthropic_api_key
178
181
  g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
179
182
  ```
180
183
 
@@ -208,14 +211,18 @@ step, so it asks for confirmation (decline to skip it, or pass
208
211
  `--skip-delete` to skip without prompting); on a first deploy, when no
209
212
  previous batch exists locally, deletion is skipped automatically.
210
213
 
211
- `deploy` without `--studies` falls back to the original ACDC demo set
212
- (`AusDiab_Simulated,Baker-Biobank_Simulated,BioHeart-CT_Simulated,CAUGHT-CAD_Simulated`
213
- at 30,60,20,55 records, previous batch `v1.0.0`) — kept for continuity; any
214
- other project should always pass its own `--studies`. The deploy's delete step
215
- targets exactly the studies being regenerated, at `--prev-version`.
214
+ `deploy` requires `--studies`: the list scopes every step — the previous
215
+ batch deleted (at `--prev-version`), the batch generated, and the batch
216
+ uploaded. Stale study directories from an earlier run that happen to sit in
217
+ the same version directory are skipped with a log line, never submitted.
218
+ (Before 4.0.0, omitting `--studies` silently deployed a hardcoded ACDC demo
219
+ set — wrong for every other project.)
216
220
 
217
221
  ### Kubernetes restart targets
218
222
 
223
+ Restarts and dictionary uploads against a production context require typing
224
+ the context name to confirm, the same gate the synth and delete commands use.
225
+
219
226
  `k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
220
227
  restart the commons' schema microservices **serially, in a configured order**,
221
228
  waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "gen3-dataops-toolkit"
3
- version = "3.8.0"
3
+ version = "4.0.1"
4
4
  description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
@@ -1,4 +1,4 @@
1
- """The ``acdc`` operations CLI.
1
+ """The ``g3dt`` operations CLI.
2
2
 
3
3
  A thin, self-documenting command layer over the project's existing service
4
4
  scripts and library functions. Run ``acdc --help`` or ``acdc docs`` to start.
@@ -0,0 +1,36 @@
1
+ """Scoped botocore log suppression for credential-probing commands.
2
+
3
+ botocore logs its own WARNING tracebacks (with ``exc_info``) when an SSO
4
+ token refresh fails — e.g. ``tokens.py: "SSO token refresh attempt failed"``
5
+ and ``credentials.py: "Refreshing temporary credentials failed"``. Commands
6
+ that deliberately probe possibly-stale profiles (``config contexts
7
+ --verify``, ``config discover``) already render those failures as a tidy
8
+ ``?`` / "skipped" line, so the tracebacks are pure noise that drowns the
9
+ intended output.
10
+
11
+ A permanent ``logging.getLogger("botocore").setLevel(logging.ERROR)`` would
12
+ also hide real botocore warnings for the rest of the process; this context
13
+ manager scopes the silence to the probing block and restores the previous
14
+ levels afterwards, even on error.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import logging
19
+ from contextlib import contextmanager
20
+
21
+ _NOISY_LOGGERS = ("botocore", "boto3", "urllib3")
22
+
23
+
24
+ @contextmanager
25
+ def quiet_botocore():
26
+ """Silence botocore/boto3/urllib3 warnings for the enclosed block."""
27
+ saved = {}
28
+ for name in _NOISY_LOGGERS:
29
+ logger = logging.getLogger(name)
30
+ saved[name] = logger.level
31
+ logger.setLevel(logging.ERROR)
32
+ try:
33
+ yield
34
+ finally:
35
+ for name, level in saved.items():
36
+ logging.getLogger(name).setLevel(level)
@@ -0,0 +1,15 @@
1
+ """Shared help strings for options that appear across command groups.
2
+
3
+ --env is offered by ~20 commands; before this module each declared its own
4
+ phrasing (five variants had accumulated), so the same option read
5
+ differently depending on which command's --help you happened to open. One
6
+ constant keeps the story identical everywhere; append command-specific
7
+ notes with ``+`` where a command genuinely differs.
8
+ """
9
+
10
+ ENV_OPT = (
11
+ "Environment; defaults to the current context "
12
+ "(switch with 'g3dt config use', one-shot with --ctx)."
13
+ )
14
+
15
+ ENV_OPT_SYNTH = ENV_OPT + " Production targets require typed confirmation."
@@ -26,10 +26,17 @@ import typer
26
26
  from g3dt import config, contexts
27
27
  from g3dt.cli._internal import resolve
28
28
  from g3dt.cli._internal.resolve import env_of, study_of
29
+ from g3dt.cli._internal.helptext import ENV_OPT
29
30
 
30
31
  app = typer.Typer(
31
32
  no_args_is_help=True,
32
- help="Contexts (use/discover/contexts), plus the resolved SSM settings.",
33
+ help="Contexts and resolved settings.\n\n"
34
+ "A context is a named (project, env, AWS profile, region) tuple "
35
+ "stored in the local g3dt.yaml marker. Every command acts on the "
36
+ "current context — switch with 'g3dt config use <name>', one-shot "
37
+ "with --ctx. Discovery scans one SSO profile's AWS account at a "
38
+ "time ('g3dt config discover <profile>'). Run 'g3dt docs' for the "
39
+ "full model.",
33
40
  )
34
41
 
35
42
 
@@ -91,6 +98,8 @@ def _print_context_row(ctx: contexts.Context, current: Optional[str],
91
98
  f"region={ctx.region}")
92
99
  if deployed is not None:
93
100
  line += f" deployed={deployed}"
101
+ if deployed == "?" and ctx.profile:
102
+ line += f" (try: aws sso login --profile {ctx.profile})"
94
103
  typer.secho(line, fg=typer.colors.RED if prod else None, bold=prod)
95
104
 
96
105
 
@@ -99,25 +108,55 @@ def _print_context_row(ctx: contexts.Context, current: Optional[str],
99
108
  # --------------------------------------------------------------------------- #
100
109
  @app.command("contexts")
101
110
  def contexts_list(
102
- no_verify: bool = typer.Option(
103
- False, "--no-verify", help="Skip the per-context deployed check (offline)."
111
+ verify: bool = typer.Option(
112
+ False,
113
+ "--verify/--no-verify",
114
+ help="Check each context's deployed toolkit version (one SSM read "
115
+ "per context; needs live AWS credentials). Default: list the "
116
+ "local marker only, fully offline.",
104
117
  ),
105
118
  ) -> None:
106
- """List the configured contexts (current marked with *)."""
119
+ """List the locally registered contexts (current marked with *).
120
+
121
+ Reads only the local g3dt.yaml marker — no AWS/SSO calls are made
122
+ unless --verify is passed.
123
+ """
107
124
  resolve.announce_context()
108
125
  ctxs = contexts.list_contexts()
109
126
  if not ctxs:
110
127
  typer.secho(
111
- "No contexts configured. Run `g3dt config discover --all-profiles "
112
- "--add`, or `g3dt config add <name> --project <p> --env <e> "
113
- "--profile <profile>`.",
128
+ "No contexts configured. Run 'g3dt config discover <aws-profile> "
129
+ "--add', or 'g3dt config add <name> --project <p> --env <e> "
130
+ "--profile <profile>'.",
114
131
  fg=typer.colors.YELLOW,
115
132
  )
116
133
  return
117
134
  current = contexts.current_context_name()
118
- for ctx in ctxs.values():
119
- deployed = None if no_verify else _deployed_version(ctx)
120
- _print_context_row(ctx, current, deployed)
135
+ if verify:
136
+ from g3dt.cli._internal.aws_quiet import quiet_botocore
137
+
138
+ with quiet_botocore():
139
+ for ctx in ctxs.values():
140
+ _print_context_row(ctx, current, _deployed_version(ctx))
141
+ else:
142
+ for ctx in ctxs.values():
143
+ _print_context_row(ctx, current, None)
144
+
145
+
146
+ @app.command("current")
147
+ def current_cmd() -> None:
148
+ """Print the current context name (script-friendly; exit 1 if none)."""
149
+ resolve.announce_context()
150
+ name = contexts.current_context_name()
151
+ if name is None:
152
+ typer.secho(
153
+ "No context selected — run 'g3dt config use <name>' "
154
+ "(see 'g3dt config contexts').",
155
+ fg=typer.colors.YELLOW,
156
+ err=True,
157
+ )
158
+ raise typer.Exit(1)
159
+ typer.echo(name)
121
160
 
122
161
 
123
162
  @app.command("use")
@@ -165,18 +204,20 @@ def use(
165
204
  def add(
166
205
  name: str = typer.Argument(..., help="Context name, e.g. myproj/staging."),
167
206
  project: str = typer.Option(..., "--project", help="Project id, e.g. myproj."),
168
- env: str = typer.Option(..., "--env", help="Base environment, e.g. staging."),
169
- profile: str = typer.Option(
207
+ env: str = typer.Option(..., "--env", "-e", help="Base environment, e.g. staging."),
208
+ profile: Optional[str] = typer.Option(
170
209
  None, "--profile", help="AWS named profile (omit for the ambient chain)."
171
210
  ),
172
- region: str = typer.Option(None, "--region", help="AWS region override."),
211
+ region: Optional[str] = typer.Option(
212
+ None, "--region", "-r", help="AWS region override."
213
+ ),
173
214
  production: bool = typer.Option(
174
215
  False, "--production",
175
216
  help="Mark as production: strict typed confirmation on every "
176
217
  "destructive action.",
177
218
  ),
178
219
  ) -> None:
179
- """Register one context by hand (`config discover --add` is the usual path)."""
220
+ """Register one context by hand ('config discover <profile> --add' is the usual path)."""
180
221
  resolve.announce_context()
181
222
  marker = config.load_marker()
182
223
  ctx = contexts.Context(
@@ -226,77 +267,154 @@ def forget(
226
267
 
227
268
  @app.command("discover")
228
269
  def discover(
270
+ profile_arg: Optional[str] = typer.Argument(
271
+ None, metavar="[PROFILE]",
272
+ help="Scan just this AWS profile's account. If its SSO session is "
273
+ "stale you are offered `aws sso login` first.",
274
+ ),
229
275
  all_profiles: bool = typer.Option(
230
276
  False, "--all-profiles",
231
277
  help="Scan every profile in ~/.aws/config for deployed "
232
- "/{project}/{env} trees.",
278
+ "/{project}/{env} trees (profiles with stale sessions are "
279
+ "skipped, not logged in).",
233
280
  ),
234
281
  add_found: bool = typer.Option(
235
282
  False, "--add", help="Register discovered pairs as contexts."
236
283
  ),
237
284
  profile: List[str] = typer.Option(
238
- [], "--profile", help="Restrict --all-profiles to these profiles."
285
+ [], "--profile", hidden=True,
286
+ help="(Deprecated) Restrict --all-profiles to these profiles.",
239
287
  ),
240
- region: str = typer.Option(
241
- None, "--region", help="Region for profiles that configure none."
288
+ region: Optional[str] = typer.Option(
289
+ None, "--region", "-r", help="Region for profiles that configure none."
242
290
  ),
243
291
  ) -> None:
244
- """Find deployable infrastructure.
245
-
246
- Default: verify each configured context against its own account (nothing
247
- else is touched). With --all-profiles: enumerate every AWS profile and
248
- list all deployed environments its account holds; add --add to register
249
- them as contexts.
292
+ """Scan an AWS profile's account for deployed environments (register with --add).
293
+
294
+ Discovery works one AWS account at a time, through one SSO profile:
295
+ 'g3dt config discover <profile>' checks that profile's session, offers
296
+ 'aws sso login' if it is stale, then lists every deployed environment
297
+ the account holds — the recommended flow. --all-profiles sweeps every
298
+ configured profile instead (stale sessions are skipped, not logged in).
299
+ Run with no arguments to see your profiles and this model. To check
300
+ contexts you already registered: 'g3dt config contexts --verify'.
250
301
  """
251
302
  resolve.announce_context()
252
- # The scan probes credentials that may be stale; botocore logs its own
253
- # WARNING tracebacks for failed SSO refreshes, which would drown the
254
- # per-profile "skipped (run: aws sso login ...)" lines this command
255
- # prints deliberately. Quiet botocore for the duration.
256
- import logging
257
-
258
- logging.getLogger("botocore").setLevel(logging.ERROR)
259
303
  marker = config.load_marker()
260
304
  default_region = region or marker.get("region") or config.DEFAULT_REGION
261
305
 
262
- if not all_profiles:
263
- ctxs = contexts.list_contexts(marker)
264
- if not ctxs:
306
+ if profile_arg and all_profiles:
307
+ typer.secho("Pass either a PROFILE argument or --all-profiles, not both.",
308
+ fg=typer.colors.RED, err=True)
309
+ raise typer.Exit(1)
310
+ if profile and profile_arg:
311
+ typer.secho(
312
+ "Pass either a PROFILE argument or --profile, not both "
313
+ "(--profile is deprecated).",
314
+ fg=typer.colors.RED, err=True,
315
+ )
316
+ raise typer.Exit(1)
317
+ if profile and not all_profiles:
318
+ typer.secho(
319
+ "--profile is deprecated and only valid with --all-profiles; "
320
+ "run 'g3dt config discover <profile>' instead.",
321
+ fg=typer.colors.RED, err=True,
322
+ )
323
+ raise typer.Exit(1)
324
+ if profile:
325
+ typer.secho(
326
+ "--profile is deprecated and will be removed in the next major "
327
+ "release; run 'g3dt config discover <profile>' per profile "
328
+ "instead.",
329
+ fg=typer.colors.YELLOW, err=True,
330
+ )
331
+
332
+ import botocore.session
333
+
334
+ profiles_cfg = botocore.session.Session().full_config.get("profiles", {})
335
+
336
+ if not profile_arg and not all_profiles:
337
+ # Guidance mode: no network — explain the model and show what is
338
+ # available locally, so a first run of the bare command teaches the
339
+ # flow instead of surprising the operator with live AWS calls.
340
+ typer.echo(
341
+ "Discovery scans one AWS account at a time, through one SSO "
342
+ "profile:"
343
+ )
344
+ typer.echo(" g3dt config discover <profile> [--add]")
345
+ if profiles_cfg:
346
+ typer.echo(
347
+ f"Profiles in ~/.aws/config: {', '.join(sorted(profiles_cfg))}"
348
+ )
349
+ else:
265
350
  typer.secho(
266
- "No contexts configured to verify. Try "
267
- "`g3dt config discover --all-profiles [--add]`.",
351
+ "No profiles found in ~/.aws/config — configure AWS SSO "
352
+ "first (aws configure sso).",
268
353
  fg=typer.colors.YELLOW,
269
354
  )
270
- return
271
- for ctx in ctxs.values():
272
- state = _deployed_version(ctx)
273
- if state.startswith("✓"):
274
- label, color = f"OK ({state[2:]})", typer.colors.GREEN
275
- elif state == "—":
276
- label, color = "NOT DEPLOYED", typer.colors.YELLOW
277
- else:
278
- label, color = (
279
- f"AUTH FAILED (try: aws sso login --profile {ctx.profile})",
280
- typer.colors.RED,
281
- )
282
- typer.secho(f" {ctx.name:<24} {label}", fg=color)
355
+ typer.echo(
356
+ "Sweep every profile with --all-profiles (stale SSO sessions "
357
+ "are skipped)."
358
+ )
359
+ typer.echo(
360
+ "To check contexts you already registered: "
361
+ "g3dt config contexts --verify"
362
+ )
283
363
  return
284
364
 
285
- import botocore.session
365
+ from g3dt.cli._internal.aws_quiet import quiet_botocore
286
366
 
287
- profiles_cfg = botocore.session.Session().full_config.get("profiles", {})
288
- names = profile or sorted(profiles_cfg)
289
- if not names:
290
- typer.secho("No AWS profiles found in ~/.aws/config.",
291
- fg=typer.colors.YELLOW)
292
- return
367
+ if profile_arg:
368
+ if profile_arg not in profiles_cfg:
369
+ typer.secho(
370
+ f"No profile '{profile_arg}' in ~/.aws/config. Configured: "
371
+ f"{', '.join(sorted(profiles_cfg)) or '(none)'}.",
372
+ fg=typer.colors.RED, err=True,
373
+ )
374
+ raise typer.Exit(1)
375
+ with quiet_botocore():
376
+ session_ok = _profile_session_valid(profile_arg)
377
+ if not session_ok:
378
+ typer.secho(
379
+ f"The SSO session for '{profile_arg}' is not valid.",
380
+ fg=typer.colors.YELLOW,
381
+ )
382
+ if not typer.confirm(
383
+ f"Run `aws sso login --profile {profile_arg}` now?",
384
+ default=True,
385
+ ):
386
+ typer.secho("Aborted — log in and re-run discover.",
387
+ fg=typer.colors.YELLOW)
388
+ raise typer.Exit(1)
389
+ import subprocess
390
+
391
+ login = subprocess.run(
392
+ ["aws", "sso", "login", "--profile", profile_arg]
393
+ )
394
+ with quiet_botocore():
395
+ session_ok = _profile_session_valid(profile_arg)
396
+ if login.returncode != 0 or not session_ok:
397
+ typer.secho(
398
+ f"Still no valid session for '{profile_arg}'. Aborting.",
399
+ fg=typer.colors.RED, err=True,
400
+ )
401
+ raise typer.Exit(1)
402
+ names = [profile_arg]
403
+ else:
404
+ names = profile or sorted(profiles_cfg)
405
+ if not names:
406
+ typer.secho("No AWS profiles found in ~/.aws/config.",
407
+ fg=typer.colors.YELLOW)
408
+ return
293
409
 
294
410
  found: List[contexts.Context] = []
411
+ seen_names = set()
295
412
  for prof in names:
296
413
  prof_region = (profiles_cfg.get(prof, {}).get("region")
297
414
  or default_region)
298
415
  try:
299
- pairs = _scan_profile(prof, prof_region)
416
+ with quiet_botocore():
417
+ pairs = _scan_profile(prof, prof_region)
300
418
  except Exception as exc:
301
419
  typer.secho(
302
420
  f" {prof}: skipped ({type(exc).__name__} — "
@@ -309,6 +427,12 @@ def discover(
309
427
  continue
310
428
  for project, env in pairs:
311
429
  name = f"{project}/{env}"
430
+ if name in seen_names:
431
+ # Same account visible through several profiles (e.g. a
432
+ # 'default' alias): the first profile wins; don't print
433
+ # duplicate suggestions.
434
+ continue
435
+ seen_names.add(name)
312
436
  ctx = contexts.Context(
313
437
  name=name, project=project, env=env, profile=prof,
314
438
  region=prof_region,
@@ -340,6 +464,23 @@ def discover(
340
464
  )
341
465
 
342
466
 
467
+ def _profile_session_valid(profile: str) -> bool:
468
+ """Cheap credential check: can this profile sign a request right now?"""
469
+ import boto3
470
+ from botocore.config import Config as _BotoConfig
471
+
472
+ try:
473
+ session = boto3.Session(profile_name=profile)
474
+ session.client(
475
+ "sts",
476
+ config=_BotoConfig(connect_timeout=3, read_timeout=5,
477
+ retries={"max_attempts": 1}),
478
+ ).get_caller_identity()
479
+ return True
480
+ except Exception:
481
+ return False
482
+
483
+
343
484
  def _scan_profile(profile: str, region: str) -> List[tuple]:
344
485
  """All deployed (project, env) pairs visible to one profile's account.
345
486
 
@@ -387,11 +528,14 @@ def envs() -> None:
387
528
 
388
529
  @app.command()
389
530
  def studies(
390
- env: str = typer.Option(
531
+ env: Optional[str] = typer.Option(
391
532
  None,
392
533
  "--env",
393
534
  "-e",
394
- help="Also check the env's S3 registry (s3://<metadata-bucket>/config/studies.yaml).",
535
+ help="Read the study registry from this env's S3 bucket "
536
+ "(s3://<metadata-bucket>/config/studies.yaml) instead of the "
537
+ "local marker — this changes the data source, not just the "
538
+ "target.",
395
539
  ),
396
540
  ) -> None:
397
541
  """List the configured studies (bare names).
@@ -418,11 +562,8 @@ def studies(
418
562
 
419
563
  @app.command()
420
564
  def show(
421
- env: str = typer.Option(
422
- None, "--env", "-e",
423
- help="Environment; defaults to the current context.",
424
- ),
425
- study: str = typer.Option(
565
+ env: Optional[str] = typer.Option(None, "--env", "-e", help=ENV_OPT),
566
+ study: Optional[str] = typer.Option(
426
567
  None, "--study", "-s", help="Optional study to resolve against the env."
427
568
  ),
428
569
  full: bool = typer.Option(
@@ -480,14 +621,12 @@ def show(
480
621
 
481
622
  @app.command()
482
623
  def diff(
483
- env: str = typer.Option(
484
- None, "--env", "-e",
485
- help="Environment; defaults to the current context.",
486
- ),
624
+ env: Optional[str] = typer.Option(None, "--env", "-e", help=ENV_OPT),
487
625
  file: Path = typer.Option(
488
626
  ...,
489
627
  "--file",
490
- "-f",
628
+ exists=True,
629
+ dir_okay=False,
491
630
  help="The env's INPUT file in the deployment wrapper, e.g. "
492
631
  "../myproj-pipeline-deploy/config/myproj.staging.json.",
493
632
  ),
@@ -565,10 +704,9 @@ def diff(
565
704
 
566
705
  @app.command("dbt-env")
567
706
  def dbt_env(
568
- env: str = typer.Option(
707
+ env: Optional[str] = typer.Option(
569
708
  None, "--env", "-e",
570
- help="Environment; defaults to the current context. CodeBuild always "
571
- "passes this explicitly.",
709
+ help=ENV_OPT + " CodeBuild always passes this explicitly.",
572
710
  ),
573
711
  ) -> None:
574
712
  """Emit `export` lines for the env's dbt settings (resolved from SSM).