gen3-dataops-toolkit 3.8.0__tar.gz → 4.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/PKG-INFO +20 -13
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/README.md +19 -12
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/pyproject.toml +1 -1
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/__init__.py +1 -1
- gen3_dataops_toolkit-4.0.1/src/g3dt/cli/_internal/aws_quiet.py +36 -0
- gen3_dataops_toolkit-4.0.1/src/g3dt/cli/_internal/helptext.py +15 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/config_cmds.py +210 -72
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/delete_cmds.py +14 -7
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/dict_cmds.py +23 -11
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/ec2_cmds.py +6 -3
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/indexd_cmds.py +14 -8
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/jobs.py +1 -1
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/k8s.py +23 -9
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/main.py +14 -8
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/metadata.py +10 -6
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/pipeline_cmds.py +26 -8
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/release_cmds.py +18 -8
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/synth.py +107 -105
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/config.py +2 -2
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/contexts.py +2 -2
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/delete/delete_metadata.sh +2 -2
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/dictionary/deploy_dd.sh +8 -8
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +7 -7
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +18 -20
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +59 -7
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/dispatch.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/registry.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/resolve.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/runner.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/cli/_internal/safety.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/indexd/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/indexd/file_access.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/indexd/indexd_registrar.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/ingest/ingest.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/resolver.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/indexd/register_indexd.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/indexd/verify_file_access.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/upload/metadata/upload_all_studies.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/services/upload/metadata/upload_metadata.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/metadata_deleter.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/metadata_submitter.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/utils/athena_utils.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/utils/dbt_utils.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/utils/release_writer.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-4.0.1}/src/g3dt/validate/validate.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gen3-dataops-toolkit
|
|
3
|
-
Version:
|
|
3
|
+
Version: 4.0.1
|
|
4
4
|
Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: JoshuaHarris391
|
|
@@ -59,10 +59,12 @@ pip install gen3-dataops-toolkit
|
|
|
59
59
|
## Bootstrap: discover, then use
|
|
60
60
|
|
|
61
61
|
```bash
|
|
62
|
-
g3dt config discover
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
g3dt config contexts # list them (current marked *)
|
|
62
|
+
g3dt config discover <aws-profile> --add # logs the profile in if needed,
|
|
63
|
+
# lists every deployed env its
|
|
64
|
+
# account holds, registers them
|
|
65
|
+
g3dt config contexts # list them (current marked *);
|
|
66
|
+
# offline — add --verify to check
|
|
67
|
+
# what each account has deployed
|
|
66
68
|
g3dt config use myproj/test # point g3dt at one
|
|
67
69
|
```
|
|
68
70
|
|
|
@@ -184,8 +186,9 @@ g3dt dict deploy --env staging --version v1.1.7 # same tag, no cdk deploy
|
|
|
184
186
|
```
|
|
185
187
|
|
|
186
188
|
An override does not persist, so `config show` keeps reporting the declared
|
|
187
|
-
version until the CDK config catches up —
|
|
188
|
-
|
|
189
|
+
version until the CDK config catches up —
|
|
190
|
+
`g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
|
|
191
|
+
reports exactly that gap and exits 1, so it can gate CI.
|
|
189
192
|
|
|
190
193
|
Synthetic data is only schema-valid against the dictionary that generated it, so
|
|
191
194
|
`synth generate` records the dictionary version in each batch and `synth upload`
|
|
@@ -208,7 +211,7 @@ Only the API key stays local — as a *path* to the file holding it, never the
|
|
|
208
211
|
key itself, set once per operator:
|
|
209
212
|
|
|
210
213
|
```bash
|
|
211
|
-
g3dt
|
|
214
|
+
g3dt synth set-key ~/.g3dt/anthropic_api_key
|
|
212
215
|
g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
|
|
213
216
|
```
|
|
214
217
|
|
|
@@ -242,14 +245,18 @@ step, so it asks for confirmation (decline to skip it, or pass
|
|
|
242
245
|
`--skip-delete` to skip without prompting); on a first deploy, when no
|
|
243
246
|
previous batch exists locally, deletion is skipped automatically.
|
|
244
247
|
|
|
245
|
-
`deploy`
|
|
246
|
-
(`
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
248
|
+
`deploy` requires `--studies`: the list scopes every step — the previous
|
|
249
|
+
batch deleted (at `--prev-version`), the batch generated, and the batch
|
|
250
|
+
uploaded. Stale study directories from an earlier run that happen to sit in
|
|
251
|
+
the same version directory are skipped with a log line, never submitted.
|
|
252
|
+
(Before 4.0.0, omitting `--studies` silently deployed a hardcoded ACDC demo
|
|
253
|
+
set — wrong for every other project.)
|
|
250
254
|
|
|
251
255
|
### Kubernetes restart targets
|
|
252
256
|
|
|
257
|
+
Restarts and dictionary uploads against a production context require typing
|
|
258
|
+
the context name to confirm, the same gate the synth and delete commands use.
|
|
259
|
+
|
|
253
260
|
`k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
|
|
254
261
|
restart the commons' schema microservices **serially, in a configured order**,
|
|
255
262
|
waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
|
|
@@ -25,10 +25,12 @@ pip install gen3-dataops-toolkit
|
|
|
25
25
|
## Bootstrap: discover, then use
|
|
26
26
|
|
|
27
27
|
```bash
|
|
28
|
-
g3dt config discover
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
g3dt config contexts # list them (current marked *)
|
|
28
|
+
g3dt config discover <aws-profile> --add # logs the profile in if needed,
|
|
29
|
+
# lists every deployed env its
|
|
30
|
+
# account holds, registers them
|
|
31
|
+
g3dt config contexts # list them (current marked *);
|
|
32
|
+
# offline — add --verify to check
|
|
33
|
+
# what each account has deployed
|
|
32
34
|
g3dt config use myproj/test # point g3dt at one
|
|
33
35
|
```
|
|
34
36
|
|
|
@@ -150,8 +152,9 @@ g3dt dict deploy --env staging --version v1.1.7 # same tag, no cdk deploy
|
|
|
150
152
|
```
|
|
151
153
|
|
|
152
154
|
An override does not persist, so `config show` keeps reporting the declared
|
|
153
|
-
version until the CDK config catches up —
|
|
154
|
-
|
|
155
|
+
version until the CDK config catches up —
|
|
156
|
+
`g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
|
|
157
|
+
reports exactly that gap and exits 1, so it can gate CI.
|
|
155
158
|
|
|
156
159
|
Synthetic data is only schema-valid against the dictionary that generated it, so
|
|
157
160
|
`synth generate` records the dictionary version in each batch and `synth upload`
|
|
@@ -174,7 +177,7 @@ Only the API key stays local — as a *path* to the file holding it, never the
|
|
|
174
177
|
key itself, set once per operator:
|
|
175
178
|
|
|
176
179
|
```bash
|
|
177
|
-
g3dt
|
|
180
|
+
g3dt synth set-key ~/.g3dt/anthropic_api_key
|
|
178
181
|
g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
|
|
179
182
|
```
|
|
180
183
|
|
|
@@ -208,14 +211,18 @@ step, so it asks for confirmation (decline to skip it, or pass
|
|
|
208
211
|
`--skip-delete` to skip without prompting); on a first deploy, when no
|
|
209
212
|
previous batch exists locally, deletion is skipped automatically.
|
|
210
213
|
|
|
211
|
-
`deploy`
|
|
212
|
-
(`
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
214
|
+
`deploy` requires `--studies`: the list scopes every step — the previous
|
|
215
|
+
batch deleted (at `--prev-version`), the batch generated, and the batch
|
|
216
|
+
uploaded. Stale study directories from an earlier run that happen to sit in
|
|
217
|
+
the same version directory are skipped with a log line, never submitted.
|
|
218
|
+
(Before 4.0.0, omitting `--studies` silently deployed a hardcoded ACDC demo
|
|
219
|
+
set — wrong for every other project.)
|
|
216
220
|
|
|
217
221
|
### Kubernetes restart targets
|
|
218
222
|
|
|
223
|
+
Restarts and dictionary uploads against a production context require typing
|
|
224
|
+
the context name to confirm, the same gate the synth and delete commands use.
|
|
225
|
+
|
|
219
226
|
`k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
|
|
220
227
|
restart the commons' schema microservices **serially, in a configured order**,
|
|
221
228
|
waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Scoped botocore log suppression for credential-probing commands.
|
|
2
|
+
|
|
3
|
+
botocore logs its own WARNING tracebacks (with ``exc_info``) when an SSO
|
|
4
|
+
token refresh fails — e.g. ``tokens.py: "SSO token refresh attempt failed"``
|
|
5
|
+
and ``credentials.py: "Refreshing temporary credentials failed"``. Commands
|
|
6
|
+
that deliberately probe possibly-stale profiles (``config contexts
|
|
7
|
+
--verify``, ``config discover``) already render those failures as a tidy
|
|
8
|
+
``?`` / "skipped" line, so the tracebacks are pure noise that drowns the
|
|
9
|
+
intended output.
|
|
10
|
+
|
|
11
|
+
A permanent ``logging.getLogger("botocore").setLevel(logging.ERROR)`` would
|
|
12
|
+
also hide real botocore warnings for the rest of the process; this context
|
|
13
|
+
manager scopes the silence to the probing block and restores the previous
|
|
14
|
+
levels afterwards, even on error.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
|
|
21
|
+
_NOISY_LOGGERS = ("botocore", "boto3", "urllib3")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@contextmanager
|
|
25
|
+
def quiet_botocore():
|
|
26
|
+
"""Silence botocore/boto3/urllib3 warnings for the enclosed block."""
|
|
27
|
+
saved = {}
|
|
28
|
+
for name in _NOISY_LOGGERS:
|
|
29
|
+
logger = logging.getLogger(name)
|
|
30
|
+
saved[name] = logger.level
|
|
31
|
+
logger.setLevel(logging.ERROR)
|
|
32
|
+
try:
|
|
33
|
+
yield
|
|
34
|
+
finally:
|
|
35
|
+
for name, level in saved.items():
|
|
36
|
+
logging.getLogger(name).setLevel(level)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Shared help strings for options that appear across command groups.
|
|
2
|
+
|
|
3
|
+
--env is offered by ~20 commands; before this module each declared its own
|
|
4
|
+
phrasing (five variants had accumulated), so the same option read
|
|
5
|
+
differently depending on which command's --help you happened to open. One
|
|
6
|
+
constant keeps the story identical everywhere; append command-specific
|
|
7
|
+
notes with ``+`` where a command genuinely differs.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
ENV_OPT = (
|
|
11
|
+
"Environment; defaults to the current context "
|
|
12
|
+
"(switch with 'g3dt config use', one-shot with --ctx)."
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
ENV_OPT_SYNTH = ENV_OPT + " Production targets require typed confirmation."
|
|
@@ -26,10 +26,17 @@ import typer
|
|
|
26
26
|
from g3dt import config, contexts
|
|
27
27
|
from g3dt.cli._internal import resolve
|
|
28
28
|
from g3dt.cli._internal.resolve import env_of, study_of
|
|
29
|
+
from g3dt.cli._internal.helptext import ENV_OPT
|
|
29
30
|
|
|
30
31
|
app = typer.Typer(
|
|
31
32
|
no_args_is_help=True,
|
|
32
|
-
help="Contexts
|
|
33
|
+
help="Contexts and resolved settings.\n\n"
|
|
34
|
+
"A context is a named (project, env, AWS profile, region) tuple "
|
|
35
|
+
"stored in the local g3dt.yaml marker. Every command acts on the "
|
|
36
|
+
"current context — switch with 'g3dt config use <name>', one-shot "
|
|
37
|
+
"with --ctx. Discovery scans one SSO profile's AWS account at a "
|
|
38
|
+
"time ('g3dt config discover <profile>'). Run 'g3dt docs' for the "
|
|
39
|
+
"full model.",
|
|
33
40
|
)
|
|
34
41
|
|
|
35
42
|
|
|
@@ -91,6 +98,8 @@ def _print_context_row(ctx: contexts.Context, current: Optional[str],
|
|
|
91
98
|
f"region={ctx.region}")
|
|
92
99
|
if deployed is not None:
|
|
93
100
|
line += f" deployed={deployed}"
|
|
101
|
+
if deployed == "?" and ctx.profile:
|
|
102
|
+
line += f" (try: aws sso login --profile {ctx.profile})"
|
|
94
103
|
typer.secho(line, fg=typer.colors.RED if prod else None, bold=prod)
|
|
95
104
|
|
|
96
105
|
|
|
@@ -99,25 +108,55 @@ def _print_context_row(ctx: contexts.Context, current: Optional[str],
|
|
|
99
108
|
# --------------------------------------------------------------------------- #
|
|
100
109
|
@app.command("contexts")
|
|
101
110
|
def contexts_list(
|
|
102
|
-
|
|
103
|
-
False,
|
|
111
|
+
verify: bool = typer.Option(
|
|
112
|
+
False,
|
|
113
|
+
"--verify/--no-verify",
|
|
114
|
+
help="Check each context's deployed toolkit version (one SSM read "
|
|
115
|
+
"per context; needs live AWS credentials). Default: list the "
|
|
116
|
+
"local marker only, fully offline.",
|
|
104
117
|
),
|
|
105
118
|
) -> None:
|
|
106
|
-
"""List the
|
|
119
|
+
"""List the locally registered contexts (current marked with *).
|
|
120
|
+
|
|
121
|
+
Reads only the local g3dt.yaml marker — no AWS/SSO calls are made
|
|
122
|
+
unless --verify is passed.
|
|
123
|
+
"""
|
|
107
124
|
resolve.announce_context()
|
|
108
125
|
ctxs = contexts.list_contexts()
|
|
109
126
|
if not ctxs:
|
|
110
127
|
typer.secho(
|
|
111
|
-
"No contexts configured. Run
|
|
112
|
-
"--add
|
|
113
|
-
"--profile <profile
|
|
128
|
+
"No contexts configured. Run 'g3dt config discover <aws-profile> "
|
|
129
|
+
"--add', or 'g3dt config add <name> --project <p> --env <e> "
|
|
130
|
+
"--profile <profile>'.",
|
|
114
131
|
fg=typer.colors.YELLOW,
|
|
115
132
|
)
|
|
116
133
|
return
|
|
117
134
|
current = contexts.current_context_name()
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
135
|
+
if verify:
|
|
136
|
+
from g3dt.cli._internal.aws_quiet import quiet_botocore
|
|
137
|
+
|
|
138
|
+
with quiet_botocore():
|
|
139
|
+
for ctx in ctxs.values():
|
|
140
|
+
_print_context_row(ctx, current, _deployed_version(ctx))
|
|
141
|
+
else:
|
|
142
|
+
for ctx in ctxs.values():
|
|
143
|
+
_print_context_row(ctx, current, None)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@app.command("current")
|
|
147
|
+
def current_cmd() -> None:
|
|
148
|
+
"""Print the current context name (script-friendly; exit 1 if none)."""
|
|
149
|
+
resolve.announce_context()
|
|
150
|
+
name = contexts.current_context_name()
|
|
151
|
+
if name is None:
|
|
152
|
+
typer.secho(
|
|
153
|
+
"No context selected — run 'g3dt config use <name>' "
|
|
154
|
+
"(see 'g3dt config contexts').",
|
|
155
|
+
fg=typer.colors.YELLOW,
|
|
156
|
+
err=True,
|
|
157
|
+
)
|
|
158
|
+
raise typer.Exit(1)
|
|
159
|
+
typer.echo(name)
|
|
121
160
|
|
|
122
161
|
|
|
123
162
|
@app.command("use")
|
|
@@ -165,18 +204,20 @@ def use(
|
|
|
165
204
|
def add(
|
|
166
205
|
name: str = typer.Argument(..., help="Context name, e.g. myproj/staging."),
|
|
167
206
|
project: str = typer.Option(..., "--project", help="Project id, e.g. myproj."),
|
|
168
|
-
env: str = typer.Option(..., "--env", help="Base environment, e.g. staging."),
|
|
169
|
-
profile: str = typer.Option(
|
|
207
|
+
env: str = typer.Option(..., "--env", "-e", help="Base environment, e.g. staging."),
|
|
208
|
+
profile: Optional[str] = typer.Option(
|
|
170
209
|
None, "--profile", help="AWS named profile (omit for the ambient chain)."
|
|
171
210
|
),
|
|
172
|
-
region: str = typer.Option(
|
|
211
|
+
region: Optional[str] = typer.Option(
|
|
212
|
+
None, "--region", "-r", help="AWS region override."
|
|
213
|
+
),
|
|
173
214
|
production: bool = typer.Option(
|
|
174
215
|
False, "--production",
|
|
175
216
|
help="Mark as production: strict typed confirmation on every "
|
|
176
217
|
"destructive action.",
|
|
177
218
|
),
|
|
178
219
|
) -> None:
|
|
179
|
-
"""Register one context by hand (
|
|
220
|
+
"""Register one context by hand ('config discover <profile> --add' is the usual path)."""
|
|
180
221
|
resolve.announce_context()
|
|
181
222
|
marker = config.load_marker()
|
|
182
223
|
ctx = contexts.Context(
|
|
@@ -226,77 +267,154 @@ def forget(
|
|
|
226
267
|
|
|
227
268
|
@app.command("discover")
|
|
228
269
|
def discover(
|
|
270
|
+
profile_arg: Optional[str] = typer.Argument(
|
|
271
|
+
None, metavar="[PROFILE]",
|
|
272
|
+
help="Scan just this AWS profile's account. If its SSO session is "
|
|
273
|
+
"stale you are offered `aws sso login` first.",
|
|
274
|
+
),
|
|
229
275
|
all_profiles: bool = typer.Option(
|
|
230
276
|
False, "--all-profiles",
|
|
231
277
|
help="Scan every profile in ~/.aws/config for deployed "
|
|
232
|
-
"/{project}/{env} trees
|
|
278
|
+
"/{project}/{env} trees (profiles with stale sessions are "
|
|
279
|
+
"skipped, not logged in).",
|
|
233
280
|
),
|
|
234
281
|
add_found: bool = typer.Option(
|
|
235
282
|
False, "--add", help="Register discovered pairs as contexts."
|
|
236
283
|
),
|
|
237
284
|
profile: List[str] = typer.Option(
|
|
238
|
-
[], "--profile",
|
|
285
|
+
[], "--profile", hidden=True,
|
|
286
|
+
help="(Deprecated) Restrict --all-profiles to these profiles.",
|
|
239
287
|
),
|
|
240
|
-
region: str = typer.Option(
|
|
241
|
-
None, "--region", help="Region for profiles that configure none."
|
|
288
|
+
region: Optional[str] = typer.Option(
|
|
289
|
+
None, "--region", "-r", help="Region for profiles that configure none."
|
|
242
290
|
),
|
|
243
291
|
) -> None:
|
|
244
|
-
"""
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
292
|
+
"""Scan an AWS profile's account for deployed environments (register with --add).
|
|
293
|
+
|
|
294
|
+
Discovery works one AWS account at a time, through one SSO profile:
|
|
295
|
+
'g3dt config discover <profile>' checks that profile's session, offers
|
|
296
|
+
'aws sso login' if it is stale, then lists every deployed environment
|
|
297
|
+
the account holds — the recommended flow. --all-profiles sweeps every
|
|
298
|
+
configured profile instead (stale sessions are skipped, not logged in).
|
|
299
|
+
Run with no arguments to see your profiles and this model. To check
|
|
300
|
+
contexts you already registered: 'g3dt config contexts --verify'.
|
|
250
301
|
"""
|
|
251
302
|
resolve.announce_context()
|
|
252
|
-
# The scan probes credentials that may be stale; botocore logs its own
|
|
253
|
-
# WARNING tracebacks for failed SSO refreshes, which would drown the
|
|
254
|
-
# per-profile "skipped (run: aws sso login ...)" lines this command
|
|
255
|
-
# prints deliberately. Quiet botocore for the duration.
|
|
256
|
-
import logging
|
|
257
|
-
|
|
258
|
-
logging.getLogger("botocore").setLevel(logging.ERROR)
|
|
259
303
|
marker = config.load_marker()
|
|
260
304
|
default_region = region or marker.get("region") or config.DEFAULT_REGION
|
|
261
305
|
|
|
262
|
-
if
|
|
263
|
-
|
|
264
|
-
|
|
306
|
+
if profile_arg and all_profiles:
|
|
307
|
+
typer.secho("Pass either a PROFILE argument or --all-profiles, not both.",
|
|
308
|
+
fg=typer.colors.RED, err=True)
|
|
309
|
+
raise typer.Exit(1)
|
|
310
|
+
if profile and profile_arg:
|
|
311
|
+
typer.secho(
|
|
312
|
+
"Pass either a PROFILE argument or --profile, not both "
|
|
313
|
+
"(--profile is deprecated).",
|
|
314
|
+
fg=typer.colors.RED, err=True,
|
|
315
|
+
)
|
|
316
|
+
raise typer.Exit(1)
|
|
317
|
+
if profile and not all_profiles:
|
|
318
|
+
typer.secho(
|
|
319
|
+
"--profile is deprecated and only valid with --all-profiles; "
|
|
320
|
+
"run 'g3dt config discover <profile>' instead.",
|
|
321
|
+
fg=typer.colors.RED, err=True,
|
|
322
|
+
)
|
|
323
|
+
raise typer.Exit(1)
|
|
324
|
+
if profile:
|
|
325
|
+
typer.secho(
|
|
326
|
+
"--profile is deprecated and will be removed in the next major "
|
|
327
|
+
"release; run 'g3dt config discover <profile>' per profile "
|
|
328
|
+
"instead.",
|
|
329
|
+
fg=typer.colors.YELLOW, err=True,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
import botocore.session
|
|
333
|
+
|
|
334
|
+
profiles_cfg = botocore.session.Session().full_config.get("profiles", {})
|
|
335
|
+
|
|
336
|
+
if not profile_arg and not all_profiles:
|
|
337
|
+
# Guidance mode: no network — explain the model and show what is
|
|
338
|
+
# available locally, so a first run of the bare command teaches the
|
|
339
|
+
# flow instead of surprising the operator with live AWS calls.
|
|
340
|
+
typer.echo(
|
|
341
|
+
"Discovery scans one AWS account at a time, through one SSO "
|
|
342
|
+
"profile:"
|
|
343
|
+
)
|
|
344
|
+
typer.echo(" g3dt config discover <profile> [--add]")
|
|
345
|
+
if profiles_cfg:
|
|
346
|
+
typer.echo(
|
|
347
|
+
f"Profiles in ~/.aws/config: {', '.join(sorted(profiles_cfg))}"
|
|
348
|
+
)
|
|
349
|
+
else:
|
|
265
350
|
typer.secho(
|
|
266
|
-
"No
|
|
267
|
-
"
|
|
351
|
+
"No profiles found in ~/.aws/config — configure AWS SSO "
|
|
352
|
+
"first (aws configure sso).",
|
|
268
353
|
fg=typer.colors.YELLOW,
|
|
269
354
|
)
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
label, color = (
|
|
279
|
-
f"AUTH FAILED (try: aws sso login --profile {ctx.profile})",
|
|
280
|
-
typer.colors.RED,
|
|
281
|
-
)
|
|
282
|
-
typer.secho(f" {ctx.name:<24} {label}", fg=color)
|
|
355
|
+
typer.echo(
|
|
356
|
+
"Sweep every profile with --all-profiles (stale SSO sessions "
|
|
357
|
+
"are skipped)."
|
|
358
|
+
)
|
|
359
|
+
typer.echo(
|
|
360
|
+
"To check contexts you already registered: "
|
|
361
|
+
"g3dt config contexts --verify"
|
|
362
|
+
)
|
|
283
363
|
return
|
|
284
364
|
|
|
285
|
-
import
|
|
365
|
+
from g3dt.cli._internal.aws_quiet import quiet_botocore
|
|
286
366
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
367
|
+
if profile_arg:
|
|
368
|
+
if profile_arg not in profiles_cfg:
|
|
369
|
+
typer.secho(
|
|
370
|
+
f"No profile '{profile_arg}' in ~/.aws/config. Configured: "
|
|
371
|
+
f"{', '.join(sorted(profiles_cfg)) or '(none)'}.",
|
|
372
|
+
fg=typer.colors.RED, err=True,
|
|
373
|
+
)
|
|
374
|
+
raise typer.Exit(1)
|
|
375
|
+
with quiet_botocore():
|
|
376
|
+
session_ok = _profile_session_valid(profile_arg)
|
|
377
|
+
if not session_ok:
|
|
378
|
+
typer.secho(
|
|
379
|
+
f"The SSO session for '{profile_arg}' is not valid.",
|
|
380
|
+
fg=typer.colors.YELLOW,
|
|
381
|
+
)
|
|
382
|
+
if not typer.confirm(
|
|
383
|
+
f"Run `aws sso login --profile {profile_arg}` now?",
|
|
384
|
+
default=True,
|
|
385
|
+
):
|
|
386
|
+
typer.secho("Aborted — log in and re-run discover.",
|
|
387
|
+
fg=typer.colors.YELLOW)
|
|
388
|
+
raise typer.Exit(1)
|
|
389
|
+
import subprocess
|
|
390
|
+
|
|
391
|
+
login = subprocess.run(
|
|
392
|
+
["aws", "sso", "login", "--profile", profile_arg]
|
|
393
|
+
)
|
|
394
|
+
with quiet_botocore():
|
|
395
|
+
session_ok = _profile_session_valid(profile_arg)
|
|
396
|
+
if login.returncode != 0 or not session_ok:
|
|
397
|
+
typer.secho(
|
|
398
|
+
f"Still no valid session for '{profile_arg}'. Aborting.",
|
|
399
|
+
fg=typer.colors.RED, err=True,
|
|
400
|
+
)
|
|
401
|
+
raise typer.Exit(1)
|
|
402
|
+
names = [profile_arg]
|
|
403
|
+
else:
|
|
404
|
+
names = profile or sorted(profiles_cfg)
|
|
405
|
+
if not names:
|
|
406
|
+
typer.secho("No AWS profiles found in ~/.aws/config.",
|
|
407
|
+
fg=typer.colors.YELLOW)
|
|
408
|
+
return
|
|
293
409
|
|
|
294
410
|
found: List[contexts.Context] = []
|
|
411
|
+
seen_names = set()
|
|
295
412
|
for prof in names:
|
|
296
413
|
prof_region = (profiles_cfg.get(prof, {}).get("region")
|
|
297
414
|
or default_region)
|
|
298
415
|
try:
|
|
299
|
-
|
|
416
|
+
with quiet_botocore():
|
|
417
|
+
pairs = _scan_profile(prof, prof_region)
|
|
300
418
|
except Exception as exc:
|
|
301
419
|
typer.secho(
|
|
302
420
|
f" {prof}: skipped ({type(exc).__name__} — "
|
|
@@ -309,6 +427,12 @@ def discover(
|
|
|
309
427
|
continue
|
|
310
428
|
for project, env in pairs:
|
|
311
429
|
name = f"{project}/{env}"
|
|
430
|
+
if name in seen_names:
|
|
431
|
+
# Same account visible through several profiles (e.g. a
|
|
432
|
+
# 'default' alias): the first profile wins; don't print
|
|
433
|
+
# duplicate suggestions.
|
|
434
|
+
continue
|
|
435
|
+
seen_names.add(name)
|
|
312
436
|
ctx = contexts.Context(
|
|
313
437
|
name=name, project=project, env=env, profile=prof,
|
|
314
438
|
region=prof_region,
|
|
@@ -340,6 +464,23 @@ def discover(
|
|
|
340
464
|
)
|
|
341
465
|
|
|
342
466
|
|
|
467
|
+
def _profile_session_valid(profile: str) -> bool:
|
|
468
|
+
"""Cheap credential check: can this profile sign a request right now?"""
|
|
469
|
+
import boto3
|
|
470
|
+
from botocore.config import Config as _BotoConfig
|
|
471
|
+
|
|
472
|
+
try:
|
|
473
|
+
session = boto3.Session(profile_name=profile)
|
|
474
|
+
session.client(
|
|
475
|
+
"sts",
|
|
476
|
+
config=_BotoConfig(connect_timeout=3, read_timeout=5,
|
|
477
|
+
retries={"max_attempts": 1}),
|
|
478
|
+
).get_caller_identity()
|
|
479
|
+
return True
|
|
480
|
+
except Exception:
|
|
481
|
+
return False
|
|
482
|
+
|
|
483
|
+
|
|
343
484
|
def _scan_profile(profile: str, region: str) -> List[tuple]:
|
|
344
485
|
"""All deployed (project, env) pairs visible to one profile's account.
|
|
345
486
|
|
|
@@ -387,11 +528,14 @@ def envs() -> None:
|
|
|
387
528
|
|
|
388
529
|
@app.command()
|
|
389
530
|
def studies(
|
|
390
|
-
env: str = typer.Option(
|
|
531
|
+
env: Optional[str] = typer.Option(
|
|
391
532
|
None,
|
|
392
533
|
"--env",
|
|
393
534
|
"-e",
|
|
394
|
-
help="
|
|
535
|
+
help="Read the study registry from this env's S3 bucket "
|
|
536
|
+
"(s3://<metadata-bucket>/config/studies.yaml) instead of the "
|
|
537
|
+
"local marker — this changes the data source, not just the "
|
|
538
|
+
"target.",
|
|
395
539
|
),
|
|
396
540
|
) -> None:
|
|
397
541
|
"""List the configured studies (bare names).
|
|
@@ -418,11 +562,8 @@ def studies(
|
|
|
418
562
|
|
|
419
563
|
@app.command()
|
|
420
564
|
def show(
|
|
421
|
-
env: str = typer.Option(
|
|
422
|
-
|
|
423
|
-
help="Environment; defaults to the current context.",
|
|
424
|
-
),
|
|
425
|
-
study: str = typer.Option(
|
|
565
|
+
env: Optional[str] = typer.Option(None, "--env", "-e", help=ENV_OPT),
|
|
566
|
+
study: Optional[str] = typer.Option(
|
|
426
567
|
None, "--study", "-s", help="Optional study to resolve against the env."
|
|
427
568
|
),
|
|
428
569
|
full: bool = typer.Option(
|
|
@@ -480,14 +621,12 @@ def show(
|
|
|
480
621
|
|
|
481
622
|
@app.command()
|
|
482
623
|
def diff(
|
|
483
|
-
env: str = typer.Option(
|
|
484
|
-
None, "--env", "-e",
|
|
485
|
-
help="Environment; defaults to the current context.",
|
|
486
|
-
),
|
|
624
|
+
env: Optional[str] = typer.Option(None, "--env", "-e", help=ENV_OPT),
|
|
487
625
|
file: Path = typer.Option(
|
|
488
626
|
...,
|
|
489
627
|
"--file",
|
|
490
|
-
|
|
628
|
+
exists=True,
|
|
629
|
+
dir_okay=False,
|
|
491
630
|
help="The env's INPUT file in the deployment wrapper, e.g. "
|
|
492
631
|
"../myproj-pipeline-deploy/config/myproj.staging.json.",
|
|
493
632
|
),
|
|
@@ -565,10 +704,9 @@ def diff(
|
|
|
565
704
|
|
|
566
705
|
@app.command("dbt-env")
|
|
567
706
|
def dbt_env(
|
|
568
|
-
env: str = typer.Option(
|
|
707
|
+
env: Optional[str] = typer.Option(
|
|
569
708
|
None, "--env", "-e",
|
|
570
|
-
help="
|
|
571
|
-
"passes this explicitly.",
|
|
709
|
+
help=ENV_OPT + " CodeBuild always passes this explicitly.",
|
|
572
710
|
),
|
|
573
711
|
) -> None:
|
|
574
712
|
"""Emit `export` lines for the env's dbt settings (resolved from SSM).
|