gen3-dataops-toolkit 3.8.0__tar.gz → 3.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/PKG-INFO +4 -4
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/README.md +3 -3
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/pyproject.toml +1 -1
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/config_cmds.py +82 -12
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/main.py +1 -1
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/config.py +1 -1
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/contexts.py +1 -1
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/dispatch.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/registry.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/resolve.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/runner.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/safety.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/delete_cmds.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/dict_cmds.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/ec2_cmds.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/indexd_cmds.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/jobs.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/k8s.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/metadata.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/pipeline_cmds.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/release_cmds.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/synth.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/indexd/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/indexd/file_access.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/indexd/indexd_registrar.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/ingest/ingest.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/resolver.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/delete/delete_metadata.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/deploy_dd.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/indexd/register_indexd.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/indexd/verify_file_access.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/upload/metadata/upload_all_studies.sh +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/upload/metadata/upload_metadata.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/__init__.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/metadata_deleter.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/metadata_submitter.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/utils/athena_utils.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/utils/dbt_utils.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/utils/release_writer.py +0 -0
- {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/validate/validate.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gen3-dataops-toolkit
|
|
3
|
-
Version: 3.8.
|
|
3
|
+
Version: 3.8.1
|
|
4
4
|
Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: JoshuaHarris391
|
|
@@ -59,9 +59,9 @@ pip install gen3-dataops-toolkit
|
|
|
59
59
|
## Bootstrap: discover, then use
|
|
60
60
|
|
|
61
61
|
```bash
|
|
62
|
-
g3dt config discover
|
|
63
|
-
#
|
|
64
|
-
#
|
|
62
|
+
g3dt config discover <aws-profile> --add # logs the profile in if needed,
|
|
63
|
+
# lists every deployed env its
|
|
64
|
+
# account holds, registers them
|
|
65
65
|
g3dt config contexts # list them (current marked *)
|
|
66
66
|
g3dt config use myproj/test # point g3dt at one
|
|
67
67
|
```
|
|
@@ -25,9 +25,9 @@ pip install gen3-dataops-toolkit
|
|
|
25
25
|
## Bootstrap: discover, then use
|
|
26
26
|
|
|
27
27
|
```bash
|
|
28
|
-
g3dt config discover
|
|
29
|
-
#
|
|
30
|
-
#
|
|
28
|
+
g3dt config discover <aws-profile> --add # logs the profile in if needed,
|
|
29
|
+
# lists every deployed env its
|
|
30
|
+
# account holds, registers them
|
|
31
31
|
g3dt config contexts # list them (current marked *)
|
|
32
32
|
g3dt config use myproj/test # point g3dt at one
|
|
33
33
|
```
|
|
@@ -226,10 +226,16 @@ def forget(
|
|
|
226
226
|
|
|
227
227
|
@app.command("discover")
|
|
228
228
|
def discover(
|
|
229
|
+
profile_arg: str = typer.Argument(
|
|
230
|
+
None, metavar="[PROFILE]",
|
|
231
|
+
help="Scan just this AWS profile's account. If its SSO session is "
|
|
232
|
+
"stale you are offered `aws sso login` first.",
|
|
233
|
+
),
|
|
229
234
|
all_profiles: bool = typer.Option(
|
|
230
235
|
False, "--all-profiles",
|
|
231
236
|
help="Scan every profile in ~/.aws/config for deployed "
|
|
232
|
-
"/{project}/{env} trees
|
|
237
|
+
"/{project}/{env} trees (profiles with stale sessions are "
|
|
238
|
+
"skipped, not logged in).",
|
|
233
239
|
),
|
|
234
240
|
add_found: bool = typer.Option(
|
|
235
241
|
False, "--add", help="Register discovered pairs as contexts."
|
|
@@ -243,10 +249,11 @@ def discover(
|
|
|
243
249
|
) -> None:
|
|
244
250
|
"""Find deployable infrastructure.
|
|
245
251
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
252
|
+
With a PROFILE argument: log that one profile in if needed, then list
|
|
253
|
+
every deployed environment its account holds — the recommended flow.
|
|
254
|
+
With --all-profiles: sweep every configured profile (stale sessions are
|
|
255
|
+
skipped). With neither: verify each already-configured context against
|
|
256
|
+
its own account. Add --add to register findings as contexts.
|
|
250
257
|
"""
|
|
251
258
|
resolve.announce_context()
|
|
252
259
|
# The scan probes credentials that may be stale; botocore logs its own
|
|
@@ -259,12 +266,17 @@ def discover(
|
|
|
259
266
|
marker = config.load_marker()
|
|
260
267
|
default_region = region or marker.get("region") or config.DEFAULT_REGION
|
|
261
268
|
|
|
262
|
-
if
|
|
269
|
+
if profile_arg and all_profiles:
|
|
270
|
+
typer.secho("Pass either a PROFILE argument or --all-profiles, not both.",
|
|
271
|
+
fg=typer.colors.RED, err=True)
|
|
272
|
+
raise typer.Exit(1)
|
|
273
|
+
|
|
274
|
+
if not profile_arg and not all_profiles:
|
|
263
275
|
ctxs = contexts.list_contexts(marker)
|
|
264
276
|
if not ctxs:
|
|
265
277
|
typer.secho(
|
|
266
278
|
"No contexts configured to verify. Try "
|
|
267
|
-
"`g3dt config discover
|
|
279
|
+
"`g3dt config discover <aws-profile> --add`.",
|
|
268
280
|
fg=typer.colors.YELLOW,
|
|
269
281
|
)
|
|
270
282
|
return
|
|
@@ -285,13 +297,48 @@ def discover(
|
|
|
285
297
|
import botocore.session
|
|
286
298
|
|
|
287
299
|
profiles_cfg = botocore.session.Session().full_config.get("profiles", {})
|
|
288
|
-
|
|
289
|
-
if
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
300
|
+
|
|
301
|
+
if profile_arg:
|
|
302
|
+
if profile_arg not in profiles_cfg:
|
|
303
|
+
typer.secho(
|
|
304
|
+
f"No profile '{profile_arg}' in ~/.aws/config. Configured: "
|
|
305
|
+
f"{', '.join(sorted(profiles_cfg)) or '(none)'}.",
|
|
306
|
+
fg=typer.colors.RED, err=True,
|
|
307
|
+
)
|
|
308
|
+
raise typer.Exit(1)
|
|
309
|
+
if not _profile_session_valid(profile_arg):
|
|
310
|
+
typer.secho(
|
|
311
|
+
f"The SSO session for '{profile_arg}' is not valid.",
|
|
312
|
+
fg=typer.colors.YELLOW,
|
|
313
|
+
)
|
|
314
|
+
if not typer.confirm(
|
|
315
|
+
f"Run `aws sso login --profile {profile_arg}` now?",
|
|
316
|
+
default=True,
|
|
317
|
+
):
|
|
318
|
+
typer.secho("Aborted — log in and re-run discover.",
|
|
319
|
+
fg=typer.colors.YELLOW)
|
|
320
|
+
raise typer.Exit(1)
|
|
321
|
+
import subprocess
|
|
322
|
+
|
|
323
|
+
login = subprocess.run(
|
|
324
|
+
["aws", "sso", "login", "--profile", profile_arg]
|
|
325
|
+
)
|
|
326
|
+
if login.returncode != 0 or not _profile_session_valid(profile_arg):
|
|
327
|
+
typer.secho(
|
|
328
|
+
f"Still no valid session for '{profile_arg}'. Aborting.",
|
|
329
|
+
fg=typer.colors.RED, err=True,
|
|
330
|
+
)
|
|
331
|
+
raise typer.Exit(1)
|
|
332
|
+
names = [profile_arg]
|
|
333
|
+
else:
|
|
334
|
+
names = profile or sorted(profiles_cfg)
|
|
335
|
+
if not names:
|
|
336
|
+
typer.secho("No AWS profiles found in ~/.aws/config.",
|
|
337
|
+
fg=typer.colors.YELLOW)
|
|
338
|
+
return
|
|
293
339
|
|
|
294
340
|
found: List[contexts.Context] = []
|
|
341
|
+
seen_names = set()
|
|
295
342
|
for prof in names:
|
|
296
343
|
prof_region = (profiles_cfg.get(prof, {}).get("region")
|
|
297
344
|
or default_region)
|
|
@@ -309,6 +356,12 @@ def discover(
|
|
|
309
356
|
continue
|
|
310
357
|
for project, env in pairs:
|
|
311
358
|
name = f"{project}/{env}"
|
|
359
|
+
if name in seen_names:
|
|
360
|
+
# Same account visible through several profiles (e.g. a
|
|
361
|
+
# 'default' alias): the first profile wins; don't print
|
|
362
|
+
# duplicate suggestions.
|
|
363
|
+
continue
|
|
364
|
+
seen_names.add(name)
|
|
312
365
|
ctx = contexts.Context(
|
|
313
366
|
name=name, project=project, env=env, profile=prof,
|
|
314
367
|
region=prof_region,
|
|
@@ -340,6 +393,23 @@ def discover(
|
|
|
340
393
|
)
|
|
341
394
|
|
|
342
395
|
|
|
396
|
+
def _profile_session_valid(profile: str) -> bool:
|
|
397
|
+
"""Cheap credential check: can this profile sign a request right now?"""
|
|
398
|
+
import boto3
|
|
399
|
+
from botocore.config import Config as _BotoConfig
|
|
400
|
+
|
|
401
|
+
try:
|
|
402
|
+
session = boto3.Session(profile_name=profile)
|
|
403
|
+
session.client(
|
|
404
|
+
"sts",
|
|
405
|
+
config=_BotoConfig(connect_timeout=3, read_timeout=5,
|
|
406
|
+
retries={"max_attempts": 1}),
|
|
407
|
+
).get_caller_identity()
|
|
408
|
+
return True
|
|
409
|
+
except Exception:
|
|
410
|
+
return False
|
|
411
|
+
|
|
412
|
+
|
|
343
413
|
def _scan_profile(profile: str, region: str) -> List[tuple]:
|
|
344
414
|
"""All deployed (project, env) pairs visible to one profile's account.
|
|
345
415
|
|
|
@@ -71,7 +71,7 @@ Contexts: what am I pointed at?
|
|
|
71
71
|
prints the active one first (to stderr) — read that line before anything
|
|
72
72
|
else. Production contexts are marked [PROD] and gate destructive actions
|
|
73
73
|
behind typing the context name; --yes never bypasses that.
|
|
74
|
-
g3dt config discover
|
|
74
|
+
g3dt config discover <aws-profile> --add find + register deployed infra
|
|
75
75
|
g3dt config contexts list them (current marked *)
|
|
76
76
|
g3dt config use <name> act there from now on
|
|
77
77
|
g3dt --ctx <name> <command> one-shot override
|
|
@@ -180,7 +180,7 @@ def require_project(marker: Optional[dict] = None) -> str:
|
|
|
180
180
|
if not project:
|
|
181
181
|
raise ConfigError(
|
|
182
182
|
"No project configured. Register a context with "
|
|
183
|
-
"`g3dt config discover
|
|
183
|
+
"`g3dt config discover <aws-profile> --add` and select it with "
|
|
184
184
|
"`g3dt config use <name>` — or create a g3dt.yaml marker "
|
|
185
185
|
f"(searched: {', '.join(MARKER_PATHS)}) with at least:\n"
|
|
186
186
|
" project: <projectId>\n"
|
|
@@ -240,7 +240,7 @@ def resolve_context(
|
|
|
240
240
|
raise _config.ConfigError(
|
|
241
241
|
"No context selected. Pass --env/--ctx, set one with "
|
|
242
242
|
"`g3dt config use <name>`, or register some with "
|
|
243
|
-
"`g3dt config discover
|
|
243
|
+
"`g3dt config discover <aws-profile> --add`."
|
|
244
244
|
)
|
|
245
245
|
return None, None
|
|
246
246
|
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/__init__.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/dispatch.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/registry.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/indexd/indexd_registrar.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/deploy_dd.sh
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/pull_dict.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/login_to_pod.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/metadata_deleter.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/metadata_submitter.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/upload_synthdata_s3.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|