gen3-dataops-toolkit 3.8.0__tar.gz → 3.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/PKG-INFO +4 -4
  2. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/README.md +3 -3
  3. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/pyproject.toml +1 -1
  4. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/config_cmds.py +82 -12
  5. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/main.py +1 -1
  6. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/config.py +1 -1
  7. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/contexts.py +1 -1
  8. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/__init__.py +0 -0
  9. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/__init__.py +0 -0
  10. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/__init__.py +0 -0
  11. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/dispatch.py +0 -0
  12. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/registry.py +0 -0
  13. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/resolve.py +0 -0
  14. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/runner.py +0 -0
  15. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/_internal/safety.py +0 -0
  16. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/delete_cmds.py +0 -0
  17. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/dict_cmds.py +0 -0
  18. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/ec2_cmds.py +0 -0
  19. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/indexd_cmds.py +0 -0
  20. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/jobs.py +0 -0
  21. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/k8s.py +0 -0
  22. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/metadata.py +0 -0
  23. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/pipeline_cmds.py +0 -0
  24. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/release_cmds.py +0 -0
  25. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/cli/synth.py +0 -0
  26. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/indexd/__init__.py +0 -0
  27. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/indexd/file_access.py +0 -0
  28. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/indexd/indexd_registrar.py +0 -0
  29. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/ingest/ingest.py +0 -0
  30. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/resolver.py +0 -0
  31. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
  32. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/delete/delete_metadata.sh +0 -0
  33. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
  34. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/deploy_dd.sh +0 -0
  35. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
  36. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
  37. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/indexd/register_indexd.py +0 -0
  38. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/indexd/verify_file_access.py +0 -0
  39. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +0 -0
  40. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +0 -0
  41. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +0 -0
  42. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
  43. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +0 -0
  44. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
  45. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +0 -0
  46. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
  47. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +0 -0
  48. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/upload/metadata/upload_all_studies.sh +0 -0
  49. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/services/upload/metadata/upload_metadata.py +0 -0
  50. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/__init__.py +0 -0
  51. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/metadata_deleter.py +0 -0
  52. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/metadata_submitter.py +0 -0
  53. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
  54. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/utils/athena_utils.py +0 -0
  55. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/utils/dbt_utils.py +0 -0
  56. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/utils/release_writer.py +0 -0
  57. {gen3_dataops_toolkit-3.8.0 → gen3_dataops_toolkit-3.8.1}/src/g3dt/validate/validate.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gen3-dataops-toolkit
3
- Version: 3.8.0
3
+ Version: 3.8.1
4
4
  Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
5
5
  License: Apache-2.0
6
6
  Author: JoshuaHarris391
@@ -59,9 +59,9 @@ pip install gen3-dataops-toolkit
59
59
  ## Bootstrap: discover, then use
60
60
 
61
61
  ```bash
62
- g3dt config discover --all-profiles --add # find every deployed env your
63
- # AWS profiles can see; register
64
- # them as contexts
62
+ g3dt config discover <aws-profile> --add # logs the profile in if needed,
63
+ # lists every deployed env its
64
+ # account holds, registers them
65
65
  g3dt config contexts # list them (current marked *)
66
66
  g3dt config use myproj/test # point g3dt at one
67
67
  ```
@@ -25,9 +25,9 @@ pip install gen3-dataops-toolkit
25
25
  ## Bootstrap: discover, then use
26
26
 
27
27
  ```bash
28
- g3dt config discover --all-profiles --add # find every deployed env your
29
- # AWS profiles can see; register
30
- # them as contexts
28
+ g3dt config discover <aws-profile> --add # logs the profile in if needed,
29
+ # lists every deployed env its
30
+ # account holds, registers them
31
31
  g3dt config contexts # list them (current marked *)
32
32
  g3dt config use myproj/test # point g3dt at one
33
33
  ```
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "gen3-dataops-toolkit"
3
- version = "3.8.0"
3
+ version = "3.8.1"
4
4
  description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
@@ -226,10 +226,16 @@ def forget(
226
226
 
227
227
  @app.command("discover")
228
228
  def discover(
229
+ profile_arg: str = typer.Argument(
230
+ None, metavar="[PROFILE]",
231
+ help="Scan just this AWS profile's account. If its SSO session is "
232
+ "stale you are offered `aws sso login` first.",
233
+ ),
229
234
  all_profiles: bool = typer.Option(
230
235
  False, "--all-profiles",
231
236
  help="Scan every profile in ~/.aws/config for deployed "
232
- "/{project}/{env} trees.",
237
+ "/{project}/{env} trees (profiles with stale sessions are "
238
+ "skipped, not logged in).",
233
239
  ),
234
240
  add_found: bool = typer.Option(
235
241
  False, "--add", help="Register discovered pairs as contexts."
@@ -243,10 +249,11 @@ def discover(
243
249
  ) -> None:
244
250
  """Find deployable infrastructure.
245
251
 
246
- Default: verify each configured context against its own account (nothing
247
- else is touched). With --all-profiles: enumerate every AWS profile and
248
- list all deployed environments its account holds; add --add to register
249
- them as contexts.
252
+ With a PROFILE argument: log that one profile in if needed, then list
253
+ every deployed environment its account holds — the recommended flow.
254
+ With --all-profiles: sweep every configured profile (stale sessions are
255
+ skipped). With neither: verify each already-configured context against
256
+ its own account. Add --add to register findings as contexts.
250
257
  """
251
258
  resolve.announce_context()
252
259
  # The scan probes credentials that may be stale; botocore logs its own
@@ -259,12 +266,17 @@ def discover(
259
266
  marker = config.load_marker()
260
267
  default_region = region or marker.get("region") or config.DEFAULT_REGION
261
268
 
262
- if not all_profiles:
269
+ if profile_arg and all_profiles:
270
+ typer.secho("Pass either a PROFILE argument or --all-profiles, not both.",
271
+ fg=typer.colors.RED, err=True)
272
+ raise typer.Exit(1)
273
+
274
+ if not profile_arg and not all_profiles:
263
275
  ctxs = contexts.list_contexts(marker)
264
276
  if not ctxs:
265
277
  typer.secho(
266
278
  "No contexts configured to verify. Try "
267
- "`g3dt config discover --all-profiles [--add]`.",
279
+ "`g3dt config discover <aws-profile> --add`.",
268
280
  fg=typer.colors.YELLOW,
269
281
  )
270
282
  return
@@ -285,13 +297,48 @@ def discover(
285
297
  import botocore.session
286
298
 
287
299
  profiles_cfg = botocore.session.Session().full_config.get("profiles", {})
288
- names = profile or sorted(profiles_cfg)
289
- if not names:
290
- typer.secho("No AWS profiles found in ~/.aws/config.",
291
- fg=typer.colors.YELLOW)
292
- return
300
+
301
+ if profile_arg:
302
+ if profile_arg not in profiles_cfg:
303
+ typer.secho(
304
+ f"No profile '{profile_arg}' in ~/.aws/config. Configured: "
305
+ f"{', '.join(sorted(profiles_cfg)) or '(none)'}.",
306
+ fg=typer.colors.RED, err=True,
307
+ )
308
+ raise typer.Exit(1)
309
+ if not _profile_session_valid(profile_arg):
310
+ typer.secho(
311
+ f"The SSO session for '{profile_arg}' is not valid.",
312
+ fg=typer.colors.YELLOW,
313
+ )
314
+ if not typer.confirm(
315
+ f"Run `aws sso login --profile {profile_arg}` now?",
316
+ default=True,
317
+ ):
318
+ typer.secho("Aborted — log in and re-run discover.",
319
+ fg=typer.colors.YELLOW)
320
+ raise typer.Exit(1)
321
+ import subprocess
322
+
323
+ login = subprocess.run(
324
+ ["aws", "sso", "login", "--profile", profile_arg]
325
+ )
326
+ if login.returncode != 0 or not _profile_session_valid(profile_arg):
327
+ typer.secho(
328
+ f"Still no valid session for '{profile_arg}'. Aborting.",
329
+ fg=typer.colors.RED, err=True,
330
+ )
331
+ raise typer.Exit(1)
332
+ names = [profile_arg]
333
+ else:
334
+ names = profile or sorted(profiles_cfg)
335
+ if not names:
336
+ typer.secho("No AWS profiles found in ~/.aws/config.",
337
+ fg=typer.colors.YELLOW)
338
+ return
293
339
 
294
340
  found: List[contexts.Context] = []
341
+ seen_names = set()
295
342
  for prof in names:
296
343
  prof_region = (profiles_cfg.get(prof, {}).get("region")
297
344
  or default_region)
@@ -309,6 +356,12 @@ def discover(
309
356
  continue
310
357
  for project, env in pairs:
311
358
  name = f"{project}/{env}"
359
+ if name in seen_names:
360
+ # Same account visible through several profiles (e.g. a
361
+ # 'default' alias): the first profile wins; don't print
362
+ # duplicate suggestions.
363
+ continue
364
+ seen_names.add(name)
312
365
  ctx = contexts.Context(
313
366
  name=name, project=project, env=env, profile=prof,
314
367
  region=prof_region,
@@ -340,6 +393,23 @@ def discover(
340
393
  )
341
394
 
342
395
 
396
+ def _profile_session_valid(profile: str) -> bool:
397
+ """Cheap credential check: can this profile sign a request right now?"""
398
+ import boto3
399
+ from botocore.config import Config as _BotoConfig
400
+
401
+ try:
402
+ session = boto3.Session(profile_name=profile)
403
+ session.client(
404
+ "sts",
405
+ config=_BotoConfig(connect_timeout=3, read_timeout=5,
406
+ retries={"max_attempts": 1}),
407
+ ).get_caller_identity()
408
+ return True
409
+ except Exception:
410
+ return False
411
+
412
+
343
413
  def _scan_profile(profile: str, region: str) -> List[tuple]:
344
414
  """All deployed (project, env) pairs visible to one profile's account.
345
415
 
@@ -71,7 +71,7 @@ Contexts: what am I pointed at?
71
71
  prints the active one first (to stderr) — read that line before anything
72
72
  else. Production contexts are marked [PROD] and gate destructive actions
73
73
  behind typing the context name; --yes never bypasses that.
74
- g3dt config discover --all-profiles --add find + register deployed infra
74
+ g3dt config discover <aws-profile> --add find + register deployed infra
75
75
  g3dt config contexts list them (current marked *)
76
76
  g3dt config use <name> act there from now on
77
77
  g3dt --ctx <name> <command> one-shot override
@@ -180,7 +180,7 @@ def require_project(marker: Optional[dict] = None) -> str:
180
180
  if not project:
181
181
  raise ConfigError(
182
182
  "No project configured. Register a context with "
183
- "`g3dt config discover --all-profiles --add` and select it with "
183
+ "`g3dt config discover <aws-profile> --add` and select it with "
184
184
  "`g3dt config use <name>` — or create a g3dt.yaml marker "
185
185
  f"(searched: {', '.join(MARKER_PATHS)}) with at least:\n"
186
186
  " project: <projectId>\n"
@@ -240,7 +240,7 @@ def resolve_context(
240
240
  raise _config.ConfigError(
241
241
  "No context selected. Pass --env/--ctx, set one with "
242
242
  "`g3dt config use <name>`, or register some with "
243
- "`g3dt config discover --all-profiles --add`."
243
+ "`g3dt config discover <aws-profile> --add`."
244
244
  )
245
245
  return None, None
246
246