gen3-dataops-toolkit 3.4.0__tar.gz → 3.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/PKG-INFO +16 -1
  2. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/README.md +15 -0
  3. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/pyproject.toml +1 -1
  4. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/config_cmds.py +14 -0
  5. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/dict_cmds.py +9 -1
  6. gen3_dataops_toolkit-3.5.1/src/g3dt/cli/k8s.py +104 -0
  7. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/synth.py +16 -2
  8. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/config.py +36 -0
  9. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +4 -3
  10. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +14 -3
  11. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +15 -5
  12. gen3_dataops_toolkit-3.4.0/src/g3dt/cli/k8s.py +0 -54
  13. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/__init__.py +0 -0
  14. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/__init__.py +0 -0
  15. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/__init__.py +0 -0
  16. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/dispatch.py +0 -0
  17. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/registry.py +0 -0
  18. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/resolve.py +0 -0
  19. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/runner.py +0 -0
  20. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/safety.py +0 -0
  21. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/delete_cmds.py +0 -0
  22. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/ec2_cmds.py +0 -0
  23. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/indexd_cmds.py +0 -0
  24. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/jobs.py +0 -0
  25. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/main.py +0 -0
  26. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/metadata.py +0 -0
  27. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/pipeline_cmds.py +0 -0
  28. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/release_cmds.py +0 -0
  29. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/indexd/__init__.py +0 -0
  30. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/indexd/file_access.py +0 -0
  31. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/indexd/indexd_registrar.py +0 -0
  32. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/ingest/ingest.py +0 -0
  33. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/resolver.py +0 -0
  34. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
  35. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/delete/delete_metadata.sh +0 -0
  36. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
  37. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/deploy_dd.sh +0 -0
  38. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
  39. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
  40. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/indexd/register_indexd.py +0 -0
  41. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/indexd/verify_file_access.py +0 -0
  42. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
  43. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +0 -0
  44. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
  45. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +0 -0
  46. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
  47. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +0 -0
  48. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/upload/metadata/upload_all_studies.sh +0 -0
  49. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/upload/metadata/upload_metadata.py +0 -0
  50. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/__init__.py +0 -0
  51. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/metadata_deleter.py +0 -0
  52. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/metadata_submitter.py +0 -0
  53. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
  54. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/utils/athena_utils.py +0 -0
  55. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/utils/dbt_utils.py +0 -0
  56. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/utils/release_writer.py +0 -0
  57. {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/validate/validate.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gen3-dataops-toolkit
3
- Version: 3.4.0
3
+ Version: 3.5.1
4
4
  Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
5
5
  License: Apache-2.0
6
6
  Author: JoshuaHarris391
@@ -208,6 +208,21 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
208
208
  `g3dt config show --env <env>` prints the resolved provider, model, and key
209
209
  path.
210
210
 
211
+ ### Kubernetes restart targets
212
+
213
+ `k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
214
+ restart the commons' schema microservices **serially, in a configured order**,
215
+ waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
216
+ the deploy flows run a named ETL cronjob. Both targets resolve with precedence
217
+ **CLI flags > SSM > default**: the CDK config's optional `k8s` block publishes
218
+ `app/restart_services` (comma-separated deployment names) and
219
+ `app/etl_cronjob`, `--restart-services` / `--etl-cronjob` override them for
220
+ one run, and environments deployed without the block keep the classic Gen3 set
221
+ (`sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment`
222
+ / `etl-cronjob`). A commons that manages some service outside this flow (e.g.
223
+ a manually redeployed frontend) simply omits it from the list in its wrapper
224
+ config. `g3dt config show --env <env>` prints the resolved values.
225
+
211
226
  ## Verifying download access (check-download)
212
227
 
213
228
  Registration alone does not prove a file can be downloaded. Two failure modes
@@ -174,6 +174,21 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
174
174
  `g3dt config show --env <env>` prints the resolved provider, model, and key
175
175
  path.
176
176
 
177
+ ### Kubernetes restart targets
178
+
179
+ `k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
180
+ restart the commons' schema microservices **serially, in a configured order**,
181
+ waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
182
+ the deploy flows run a named ETL cronjob. Both targets resolve with precedence
183
+ **CLI flags > SSM > default**: the CDK config's optional `k8s` block publishes
184
+ `app/restart_services` (comma-separated deployment names) and
185
+ `app/etl_cronjob`, `--restart-services` / `--etl-cronjob` override them for
186
+ one run, and environments deployed without the block keep the classic Gen3 set
187
+ (`sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment`
188
+ / `etl-cronjob`). A commons that manages some service outside this flow (e.g.
189
+ a manually redeployed frontend) simply omits it from the list in its wrapper
190
+ config. `g3dt config show --env <env>` prints the resolved values.
191
+
177
192
  ## Verifying download access (check-download)
178
193
 
179
194
  Registration alone does not prove a file can be downloaded. Two failure modes
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "gen3-dataops-toolkit"
3
- version = "3.4.0"
3
+ version = "3.5.1"
4
4
  description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
@@ -123,6 +123,10 @@ def show(
123
123
  typer.echo(f" llm_provider : {e.llm_provider}")
124
124
  typer.echo(f" llm_model : {e.llm_model or '(not set — pass --llm-model or add the llm block to the CDK config)'}")
125
125
  typer.echo(f" llm_api_key_file : {config.llm_api_key_file() or '(not set — g3dt config set llm_api_key_file <path>)'}")
126
+ # k8s restart targets: from SSM (the CDK's optional k8s block), restarted
127
+ # in the listed order by restart-schema/restart-ms/dict deploy/synth deploy.
128
+ typer.echo(f" restart_services : {e.restart_services}")
129
+ typer.echo(f" etl_cronjob : {e.etl_cronjob}")
126
130
  if study:
127
131
  s = study_of(study, env)
128
132
  typer.secho(f"Study: {study} -> {s.key}", bold=True)
@@ -217,6 +221,16 @@ def diff(
217
221
  for camel, leaf in {"provider": "llm_provider", "model": "llm_model"}.items():
218
222
  if camel in llm:
219
223
  check(f"llm.{camel}", llm.get(camel), rc.get(f"app/{leaf}"))
224
+ k8s = inputs.get("k8s") or {}
225
+ if "schemaRestartServices" in k8s:
226
+ # The CDK publishes the list comma-joined; compare the same shape.
227
+ check(
228
+ "k8s.schemaRestartServices",
229
+ ",".join(k8s.get("schemaRestartServices") or []),
230
+ rc.get("app/restart_services"),
231
+ )
232
+ if "etlCronjob" in k8s:
233
+ check("k8s.etlCronjob", k8s.get("etlCronjob"), rc.get("app/etl_cronjob"))
220
234
 
221
235
  if not drift:
222
236
  typer.secho(
@@ -103,6 +103,11 @@ def deploy(
103
103
  version: str = typer.Option(
104
104
  None, "--version", help="Dictionary git tag (default: the env's version)."
105
105
  ),
106
+ restart_services: Optional[str] = typer.Option(
107
+ None, "--restart-services",
108
+ help="Comma-separated deployment names restarted after the upload, in "
109
+ "order; default: the env's SSM app/restart_services.",
110
+ ),
106
111
  ) -> None:
107
112
  """Pull + upload the dictionary and restart Gen3 schema microservices.
108
113
 
@@ -123,7 +128,10 @@ def deploy(
123
128
  """
124
129
  e = env_of(env)
125
130
  warn_if_overridden(e, version)
131
+ env_vars = script_env(e, _version(e, version))
132
+ if restart_services:
133
+ env_vars["G3DT_RESTART_SERVICES"] = restart_services
126
134
  runner.run(
127
135
  runner.bash_script("services/dictionary/deploy_dd.sh", env),
128
- env=script_env(e, _version(e, version)),
136
+ env=env_vars,
129
137
  )
@@ -0,0 +1,104 @@
1
+ """`g3dt k8s` — restart Gen3 microservices / ETL via ArgoCD (LOCAL only).
2
+
3
+ These use ``argocd login --sso`` (a browser flow), so they cannot run headless
4
+ on EC2. The wrapped scripts receive their settings as ``G3DT_*`` environment
5
+ variables resolved from SSM — they read no config files.
6
+
7
+ The restart targets resolve with precedence CLI flags > SSM > default: the
8
+ CDK config's optional ``k8s`` block publishes ``app/restart_services`` (a
9
+ comma-separated list, restarted in order) and ``app/etl_cronjob``;
10
+ ``--restart-services`` / ``--etl-cronjob`` override them for one run, and
11
+ environments deployed without the block keep the classic Gen3 set
12
+ (sheepdog, peregrine, guppy, portal / etl-cronjob).
13
+ """
14
+ from __future__ import annotations
15
+
16
+ from typing import Optional
17
+
18
+ import typer
19
+
20
+ from g3dt.config import script_env
21
+ from g3dt.cli._internal import runner
22
+ from g3dt.cli._internal.resolve import env_of
23
+
24
+ app = typer.Typer(no_args_is_help=True, help="ArgoCD / Kubernetes restarts (local).")
25
+
26
+ _SCHEMA = "services/k8s_ops/argocd_restart_schema.sh"
27
+ _ETL = "services/k8s_ops/argocd_restart_etl.sh"
28
+ _ETL_AND_MS = "services/k8s_ops/restart_etl_and_ms.sh"
29
+
30
+ _RESTART_SERVICES_HELP = (
31
+ "Comma-separated deployment names to restart, in order; default: the "
32
+ "env's SSM app/restart_services (the CDK config's k8s.schemaRestartServices)."
33
+ )
34
+ _ETL_CRONJOB_HELP = (
35
+ "ETL cronjob name; default: the env's SSM app/etl_cronjob "
36
+ "(the CDK config's k8s.etlCronjob)."
37
+ )
38
+
39
+
40
+ def restart_env(e, restart_services: Optional[str] = None,
41
+ etl_cronjob: Optional[str] = None) -> dict:
42
+ """script_env plus per-run restart-target overrides (flags beat SSM)."""
43
+ env_vars = script_env(e)
44
+ if restart_services:
45
+ env_vars["G3DT_RESTART_SERVICES"] = restart_services
46
+ if etl_cronjob:
47
+ env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob
48
+ return env_vars
49
+
50
+
51
+ @app.command(name="restart-schema")
52
+ def restart_schema(
53
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
54
+ sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
55
+ restart_services: Optional[str] = typer.Option(
56
+ None, "--restart-services", help=_RESTART_SERVICES_HELP
57
+ ),
58
+ ) -> None:
59
+ """Restart the schema microservices, in the env's configured order."""
60
+ e = env_of(env)
61
+ args = ["-d", e.domain, "-a", e.app_name, "-n", e.namespace]
62
+ if sync:
63
+ args.append("-s")
64
+ runner.run(
65
+ runner.bash_script(_SCHEMA, *args),
66
+ env=restart_env(e, restart_services=restart_services),
67
+ )
68
+
69
+
70
+ @app.command(name="restart-etl")
71
+ def restart_etl(
72
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
73
+ sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
74
+ etl_cronjob: Optional[str] = typer.Option(
75
+ None, "--etl-cronjob", help=_ETL_CRONJOB_HELP
76
+ ),
77
+ ) -> None:
78
+ """Create + run the ETL cronjob and wait for completion."""
79
+ e = env_of(env)
80
+ args = ["-e", env]
81
+ if sync:
82
+ args.append("-s")
83
+ runner.run(
84
+ runner.bash_script(_ETL, *args),
85
+ env=restart_env(e, etl_cronjob=etl_cronjob),
86
+ )
87
+
88
+
89
+ @app.command(name="restart-ms")
90
+ def restart_ms(
91
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
92
+ restart_services: Optional[str] = typer.Option(
93
+ None, "--restart-services", help=_RESTART_SERVICES_HELP
94
+ ),
95
+ etl_cronjob: Optional[str] = typer.Option(
96
+ None, "--etl-cronjob", help=_ETL_CRONJOB_HELP
97
+ ),
98
+ ) -> None:
99
+ """Restart both ETL and schema microservices (wraps restart_etl_and_ms.sh)."""
100
+ e = env_of(env)
101
+ runner.run(
102
+ runner.bash_script(_ETL_AND_MS, env),
103
+ env=restart_env(e, restart_services=restart_services, etl_cronjob=etl_cronjob),
104
+ )
@@ -168,17 +168,31 @@ def deploy(
168
168
  help="Path to the file holding the LLM API key; default: the marker's "
169
169
  "llm_api_key_file (set once: g3dt config set llm_api_key_file <path>).",
170
170
  ),
171
+ restart_services: Optional[str] = typer.Option(
172
+ None, "--restart-services",
173
+ help="Comma-separated deployment names restarted during the deploy, in "
174
+ "order; default: the env's SSM app/restart_services.",
175
+ ),
176
+ etl_cronjob: Optional[str] = typer.Option(
177
+ None, "--etl-cronjob",
178
+ help="ETL cronjob name; default: the env's SSM app/etl_cronjob.",
179
+ ),
171
180
  ) -> None:
172
181
  """Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts).
173
182
 
174
183
  Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed
175
- generation). Provider/model come from the env's SSM tree unless
176
- overridden; the API key path comes from --llm-api-key-file or the marker.
184
+ generation). Provider/model and the restart targets come from the env's
185
+ SSM tree unless overridden; the API key path comes from
186
+ --llm-api-key-file or the marker.
177
187
  """
178
188
  e = env_of(env)
179
189
  safety.confirm_prod_strict("synthetic full deploy", env)
180
190
  env_vars = script_env(e)
181
191
  env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
192
+ if restart_services:
193
+ env_vars["G3DT_RESTART_SERVICES"] = restart_services
194
+ if etl_cronjob:
195
+ env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob
182
196
  runner.run(
183
197
  runner.bash_script(
184
198
  "services/synthetic_data/full_deploy_dd_and_synth.sh", env
@@ -58,6 +58,17 @@ DEFAULT_DICT_PATH = "dictionary/prod_dict/acdc_schema.json"
58
58
  #: when the ``--llm`` path is used and no model is configured anywhere.
59
59
  DEFAULT_LLM_PROVIDER = "anthropic"
60
60
 
61
+ #: Kubernetes restart targets, published by the CDK's OPTIONAL ``k8s`` config
62
+ #: block as ``app/restart_services`` (comma-separated, restarted in order) and
63
+ #: ``app/etl_cronjob``. Optional app inputs: environments deployed without the
64
+ #: block keep the classic Gen3 set. Consumed by every restart path — `g3dt
65
+ #: k8s restart-schema/restart-etl/restart-ms`, `dict deploy`, `synth deploy` —
66
+ #: via $G3DT_RESTART_SERVICES / $G3DT_ETL_CRONJOB in the service scripts.
67
+ DEFAULT_RESTART_SERVICES = (
68
+ "sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment"
69
+ )
70
+ DEFAULT_ETL_CRONJOB = "etl-cronjob"
71
+
61
72
  #: Marker locations, most specific first.
62
73
  MARKER_PATHS = ("g3dt.yaml", "~/.g3dt/g3dt.yaml", "/etc/g3dt/g3dt.yaml")
63
74
 
@@ -257,6 +268,9 @@ class EnvConfig:
257
268
  # with guidance rather than silently picking a model.
258
269
  llm_provider: str = DEFAULT_LLM_PROVIDER
259
270
  llm_model: Optional[str] = None
271
+ # Optional k8s restart targets (SSM app/restart_services, app/etl_cronjob).
272
+ restart_services: str = DEFAULT_RESTART_SERVICES
273
+ etl_cronjob: str = DEFAULT_ETL_CRONJOB
260
274
 
261
275
 
262
276
  def _app_or_default(rc, leaf: str, default: str) -> str:
@@ -320,6 +334,10 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
320
334
  # the config has an llm block, so absence means "use the defaults".
321
335
  llm_provider=_app_or_default(rc, "llm_provider", DEFAULT_LLM_PROVIDER),
322
336
  llm_model=(_app_or_default(rc, "llm_model", "") or None),
337
+ restart_services=_app_or_default(
338
+ rc, "restart_services", DEFAULT_RESTART_SERVICES
339
+ ),
340
+ etl_cronjob=_app_or_default(rc, "etl_cronjob", DEFAULT_ETL_CRONJOB),
323
341
  )
324
342
 
325
343
 
@@ -410,8 +428,24 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
410
428
  scripts compose a URL keeps Python the single source of truth: the scripts
411
429
  read ``G3DT_DICT_URL``/``G3DT_DICT_FILENAME`` and never build either
412
430
  themselves, so there is only one implementation to keep correct.
431
+
432
+ The interpreter's own bin directory is prepended to PATH: console scripts
433
+ installed next to g3dt — gen3-metadata-simulator via
434
+ ``g3dt synth install-simulator`` — land there, but a pipx install exposes
435
+ only g3dt's own entry points on the caller's PATH, so without this the
436
+ wrapped scripts' ``command -v gen3-metadata-simulator`` check fails even
437
+ though the tool is installed.
413
438
  """
439
+ import sys
440
+
414
441
  env = dict(os.environ)
442
+ # No .resolve(): a venv's python is a symlink to the base interpreter, and
443
+ # resolving it would point at the base install's bin instead of the venv
444
+ # bin where console scripts are actually created.
445
+ venv_bin = str(Path(sys.executable).parent)
446
+ path_entries = env.get("PATH", "").split(os.pathsep)
447
+ if venv_bin not in path_entries:
448
+ env["PATH"] = venv_bin + os.pathsep + env.get("PATH", "")
415
449
  values = {
416
450
  "G3DT_ENV": e.name,
417
451
  "G3DT_REGION": e.region,
@@ -428,6 +462,8 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
428
462
  "G3DT_SCHEMA_REPO": e.schema_repo,
429
463
  "G3DT_LLM_PROVIDER": e.llm_provider,
430
464
  "G3DT_LLM_MODEL": e.llm_model,
465
+ "G3DT_RESTART_SERVICES": e.restart_services,
466
+ "G3DT_ETL_CRONJOB": e.etl_cronjob,
431
467
  }
432
468
  env.update({k: v for k, v in values.items() if v is not None})
433
469
  return env
@@ -6,7 +6,7 @@ usage() {
6
6
  echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
7
7
  echo " -a APPNAME The application name (example: uatgen3)"
8
8
  echo " -n NAMESPACE The namespace for the resources (example: cad)"
9
- echo " -c ETL_CRONJOB The name of the ETL cronjob to run (default: etl-cronjob)"
9
+ echo " -c ETL_CRONJOB The name of the ETL cronjob to run (default: \$G3DT_ETL_CRONJOB — the env's SSM app/etl_cronjob, set by g3dt — else etl-cronjob)"
10
10
  echo " -t CONTAINER The name of the container to check logs from (default: tube)"
11
11
  echo " -l Bypass login"
12
12
  echo " -s Sync the argocd app before restarting resources"
@@ -15,8 +15,9 @@ usage() {
15
15
 
16
16
  set -eo pipefail
17
17
 
18
- # default values
19
- ETL_CRONJOB="etl-cronjob"
18
+ # default values. The cronjob name comes from the environment's SSM tree
19
+ # (app/etl_cronjob, exported by g3dt as G3DT_ETL_CRONJOB).
20
+ ETL_CRONJOB="${G3DT_ETL_CRONJOB:-etl-cronjob}"
20
21
  CONTAINER_TO_CHECK="tube"
21
22
  LOGIN_REQUIRED=true
22
23
  SYNC_APP=false
@@ -6,8 +6,11 @@ usage() {
6
6
  echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]"
7
7
  echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
8
8
  echo " -a APPNAME The application name (example: uatgen3)"
9
- echo " -r RESOURCES Comma-separated string of microservice names to restart (example: sheepdog-deployment,peregrine-deployment )"
10
- echo " -n NAMESPACE The namespace for the resources (example: cad)"
9
+ echo " -r RESOURCES Comma-separated microservice names, restarted in order"
10
+ echo " (default: \$G3DT_RESTART_SERVICES — the env's SSM"
11
+ echo " app/restart_services, set by g3dt — else the classic"
12
+ echo " sheepdog,peregrine,guppy,portal set)"
13
+ echo " -n NAMESPACE The namespace for the resources (default: \$G3DT_NAMESPACE)"
11
14
  echo " -k KIND The kind of resource to restart (default: Deployment)"
12
15
  echo " -l Bypass login"
13
16
  echo " -s Sync the argocd app before restarting resources"
@@ -16,7 +19,10 @@ usage() {
16
19
 
17
20
  set -eo pipefail
18
21
 
19
- # Parse command line arguments
22
+ # Parse command line arguments. The restart set defaults to the environment's
23
+ # SSM app/restart_services (exported by g3dt as G3DT_RESTART_SERVICES).
24
+ IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}"
25
+ NAMESPACE="${G3DT_NAMESPACE:-}"
20
26
  LOGIN_REQUIRED=true
21
27
  KIND="Deployment"
22
28
  SYNC_APP=false
@@ -52,6 +58,11 @@ while getopts "d:a:r:n:k:hls" opt; do
52
58
  esac
53
59
  done
54
60
 
61
+ if [ -z "$NAMESPACE" ]; then
62
+ echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2
63
+ exit 1
64
+ fi
65
+
55
66
  # Check if argocd is installed
56
67
  if ! command -v argocd &> /dev/null
57
68
  then
@@ -6,8 +6,11 @@ usage() {
6
6
  echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]"
7
7
  echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
8
8
  echo " -a APPNAME The application name (example: uatgen3)"
9
- echo " -r RESOURCES Comma-separated string of microservice names to restart (default: \"sheepdog-deployment\" \"peregrine-deployment\" \"guppy-deployment\" \"portal-deployment\")"
10
- echo " -n NAMESPACE The namespace for the resources (default: cad)"
9
+ echo " -r RESOURCES Comma-separated microservice names, restarted in order"
10
+ echo " (default: \$G3DT_RESTART_SERVICES — the env's SSM"
11
+ echo " app/restart_services, set by g3dt — else the classic"
12
+ echo " sheepdog,peregrine,guppy,portal set)"
13
+ echo " -n NAMESPACE The namespace for the resources (default: \$G3DT_NAMESPACE)"
11
14
  echo " -k KIND The kind of resource to restart (default: Deployment)"
12
15
  echo " -l Bypass login"
13
16
  echo " -s Run 'argocd app sync' before restarts"
@@ -16,9 +19,11 @@ usage() {
16
19
 
17
20
  set -eo pipefail
18
21
 
19
- # Set default values
20
- RESOURCES=("sheepdog-deployment" "peregrine-deployment" "guppy-deployment" "portal-deployment")
21
- NAMESPACE="cad"
22
+ # Set default values. The restart set comes from the environment's SSM tree
23
+ # (app/restart_services, exported by g3dt as G3DT_RESTART_SERVICES); the
24
+ # classic Gen3 set is the fallback for direct invocations outside g3dt.
25
+ IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}"
26
+ NAMESPACE="${G3DT_NAMESPACE:-}"
22
27
  KIND="Deployment"
23
28
  LOGIN_REQUIRED=true
24
29
  SYNC_APP=false
@@ -56,6 +61,11 @@ while getopts "d:a:r:n:k:hls" opt; do
56
61
  esac
57
62
  done
58
63
 
64
+ if [ -z "$NAMESPACE" ]; then
65
+ echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2
66
+ exit 1
67
+ fi
68
+
59
69
  # Check if argocd is installed
60
70
  if ! command -v argocd &> /dev/null
61
71
  then
@@ -1,54 +0,0 @@
1
- """`g3dt k8s` — restart Gen3 microservices / ETL via ArgoCD (LOCAL only).
2
-
3
- These use ``argocd login --sso`` (a browser flow), so they cannot run headless
4
- on EC2. The wrapped scripts receive their settings as ``G3DT_*`` environment
5
- variables resolved from SSM — they read no config files.
6
- """
7
- from __future__ import annotations
8
-
9
- import typer
10
-
11
- from g3dt.config import script_env
12
- from g3dt.cli._internal import runner
13
- from g3dt.cli._internal.resolve import env_of
14
-
15
- app = typer.Typer(no_args_is_help=True, help="ArgoCD / Kubernetes restarts (local).")
16
-
17
- _SCHEMA = "services/k8s_ops/argocd_restart_schema.sh"
18
- _ETL = "services/k8s_ops/argocd_restart_etl.sh"
19
- _ETL_AND_MS = "services/k8s_ops/restart_etl_and_ms.sh"
20
-
21
-
22
- @app.command(name="restart-schema")
23
- def restart_schema(
24
- env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
25
- sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
26
- ) -> None:
27
- """Restart sheepdog/peregrine/guppy/portal (schema microservices)."""
28
- e = env_of(env)
29
- args = ["-d", e.domain, "-a", e.app_name, "-n", e.namespace]
30
- if sync:
31
- args.append("-s")
32
- runner.run(runner.bash_script(_SCHEMA, *args), env=script_env(e))
33
-
34
-
35
- @app.command(name="restart-etl")
36
- def restart_etl(
37
- env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
38
- sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
39
- ) -> None:
40
- """Create + run the ETL cronjob and wait for completion."""
41
- e = env_of(env)
42
- args = ["-e", env]
43
- if sync:
44
- args.append("-s")
45
- runner.run(runner.bash_script(_ETL, *args), env=script_env(e))
46
-
47
-
48
- @app.command(name="restart-ms")
49
- def restart_ms(
50
- env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
51
- ) -> None:
52
- """Restart both ETL and schema microservices (wraps restart_etl_and_ms.sh)."""
53
- e = env_of(env)
54
- runner.run(runner.bash_script(_ETL_AND_MS, env), env=script_env(e))