gen3-dataops-toolkit 3.4.0__tar.gz → 3.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/PKG-INFO +16 -1
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/README.md +15 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/pyproject.toml +1 -1
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/config_cmds.py +14 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/dict_cmds.py +9 -1
- gen3_dataops_toolkit-3.5.1/src/g3dt/cli/k8s.py +104 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/synth.py +16 -2
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/config.py +36 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +4 -3
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +14 -3
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +15 -5
- gen3_dataops_toolkit-3.4.0/src/g3dt/cli/k8s.py +0 -54
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/__init__.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/__init__.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/__init__.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/dispatch.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/registry.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/resolve.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/runner.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/safety.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/delete_cmds.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/ec2_cmds.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/indexd_cmds.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/jobs.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/main.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/metadata.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/pipeline_cmds.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/release_cmds.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/indexd/__init__.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/indexd/file_access.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/indexd/indexd_registrar.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/ingest/ingest.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/resolver.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/delete/delete_metadata.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/deploy_dd.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/indexd/register_indexd.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/indexd/verify_file_access.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/upload/metadata/upload_all_studies.sh +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/upload/metadata/upload_metadata.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/__init__.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/metadata_deleter.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/metadata_submitter.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/utils/athena_utils.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/utils/dbt_utils.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/utils/release_writer.py +0 -0
- {gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/validate/validate.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gen3-dataops-toolkit
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.5.1
|
|
4
4
|
Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: JoshuaHarris391
|
|
@@ -208,6 +208,21 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
|
|
|
208
208
|
`g3dt config show --env <env>` prints the resolved provider, model, and key
|
|
209
209
|
path.
|
|
210
210
|
|
|
211
|
+
### Kubernetes restart targets
|
|
212
|
+
|
|
213
|
+
`k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
|
|
214
|
+
restart the commons' schema microservices **serially, in a configured order**,
|
|
215
|
+
waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
|
|
216
|
+
the deploy flows run a named ETL cronjob. Both targets resolve with precedence
|
|
217
|
+
**CLI flags > SSM > default**: the CDK config's optional `k8s` block publishes
|
|
218
|
+
`app/restart_services` (comma-separated deployment names) and
|
|
219
|
+
`app/etl_cronjob`, `--restart-services` / `--etl-cronjob` override them for
|
|
220
|
+
one run, and environments deployed without the block keep the classic Gen3 set
|
|
221
|
+
(`sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment`
|
|
222
|
+
/ `etl-cronjob`). A commons that manages some service outside this flow (e.g.
|
|
223
|
+
a manually redeployed frontend) simply omits it from the list in its wrapper
|
|
224
|
+
config. `g3dt config show --env <env>` prints the resolved values.
|
|
225
|
+
|
|
211
226
|
## Verifying download access (check-download)
|
|
212
227
|
|
|
213
228
|
Registration alone does not prove a file can be downloaded. Two failure modes
|
|
@@ -174,6 +174,21 @@ g3dt synth generate AusDiab_Simulated --llm -n 5 -e test
|
|
|
174
174
|
`g3dt config show --env <env>` prints the resolved provider, model, and key
|
|
175
175
|
path.
|
|
176
176
|
|
|
177
|
+
### Kubernetes restart targets
|
|
178
|
+
|
|
179
|
+
`k8s restart-schema`, `k8s restart-ms`, `dict deploy`, and `synth deploy`
|
|
180
|
+
restart the commons' schema microservices **serially, in a configured order**,
|
|
181
|
+
waiting for each rollout to go Healthy before the next; `k8s restart-etl` and
|
|
182
|
+
the deploy flows run a named ETL cronjob. Both targets resolve with precedence
|
|
183
|
+
**CLI flags > SSM > default**: the CDK config's optional `k8s` block publishes
|
|
184
|
+
`app/restart_services` (comma-separated deployment names) and
|
|
185
|
+
`app/etl_cronjob`, `--restart-services` / `--etl-cronjob` override them for
|
|
186
|
+
one run, and environments deployed without the block keep the classic Gen3 set
|
|
187
|
+
(`sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment`
|
|
188
|
+
/ `etl-cronjob`). A commons that manages some service outside this flow (e.g.
|
|
189
|
+
a manually redeployed frontend) simply omits it from the list in its wrapper
|
|
190
|
+
config. `g3dt config show --env <env>` prints the resolved values.
|
|
191
|
+
|
|
177
192
|
## Verifying download access (check-download)
|
|
178
193
|
|
|
179
194
|
Registration alone does not prove a file can be downloaded. Two failure modes
|
|
@@ -123,6 +123,10 @@ def show(
|
|
|
123
123
|
typer.echo(f" llm_provider : {e.llm_provider}")
|
|
124
124
|
typer.echo(f" llm_model : {e.llm_model or '(not set — pass --llm-model or add the llm block to the CDK config)'}")
|
|
125
125
|
typer.echo(f" llm_api_key_file : {config.llm_api_key_file() or '(not set — g3dt config set llm_api_key_file <path>)'}")
|
|
126
|
+
# k8s restart targets: from SSM (the CDK's optional k8s block), restarted
|
|
127
|
+
# in the listed order by restart-schema/restart-ms/dict deploy/synth deploy.
|
|
128
|
+
typer.echo(f" restart_services : {e.restart_services}")
|
|
129
|
+
typer.echo(f" etl_cronjob : {e.etl_cronjob}")
|
|
126
130
|
if study:
|
|
127
131
|
s = study_of(study, env)
|
|
128
132
|
typer.secho(f"Study: {study} -> {s.key}", bold=True)
|
|
@@ -217,6 +221,16 @@ def diff(
|
|
|
217
221
|
for camel, leaf in {"provider": "llm_provider", "model": "llm_model"}.items():
|
|
218
222
|
if camel in llm:
|
|
219
223
|
check(f"llm.{camel}", llm.get(camel), rc.get(f"app/{leaf}"))
|
|
224
|
+
k8s = inputs.get("k8s") or {}
|
|
225
|
+
if "schemaRestartServices" in k8s:
|
|
226
|
+
# The CDK publishes the list comma-joined; compare the same shape.
|
|
227
|
+
check(
|
|
228
|
+
"k8s.schemaRestartServices",
|
|
229
|
+
",".join(k8s.get("schemaRestartServices") or []),
|
|
230
|
+
rc.get("app/restart_services"),
|
|
231
|
+
)
|
|
232
|
+
if "etlCronjob" in k8s:
|
|
233
|
+
check("k8s.etlCronjob", k8s.get("etlCronjob"), rc.get("app/etl_cronjob"))
|
|
220
234
|
|
|
221
235
|
if not drift:
|
|
222
236
|
typer.secho(
|
|
@@ -103,6 +103,11 @@ def deploy(
|
|
|
103
103
|
version: str = typer.Option(
|
|
104
104
|
None, "--version", help="Dictionary git tag (default: the env's version)."
|
|
105
105
|
),
|
|
106
|
+
restart_services: Optional[str] = typer.Option(
|
|
107
|
+
None, "--restart-services",
|
|
108
|
+
help="Comma-separated deployment names restarted after the upload, in "
|
|
109
|
+
"order; default: the env's SSM app/restart_services.",
|
|
110
|
+
),
|
|
106
111
|
) -> None:
|
|
107
112
|
"""Pull + upload the dictionary and restart Gen3 schema microservices.
|
|
108
113
|
|
|
@@ -123,7 +128,10 @@ def deploy(
|
|
|
123
128
|
"""
|
|
124
129
|
e = env_of(env)
|
|
125
130
|
warn_if_overridden(e, version)
|
|
131
|
+
env_vars = script_env(e, _version(e, version))
|
|
132
|
+
if restart_services:
|
|
133
|
+
env_vars["G3DT_RESTART_SERVICES"] = restart_services
|
|
126
134
|
runner.run(
|
|
127
135
|
runner.bash_script("services/dictionary/deploy_dd.sh", env),
|
|
128
|
-
env=
|
|
136
|
+
env=env_vars,
|
|
129
137
|
)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""`g3dt k8s` — restart Gen3 microservices / ETL via ArgoCD (LOCAL only).
|
|
2
|
+
|
|
3
|
+
These use ``argocd login --sso`` (a browser flow), so they cannot run headless
|
|
4
|
+
on EC2. The wrapped scripts receive their settings as ``G3DT_*`` environment
|
|
5
|
+
variables resolved from SSM — they read no config files.
|
|
6
|
+
|
|
7
|
+
The restart targets resolve with precedence CLI flags > SSM > default: the
|
|
8
|
+
CDK config's optional ``k8s`` block publishes ``app/restart_services`` (a
|
|
9
|
+
comma-separated list, restarted in order) and ``app/etl_cronjob``;
|
|
10
|
+
``--restart-services`` / ``--etl-cronjob`` override them for one run, and
|
|
11
|
+
environments deployed without the block keep the classic Gen3 set
|
|
12
|
+
(sheepdog, peregrine, guppy, portal / etl-cronjob).
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
import typer
|
|
19
|
+
|
|
20
|
+
from g3dt.config import script_env
|
|
21
|
+
from g3dt.cli._internal import runner
|
|
22
|
+
from g3dt.cli._internal.resolve import env_of
|
|
23
|
+
|
|
24
|
+
app = typer.Typer(no_args_is_help=True, help="ArgoCD / Kubernetes restarts (local).")
|
|
25
|
+
|
|
26
|
+
_SCHEMA = "services/k8s_ops/argocd_restart_schema.sh"
|
|
27
|
+
_ETL = "services/k8s_ops/argocd_restart_etl.sh"
|
|
28
|
+
_ETL_AND_MS = "services/k8s_ops/restart_etl_and_ms.sh"
|
|
29
|
+
|
|
30
|
+
_RESTART_SERVICES_HELP = (
|
|
31
|
+
"Comma-separated deployment names to restart, in order; default: the "
|
|
32
|
+
"env's SSM app/restart_services (the CDK config's k8s.schemaRestartServices)."
|
|
33
|
+
)
|
|
34
|
+
_ETL_CRONJOB_HELP = (
|
|
35
|
+
"ETL cronjob name; default: the env's SSM app/etl_cronjob "
|
|
36
|
+
"(the CDK config's k8s.etlCronjob)."
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def restart_env(e, restart_services: Optional[str] = None,
|
|
41
|
+
etl_cronjob: Optional[str] = None) -> dict:
|
|
42
|
+
"""script_env plus per-run restart-target overrides (flags beat SSM)."""
|
|
43
|
+
env_vars = script_env(e)
|
|
44
|
+
if restart_services:
|
|
45
|
+
env_vars["G3DT_RESTART_SERVICES"] = restart_services
|
|
46
|
+
if etl_cronjob:
|
|
47
|
+
env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob
|
|
48
|
+
return env_vars
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.command(name="restart-schema")
|
|
52
|
+
def restart_schema(
|
|
53
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
54
|
+
sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
|
|
55
|
+
restart_services: Optional[str] = typer.Option(
|
|
56
|
+
None, "--restart-services", help=_RESTART_SERVICES_HELP
|
|
57
|
+
),
|
|
58
|
+
) -> None:
|
|
59
|
+
"""Restart the schema microservices, in the env's configured order."""
|
|
60
|
+
e = env_of(env)
|
|
61
|
+
args = ["-d", e.domain, "-a", e.app_name, "-n", e.namespace]
|
|
62
|
+
if sync:
|
|
63
|
+
args.append("-s")
|
|
64
|
+
runner.run(
|
|
65
|
+
runner.bash_script(_SCHEMA, *args),
|
|
66
|
+
env=restart_env(e, restart_services=restart_services),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@app.command(name="restart-etl")
|
|
71
|
+
def restart_etl(
|
|
72
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
73
|
+
sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
|
|
74
|
+
etl_cronjob: Optional[str] = typer.Option(
|
|
75
|
+
None, "--etl-cronjob", help=_ETL_CRONJOB_HELP
|
|
76
|
+
),
|
|
77
|
+
) -> None:
|
|
78
|
+
"""Create + run the ETL cronjob and wait for completion."""
|
|
79
|
+
e = env_of(env)
|
|
80
|
+
args = ["-e", env]
|
|
81
|
+
if sync:
|
|
82
|
+
args.append("-s")
|
|
83
|
+
runner.run(
|
|
84
|
+
runner.bash_script(_ETL, *args),
|
|
85
|
+
env=restart_env(e, etl_cronjob=etl_cronjob),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@app.command(name="restart-ms")
|
|
90
|
+
def restart_ms(
|
|
91
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
92
|
+
restart_services: Optional[str] = typer.Option(
|
|
93
|
+
None, "--restart-services", help=_RESTART_SERVICES_HELP
|
|
94
|
+
),
|
|
95
|
+
etl_cronjob: Optional[str] = typer.Option(
|
|
96
|
+
None, "--etl-cronjob", help=_ETL_CRONJOB_HELP
|
|
97
|
+
),
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Restart both ETL and schema microservices (wraps restart_etl_and_ms.sh)."""
|
|
100
|
+
e = env_of(env)
|
|
101
|
+
runner.run(
|
|
102
|
+
runner.bash_script(_ETL_AND_MS, env),
|
|
103
|
+
env=restart_env(e, restart_services=restart_services, etl_cronjob=etl_cronjob),
|
|
104
|
+
)
|
|
@@ -168,17 +168,31 @@ def deploy(
|
|
|
168
168
|
help="Path to the file holding the LLM API key; default: the marker's "
|
|
169
169
|
"llm_api_key_file (set once: g3dt config set llm_api_key_file <path>).",
|
|
170
170
|
),
|
|
171
|
+
restart_services: Optional[str] = typer.Option(
|
|
172
|
+
None, "--restart-services",
|
|
173
|
+
help="Comma-separated deployment names restarted during the deploy, in "
|
|
174
|
+
"order; default: the env's SSM app/restart_services.",
|
|
175
|
+
),
|
|
176
|
+
etl_cronjob: Optional[str] = typer.Option(
|
|
177
|
+
None, "--etl-cronjob",
|
|
178
|
+
help="ETL cronjob name; default: the env's SSM app/etl_cronjob.",
|
|
179
|
+
),
|
|
171
180
|
) -> None:
|
|
172
181
|
"""Full end-to-end synthetic deploy (dict + LLM-generate + upload + restarts).
|
|
173
182
|
|
|
174
183
|
Wraps services/synthetic_data/full_deploy_dd_and_synth.sh (LLM-backed
|
|
175
|
-
generation). Provider/model come from the env's
|
|
176
|
-
overridden; the API key path comes from
|
|
184
|
+
generation). Provider/model and the restart targets come from the env's
|
|
185
|
+
SSM tree unless overridden; the API key path comes from
|
|
186
|
+
--llm-api-key-file or the marker.
|
|
177
187
|
"""
|
|
178
188
|
e = env_of(env)
|
|
179
189
|
safety.confirm_prod_strict("synthetic full deploy", env)
|
|
180
190
|
env_vars = script_env(e)
|
|
181
191
|
env_vars.update(_llm_env_overrides(e, llm_provider, llm_model, llm_api_key_file))
|
|
192
|
+
if restart_services:
|
|
193
|
+
env_vars["G3DT_RESTART_SERVICES"] = restart_services
|
|
194
|
+
if etl_cronjob:
|
|
195
|
+
env_vars["G3DT_ETL_CRONJOB"] = etl_cronjob
|
|
182
196
|
runner.run(
|
|
183
197
|
runner.bash_script(
|
|
184
198
|
"services/synthetic_data/full_deploy_dd_and_synth.sh", env
|
|
@@ -58,6 +58,17 @@ DEFAULT_DICT_PATH = "dictionary/prod_dict/acdc_schema.json"
|
|
|
58
58
|
#: when the ``--llm`` path is used and no model is configured anywhere.
|
|
59
59
|
DEFAULT_LLM_PROVIDER = "anthropic"
|
|
60
60
|
|
|
61
|
+
#: Kubernetes restart targets, published by the CDK's OPTIONAL ``k8s`` config
|
|
62
|
+
#: block as ``app/restart_services`` (comma-separated, restarted in order) and
|
|
63
|
+
#: ``app/etl_cronjob``. Optional app inputs: environments deployed without the
|
|
64
|
+
#: block keep the classic Gen3 set. Consumed by every restart path — `g3dt
|
|
65
|
+
#: k8s restart-schema/restart-etl/restart-ms`, `dict deploy`, `synth deploy` —
|
|
66
|
+
#: via $G3DT_RESTART_SERVICES / $G3DT_ETL_CRONJOB in the service scripts.
|
|
67
|
+
DEFAULT_RESTART_SERVICES = (
|
|
68
|
+
"sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment"
|
|
69
|
+
)
|
|
70
|
+
DEFAULT_ETL_CRONJOB = "etl-cronjob"
|
|
71
|
+
|
|
61
72
|
#: Marker locations, most specific first.
|
|
62
73
|
MARKER_PATHS = ("g3dt.yaml", "~/.g3dt/g3dt.yaml", "/etc/g3dt/g3dt.yaml")
|
|
63
74
|
|
|
@@ -257,6 +268,9 @@ class EnvConfig:
|
|
|
257
268
|
# with guidance rather than silently picking a model.
|
|
258
269
|
llm_provider: str = DEFAULT_LLM_PROVIDER
|
|
259
270
|
llm_model: Optional[str] = None
|
|
271
|
+
# Optional k8s restart targets (SSM app/restart_services, app/etl_cronjob).
|
|
272
|
+
restart_services: str = DEFAULT_RESTART_SERVICES
|
|
273
|
+
etl_cronjob: str = DEFAULT_ETL_CRONJOB
|
|
260
274
|
|
|
261
275
|
|
|
262
276
|
def _app_or_default(rc, leaf: str, default: str) -> str:
|
|
@@ -320,6 +334,10 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
|
|
|
320
334
|
# the config has an llm block, so absence means "use the defaults".
|
|
321
335
|
llm_provider=_app_or_default(rc, "llm_provider", DEFAULT_LLM_PROVIDER),
|
|
322
336
|
llm_model=(_app_or_default(rc, "llm_model", "") or None),
|
|
337
|
+
restart_services=_app_or_default(
|
|
338
|
+
rc, "restart_services", DEFAULT_RESTART_SERVICES
|
|
339
|
+
),
|
|
340
|
+
etl_cronjob=_app_or_default(rc, "etl_cronjob", DEFAULT_ETL_CRONJOB),
|
|
323
341
|
)
|
|
324
342
|
|
|
325
343
|
|
|
@@ -410,8 +428,24 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
|
|
|
410
428
|
scripts compose a URL keeps Python the single source of truth: the scripts
|
|
411
429
|
read ``G3DT_DICT_URL``/``G3DT_DICT_FILENAME`` and never build either
|
|
412
430
|
themselves, so there is only one implementation to keep correct.
|
|
431
|
+
|
|
432
|
+
The interpreter's own bin directory is prepended to PATH: console scripts
|
|
433
|
+
installed next to g3dt — gen3-metadata-simulator via
|
|
434
|
+
``g3dt synth install-simulator`` — land there, but a pipx install exposes
|
|
435
|
+
only g3dt's own entry points on the caller's PATH, so without this the
|
|
436
|
+
wrapped scripts' ``command -v gen3-metadata-simulator`` check fails even
|
|
437
|
+
though the tool is installed.
|
|
413
438
|
"""
|
|
439
|
+
import sys
|
|
440
|
+
|
|
414
441
|
env = dict(os.environ)
|
|
442
|
+
# No .resolve(): a venv's python is a symlink to the base interpreter, and
|
|
443
|
+
# resolving it would point at the base install's bin instead of the venv
|
|
444
|
+
# bin where console scripts are actually created.
|
|
445
|
+
venv_bin = str(Path(sys.executable).parent)
|
|
446
|
+
path_entries = env.get("PATH", "").split(os.pathsep)
|
|
447
|
+
if venv_bin not in path_entries:
|
|
448
|
+
env["PATH"] = venv_bin + os.pathsep + env.get("PATH", "")
|
|
415
449
|
values = {
|
|
416
450
|
"G3DT_ENV": e.name,
|
|
417
451
|
"G3DT_REGION": e.region,
|
|
@@ -428,6 +462,8 @@ def script_env(e: EnvConfig, version: Optional[str] = None) -> Dict[str, str]:
|
|
|
428
462
|
"G3DT_SCHEMA_REPO": e.schema_repo,
|
|
429
463
|
"G3DT_LLM_PROVIDER": e.llm_provider,
|
|
430
464
|
"G3DT_LLM_MODEL": e.llm_model,
|
|
465
|
+
"G3DT_RESTART_SERVICES": e.restart_services,
|
|
466
|
+
"G3DT_ETL_CRONJOB": e.etl_cronjob,
|
|
431
467
|
}
|
|
432
468
|
env.update({k: v for k, v in values.items() if v is not None})
|
|
433
469
|
return env
|
|
@@ -6,7 +6,7 @@ usage() {
|
|
|
6
6
|
echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
|
|
7
7
|
echo " -a APPNAME The application name (example: uatgen3)"
|
|
8
8
|
echo " -n NAMESPACE The namespace for the resources (example: cad)"
|
|
9
|
-
echo " -c ETL_CRONJOB The name of the ETL cronjob to run (default: etl-cronjob)"
|
|
9
|
+
echo " -c ETL_CRONJOB The name of the ETL cronjob to run (default: \$G3DT_ETL_CRONJOB — the env's SSM app/etl_cronjob, set by g3dt — else etl-cronjob)"
|
|
10
10
|
echo " -t CONTAINER The name of the container to check logs from (default: tube)"
|
|
11
11
|
echo " -l Bypass login"
|
|
12
12
|
echo " -s Sync the argocd app before restarting resources"
|
|
@@ -15,8 +15,9 @@ usage() {
|
|
|
15
15
|
|
|
16
16
|
set -eo pipefail
|
|
17
17
|
|
|
18
|
-
# default values
|
|
19
|
-
|
|
18
|
+
# default values. The cronjob name comes from the environment's SSM tree
|
|
19
|
+
# (app/etl_cronjob, exported by g3dt as G3DT_ETL_CRONJOB).
|
|
20
|
+
ETL_CRONJOB="${G3DT_ETL_CRONJOB:-etl-cronjob}"
|
|
20
21
|
CONTAINER_TO_CHECK="tube"
|
|
21
22
|
LOGIN_REQUIRED=true
|
|
22
23
|
SYNC_APP=false
|
|
@@ -6,8 +6,11 @@ usage() {
|
|
|
6
6
|
echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]"
|
|
7
7
|
echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
|
|
8
8
|
echo " -a APPNAME The application name (example: uatgen3)"
|
|
9
|
-
echo " -r RESOURCES Comma-separated
|
|
10
|
-
echo "
|
|
9
|
+
echo " -r RESOURCES Comma-separated microservice names, restarted in order"
|
|
10
|
+
echo " (default: \$G3DT_RESTART_SERVICES — the env's SSM"
|
|
11
|
+
echo " app/restart_services, set by g3dt — else the classic"
|
|
12
|
+
echo " sheepdog,peregrine,guppy,portal set)"
|
|
13
|
+
echo " -n NAMESPACE The namespace for the resources (default: \$G3DT_NAMESPACE)"
|
|
11
14
|
echo " -k KIND The kind of resource to restart (default: Deployment)"
|
|
12
15
|
echo " -l Bypass login"
|
|
13
16
|
echo " -s Sync the argocd app before restarting resources"
|
|
@@ -16,7 +19,10 @@ usage() {
|
|
|
16
19
|
|
|
17
20
|
set -eo pipefail
|
|
18
21
|
|
|
19
|
-
# Parse command line arguments
|
|
22
|
+
# Parse command line arguments. The restart set defaults to the environment's
|
|
23
|
+
# SSM app/restart_services (exported by g3dt as G3DT_RESTART_SERVICES).
|
|
24
|
+
IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}"
|
|
25
|
+
NAMESPACE="${G3DT_NAMESPACE:-}"
|
|
20
26
|
LOGIN_REQUIRED=true
|
|
21
27
|
KIND="Deployment"
|
|
22
28
|
SYNC_APP=false
|
|
@@ -52,6 +58,11 @@ while getopts "d:a:r:n:k:hls" opt; do
|
|
|
52
58
|
esac
|
|
53
59
|
done
|
|
54
60
|
|
|
61
|
+
if [ -z "$NAMESPACE" ]; then
|
|
62
|
+
echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2
|
|
63
|
+
exit 1
|
|
64
|
+
fi
|
|
65
|
+
|
|
55
66
|
# Check if argocd is installed
|
|
56
67
|
if ! command -v argocd &> /dev/null
|
|
57
68
|
then
|
|
@@ -6,8 +6,11 @@ usage() {
|
|
|
6
6
|
echo "Usage: $0 [-d DOMAIN] [-a APPNAME] [-r RESOURCES] [-n NAMESPACE] [-k KIND] [-l] [-s]"
|
|
7
7
|
echo " -d DOMAIN The domain for argocd login (example: cd.cad.test.biocommons.org.au)"
|
|
8
8
|
echo " -a APPNAME The application name (example: uatgen3)"
|
|
9
|
-
echo " -r RESOURCES Comma-separated
|
|
10
|
-
echo "
|
|
9
|
+
echo " -r RESOURCES Comma-separated microservice names, restarted in order"
|
|
10
|
+
echo " (default: \$G3DT_RESTART_SERVICES — the env's SSM"
|
|
11
|
+
echo " app/restart_services, set by g3dt — else the classic"
|
|
12
|
+
echo " sheepdog,peregrine,guppy,portal set)"
|
|
13
|
+
echo " -n NAMESPACE The namespace for the resources (default: \$G3DT_NAMESPACE)"
|
|
11
14
|
echo " -k KIND The kind of resource to restart (default: Deployment)"
|
|
12
15
|
echo " -l Bypass login"
|
|
13
16
|
echo " -s Run 'argocd app sync' before restarts"
|
|
@@ -16,9 +19,11 @@ usage() {
|
|
|
16
19
|
|
|
17
20
|
set -eo pipefail
|
|
18
21
|
|
|
19
|
-
# Set default values
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
# Set default values. The restart set comes from the environment's SSM tree
|
|
23
|
+
# (app/restart_services, exported by g3dt as G3DT_RESTART_SERVICES); the
|
|
24
|
+
# classic Gen3 set is the fallback for direct invocations outside g3dt.
|
|
25
|
+
IFS=',' read -r -a RESOURCES <<< "${G3DT_RESTART_SERVICES:-sheepdog-deployment,peregrine-deployment,guppy-deployment,portal-deployment}"
|
|
26
|
+
NAMESPACE="${G3DT_NAMESPACE:-}"
|
|
22
27
|
KIND="Deployment"
|
|
23
28
|
LOGIN_REQUIRED=true
|
|
24
29
|
SYNC_APP=false
|
|
@@ -56,6 +61,11 @@ while getopts "d:a:r:n:k:hls" opt; do
|
|
|
56
61
|
esac
|
|
57
62
|
done
|
|
58
63
|
|
|
64
|
+
if [ -z "$NAMESPACE" ]; then
|
|
65
|
+
echo "No namespace given: pass -n, or run via the g3dt CLI (which sets G3DT_NAMESPACE)." >&2
|
|
66
|
+
exit 1
|
|
67
|
+
fi
|
|
68
|
+
|
|
59
69
|
# Check if argocd is installed
|
|
60
70
|
if ! command -v argocd &> /dev/null
|
|
61
71
|
then
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
"""`g3dt k8s` — restart Gen3 microservices / ETL via ArgoCD (LOCAL only).
|
|
2
|
-
|
|
3
|
-
These use ``argocd login --sso`` (a browser flow), so they cannot run headless
|
|
4
|
-
on EC2. The wrapped scripts receive their settings as ``G3DT_*`` environment
|
|
5
|
-
variables resolved from SSM — they read no config files.
|
|
6
|
-
"""
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import typer
|
|
10
|
-
|
|
11
|
-
from g3dt.config import script_env
|
|
12
|
-
from g3dt.cli._internal import runner
|
|
13
|
-
from g3dt.cli._internal.resolve import env_of
|
|
14
|
-
|
|
15
|
-
app = typer.Typer(no_args_is_help=True, help="ArgoCD / Kubernetes restarts (local).")
|
|
16
|
-
|
|
17
|
-
_SCHEMA = "services/k8s_ops/argocd_restart_schema.sh"
|
|
18
|
-
_ETL = "services/k8s_ops/argocd_restart_etl.sh"
|
|
19
|
-
_ETL_AND_MS = "services/k8s_ops/restart_etl_and_ms.sh"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
@app.command(name="restart-schema")
|
|
23
|
-
def restart_schema(
|
|
24
|
-
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
25
|
-
sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
|
|
26
|
-
) -> None:
|
|
27
|
-
"""Restart sheepdog/peregrine/guppy/portal (schema microservices)."""
|
|
28
|
-
e = env_of(env)
|
|
29
|
-
args = ["-d", e.domain, "-a", e.app_name, "-n", e.namespace]
|
|
30
|
-
if sync:
|
|
31
|
-
args.append("-s")
|
|
32
|
-
runner.run(runner.bash_script(_SCHEMA, *args), env=script_env(e))
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
@app.command(name="restart-etl")
|
|
36
|
-
def restart_etl(
|
|
37
|
-
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
38
|
-
sync: bool = typer.Option(False, "--sync", "-s", help="argocd app sync first."),
|
|
39
|
-
) -> None:
|
|
40
|
-
"""Create + run the ETL cronjob and wait for completion."""
|
|
41
|
-
e = env_of(env)
|
|
42
|
-
args = ["-e", env]
|
|
43
|
-
if sync:
|
|
44
|
-
args.append("-s")
|
|
45
|
-
runner.run(runner.bash_script(_ETL, *args), env=script_env(e))
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
@app.command(name="restart-ms")
|
|
49
|
-
def restart_ms(
|
|
50
|
-
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
51
|
-
) -> None:
|
|
52
|
-
"""Restart both ETL and schema microservices (wraps restart_etl_and_ms.sh)."""
|
|
53
|
-
e = env_of(env)
|
|
54
|
-
runner.run(runner.bash_script(_ETL_AND_MS, env), env=script_env(e))
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/__init__.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/dispatch.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/cli/_internal/registry.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/indexd/indexd_registrar.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/deploy_dd.sh
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/dictionary/pull_dict.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/services/k8s_ops/login_to_pod.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/metadata_deleter.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/metadata_submitter.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-3.4.0 → gen3_dataops_toolkit-3.5.1}/src/g3dt/upload/upload_synthdata_s3.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|