gen3-dataops-toolkit 2.2.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/PKG-INFO +42 -1
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/README.md +40 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/pyproject.toml +2 -1
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/safety.py +0 -12
- gen3_dataops_toolkit-2.3.0/src/g3dt/cli/delete_cmds.py +212 -0
- gen3_dataops_toolkit-2.3.0/src/g3dt/cli/indexd_cmds.py +126 -0
- gen3_dataops_toolkit-2.3.0/src/g3dt/cli/metadata.py +133 -0
- gen3_dataops_toolkit-2.3.0/src/g3dt/indexd/file_access.py +274 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/indexd/indexd_registrar.py +95 -4
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/ingest/ingest.py +117 -33
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/delete/delete_metadata.sh +52 -18
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/indexd/register_indexd.py +38 -0
- gen3_dataops_toolkit-2.3.0/src/g3dt/services/indexd/verify_file_access.py +126 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/upload/metadata/upload_all_studies.sh +29 -8
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/upload/metadata/upload_metadata.py +34 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/metadata_submitter.py +81 -8
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/utils/athena_utils.py +79 -0
- gen3_dataops_toolkit-2.2.0/src/g3dt/cli/delete_cmds.py +0 -101
- gen3_dataops_toolkit-2.2.0/src/g3dt/cli/indexd_cmds.py +0 -57
- gen3_dataops_toolkit-2.2.0/src/g3dt/cli/metadata.py +0 -76
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/__init__.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/__init__.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/__init__.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/dispatch.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/registry.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/resolve.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/runner.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/config_cmds.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/dict_cmds.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/ec2_cmds.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/jobs.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/k8s.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/main.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/pipeline_cmds.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/release_cmds.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/synth.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/config.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/indexd/__init__.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/resolver.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/dictionary/deploy_dd.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/__init__.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/metadata_deleter.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/utils/dbt_utils.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/utils/release_writer.py +0 -0
- {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/validate/validate.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gen3-dataops-toolkit
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: JoshuaHarris391
|
|
@@ -19,6 +19,7 @@ Requires-Dist: gen3 (>=4.27.4,<5.0.0)
|
|
|
19
19
|
Requires-Dist: gen3-metadata (>=1.4.0,<2.0.0)
|
|
20
20
|
Requires-Dist: gen3_validator (>=2.0.0,<3.0.0)
|
|
21
21
|
Requires-Dist: numpy (<2.0.0)
|
|
22
|
+
Requires-Dist: openpyxl (>=3.1.0)
|
|
22
23
|
Requires-Dist: pyarrow (>=14.0.0,<19.0.0)
|
|
23
24
|
Requires-Dist: pyjwt (>=2.10.1,<3.0.0)
|
|
24
25
|
Requires-Dist: python-dotenv
|
|
@@ -175,6 +176,46 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
|
|
|
175
176
|
refuses a batch that doesn't match the version being uploaded (override with
|
|
176
177
|
`--allow-version-mismatch`).
|
|
177
178
|
|
|
179
|
+
## Verifying download access (check-download)
|
|
180
|
+
|
|
181
|
+
Registration alone does not prove a file can be downloaded. Two failure modes
|
|
182
|
+
are invisible until a user clicks the file in the portal: an Indexd record
|
|
183
|
+
with no storage URL (nothing to download, ever), and a record Fence refuses
|
|
184
|
+
to sign a URL for. `g3dt indexd check-download` walks the exact chain the
|
|
185
|
+
portal hits — Indexd record → storage URL → DRS object → access methods →
|
|
186
|
+
Fence signed URL — and reports PASS/FAIL per object, exiting non-zero if any
|
|
187
|
+
object fails so it can gate a deployment step.
|
|
188
|
+
|
|
189
|
+
Run it before a release, and after registering new files. The env selects the
|
|
190
|
+
API key secret and the key's JWT selects the commons, so there is no URL to
|
|
191
|
+
pass (and none to get wrong).
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
g3dt indexd check-download --env staging # sample the 25 newest
|
|
195
|
+
g3dt indexd check-download --env staging --limit 50
|
|
196
|
+
g3dt indexd check-download --env prod PREFIX/<uuid-1> PREFIX/<uuid-2>
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
With no GUIDs, the newest objects for the env's commons are sampled from the
|
|
200
|
+
indexd registry (latest revision per baseid). The registry may live in a
|
|
201
|
+
different AWS account than the commons being checked; if the env's AWS
|
|
202
|
+
profile cannot reach it, pass GUIDs explicitly.
|
|
203
|
+
|
|
204
|
+
Reading a failure:
|
|
205
|
+
|
|
206
|
+
| Symptom | Meaning |
|
|
207
|
+
|---|---|
|
|
208
|
+
| `Indexd status: 404` | the object is not registered — a registration problem, not a download one |
|
|
209
|
+
| `urls: []` / no access methods | registered but with no storage location; it can never download |
|
|
210
|
+
| `Access endpoint … 401` | authorization: the API key's user lacks `read-storage` on the record's `authz` resource — an authz gap, not a broken key |
|
|
211
|
+
| `Access endpoint … 500` | Fence has the permission but failed to sign — a service-side fault |
|
|
212
|
+
|
|
213
|
+
On a 401, compare what the record requires
|
|
214
|
+
(`https://commons.example.org/index/<did>`, the `authz` field) with what the
|
|
215
|
+
key's user actually holds (`https://commons.example.org/user/user`):
|
|
216
|
+
downloads require `read-storage` on the record's authz resource, which a user
|
|
217
|
+
holding only `create` does not have.
|
|
218
|
+
|
|
178
219
|
## Development
|
|
179
220
|
|
|
180
221
|
```bash
|
|
@@ -142,6 +142,46 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
|
|
|
142
142
|
refuses a batch that doesn't match the version being uploaded (override with
|
|
143
143
|
`--allow-version-mismatch`).
|
|
144
144
|
|
|
145
|
+
## Verifying download access (check-download)
|
|
146
|
+
|
|
147
|
+
Registration alone does not prove a file can be downloaded. Two failure modes
|
|
148
|
+
are invisible until a user clicks the file in the portal: an Indexd record
|
|
149
|
+
with no storage URL (nothing to download, ever), and a record Fence refuses
|
|
150
|
+
to sign a URL for. `g3dt indexd check-download` walks the exact chain the
|
|
151
|
+
portal hits — Indexd record → storage URL → DRS object → access methods →
|
|
152
|
+
Fence signed URL — and reports PASS/FAIL per object, exiting non-zero if any
|
|
153
|
+
object fails so it can gate a deployment step.
|
|
154
|
+
|
|
155
|
+
Run it before a release, and after registering new files. The env selects the
|
|
156
|
+
API key secret and the key's JWT selects the commons, so there is no URL to
|
|
157
|
+
pass (and none to get wrong).
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
g3dt indexd check-download --env staging # sample the 25 newest
|
|
161
|
+
g3dt indexd check-download --env staging --limit 50
|
|
162
|
+
g3dt indexd check-download --env prod PREFIX/<uuid-1> PREFIX/<uuid-2>
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
With no GUIDs, the newest objects for the env's commons are sampled from the
|
|
166
|
+
indexd registry (latest revision per baseid). The registry may live in a
|
|
167
|
+
different AWS account than the commons being checked; if the env's AWS
|
|
168
|
+
profile cannot reach it, pass GUIDs explicitly.
|
|
169
|
+
|
|
170
|
+
Reading a failure:
|
|
171
|
+
|
|
172
|
+
| Symptom | Meaning |
|
|
173
|
+
|---|---|
|
|
174
|
+
| `Indexd status: 404` | the object is not registered — a registration problem, not a download one |
|
|
175
|
+
| `urls: []` / no access methods | registered but with no storage location; it can never download |
|
|
176
|
+
| `Access endpoint … 401` | authorization: the API key's user lacks `read-storage` on the record's `authz` resource — an authz gap, not a broken key |
|
|
177
|
+
| `Access endpoint … 500` | Fence has the permission but failed to sign — a service-side fault |
|
|
178
|
+
|
|
179
|
+
On a 401, compare what the record requires
|
|
180
|
+
(`https://commons.example.org/index/<did>`, the `authz` field) with what the
|
|
181
|
+
key's user actually holds (`https://commons.example.org/user/user`):
|
|
182
|
+
downloads require `read-storage` on the record's authz resource, which a user
|
|
183
|
+
holding only `create` does not have.
|
|
184
|
+
|
|
145
185
|
## Development
|
|
146
186
|
|
|
147
187
|
```bash
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "gen3-dataops-toolkit"
|
|
3
|
-
version = "2.
|
|
3
|
+
version = "2.3.0"
|
|
4
4
|
description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
|
|
5
5
|
authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -28,6 +28,7 @@ awswrangler = ">=3.14.0,<4.0.0"
|
|
|
28
28
|
pyarrow = ">=14.0.0,<19.0.0"
|
|
29
29
|
gen3-metadata = ">=1.4.0,<2.0.0"
|
|
30
30
|
typer = ">=0.12"
|
|
31
|
+
openpyxl = ">=3.1.0"
|
|
31
32
|
|
|
32
33
|
[tool.poetry.scripts]
|
|
33
34
|
g3dt = "g3dt.cli.main:main"
|
|
@@ -33,18 +33,6 @@ def require_test_env(env: str) -> None:
|
|
|
33
33
|
raise typer.Exit(2)
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
def abort_if_prod(env: str) -> None:
|
|
37
|
-
"""Hard abort for bulk operations that must never touch production."""
|
|
38
|
-
if is_prod(env):
|
|
39
|
-
typer.secho(
|
|
40
|
-
f"Refusing bulk operation against a production environment "
|
|
41
|
-
f"('{env}').",
|
|
42
|
-
fg=typer.colors.RED,
|
|
43
|
-
err=True,
|
|
44
|
-
)
|
|
45
|
-
raise typer.Exit(2)
|
|
46
|
-
|
|
47
|
-
|
|
48
36
|
def confirm_destructive(action: str, target: str, env: str, assume_yes: bool) -> None:
|
|
49
37
|
"""Gate a destructive operation with an appropriate confirmation.
|
|
50
38
|
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""`g3dt delete` — destructive metadata removal (data-plane).
|
|
2
|
+
|
|
3
|
+
A single ``delete metadata`` command handles one or many studies, sequentially,
|
|
4
|
+
in a single job. ``--version`` is required: pass a specific version (e.g.
|
|
5
|
+
``0.9.8``) to remove just that version (resolved via an Athena GUID lookup), or
|
|
6
|
+
``all`` to remove every version.
|
|
7
|
+
|
|
8
|
+
Every command confirms before acting. Production always requires typing the
|
|
9
|
+
target id, even with ``--yes``. Deleting ALL versions always prompts, even with
|
|
10
|
+
``--yes``. Confirmation happens locally before any EC2 dispatch (SSM has no
|
|
11
|
+
TTY), after which the remote job runs non-interactively.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
import typer
|
|
18
|
+
|
|
19
|
+
from g3dt.cli._internal import dispatch, safety
|
|
20
|
+
from g3dt.cli._internal.dispatch import Target
|
|
21
|
+
from g3dt.cli._internal.resolve import study_of
|
|
22
|
+
|
|
23
|
+
app = typer.Typer(no_args_is_help=True, help="Delete metadata from Gen3 (destructive).")
|
|
24
|
+
|
|
25
|
+
_DELETE_METADATA = "services/delete/delete_metadata.sh"
|
|
26
|
+
|
|
27
|
+
#: A version token in the form the Athena ``version`` column stores it. The
|
|
28
|
+
#: uploader writes ``group(1)`` of this same pattern (metadata_submitter's
|
|
29
|
+
#: ``_find_version_from_path``), i.e. WITHOUT any leading ``v``. The delete
|
|
30
|
+
#: query interpolates the string straight into SQL, so a ``v``-prefixed version
|
|
31
|
+
#: matches zero rows and is reported as "skipped" rather than as an error — a
|
|
32
|
+
#: silent no-op that reads as a clean run. Normalising here closes that.
|
|
33
|
+
_VERSION_RE = re.compile(r"^v?(\d+\.\d+\.\d+)$", re.IGNORECASE)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _normalise_version(raw: str, where: str) -> str:
|
|
37
|
+
"""Canonicalise one version token to the form stored in Athena.
|
|
38
|
+
|
|
39
|
+
``all`` in any case becomes ``all``; ``v1.5.4`` and ``1.5.4`` both become
|
|
40
|
+
``1.5.4``. Anything else is a usage error: the column only ever holds
|
|
41
|
+
three-part semver, so a truncated version like ``0.9`` would match nothing
|
|
42
|
+
and be counted as a skip.
|
|
43
|
+
"""
|
|
44
|
+
token = raw.strip()
|
|
45
|
+
if token.lower() == "all":
|
|
46
|
+
return "all"
|
|
47
|
+
match = _VERSION_RE.match(token)
|
|
48
|
+
if not match:
|
|
49
|
+
typer.secho(
|
|
50
|
+
f"Invalid version '{raw}' {where}: expected x.y.z (e.g. 0.9.8) "
|
|
51
|
+
"or 'all'.",
|
|
52
|
+
fg=typer.colors.RED,
|
|
53
|
+
err=True,
|
|
54
|
+
)
|
|
55
|
+
raise typer.Exit(2)
|
|
56
|
+
return match.group(1)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _parse_study_specs(studies: str, fallback, env: str):
|
|
60
|
+
"""Turn ``--studies`` into ``[(resolved_study_key, version), ...]``.
|
|
61
|
+
|
|
62
|
+
Each comma-separated entry is ``name`` or ``name:version``. A bare name
|
|
63
|
+
takes *fallback* (the ``--version`` default); *fallback* is ``None`` when
|
|
64
|
+
``--version`` was not given, which makes a bare name a usage error.
|
|
65
|
+
|
|
66
|
+
Every entry is validated before anything is dispatched, so a typo in the
|
|
67
|
+
last study cannot leave the earlier ones already deleted.
|
|
68
|
+
"""
|
|
69
|
+
specs = []
|
|
70
|
+
for entry in studies.split(","):
|
|
71
|
+
entry = entry.strip()
|
|
72
|
+
if not entry:
|
|
73
|
+
continue
|
|
74
|
+
|
|
75
|
+
# partition() rather than split(), so a trailing colon ("ausdiab:") is
|
|
76
|
+
# distinguishable from a bare name and can be rejected instead of
|
|
77
|
+
# silently taking the fallback.
|
|
78
|
+
name, sep, raw_version = entry.partition(":")
|
|
79
|
+
name = name.strip()
|
|
80
|
+
|
|
81
|
+
if not name:
|
|
82
|
+
typer.secho(
|
|
83
|
+
f"Invalid --studies entry '{entry}': missing study name.",
|
|
84
|
+
fg=typer.colors.RED,
|
|
85
|
+
err=True,
|
|
86
|
+
)
|
|
87
|
+
raise typer.Exit(2)
|
|
88
|
+
|
|
89
|
+
if sep and not raw_version.strip():
|
|
90
|
+
typer.secho(
|
|
91
|
+
f"Invalid --studies entry '{entry}': ':' with no version. "
|
|
92
|
+
f"Use '{name}:0.9.8', '{name}:all', or a bare '{name}' to take "
|
|
93
|
+
"the --version default.",
|
|
94
|
+
fg=typer.colors.RED,
|
|
95
|
+
err=True,
|
|
96
|
+
)
|
|
97
|
+
raise typer.Exit(2)
|
|
98
|
+
|
|
99
|
+
if sep:
|
|
100
|
+
version = _normalise_version(raw_version, f"for study '{name}'")
|
|
101
|
+
elif fallback is not None:
|
|
102
|
+
version = fallback
|
|
103
|
+
else:
|
|
104
|
+
typer.secho(
|
|
105
|
+
f"No version for study '{name}': add ':<version>' to it "
|
|
106
|
+
f"(e.g. '{name}:0.9.8'), or pass --version as the default for "
|
|
107
|
+
"every study. Use 'all' to delete every version.",
|
|
108
|
+
fg=typer.colors.RED,
|
|
109
|
+
err=True,
|
|
110
|
+
)
|
|
111
|
+
raise typer.Exit(2)
|
|
112
|
+
|
|
113
|
+
specs.append((study_of(name, env).key, version))
|
|
114
|
+
|
|
115
|
+
if not specs:
|
|
116
|
+
typer.secho("--studies is empty.", fg=typer.colors.RED, err=True)
|
|
117
|
+
raise typer.Exit(2)
|
|
118
|
+
return specs
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@app.command()
|
|
122
|
+
def metadata(
|
|
123
|
+
studies: str = typer.Option(
|
|
124
|
+
...,
|
|
125
|
+
"--studies",
|
|
126
|
+
help="Comma-separated studies, each optionally 'name:version', "
|
|
127
|
+
"e.g. ausdiab:0.7.5,cdah:0.8.1,edcad.",
|
|
128
|
+
),
|
|
129
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
130
|
+
version: str = typer.Option(
|
|
131
|
+
None,
|
|
132
|
+
"--version",
|
|
133
|
+
help="Default version for studies written without their own "
|
|
134
|
+
"':version', e.g. 0.9.8, or 'all' for every version.",
|
|
135
|
+
),
|
|
136
|
+
node: str = typer.Option(None, "--node", help="Delete only this node type."),
|
|
137
|
+
yes: bool = typer.Option(
|
|
138
|
+
False, "--yes", "-y", help="Skip the non-prod prompt (specific-version only)."
|
|
139
|
+
),
|
|
140
|
+
on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
|
|
141
|
+
) -> None:
|
|
142
|
+
"""Delete study metadata for one or more studies, sequentially, in one job.
|
|
143
|
+
|
|
144
|
+
Studies are processed one at a time. A study that exists but has no data at
|
|
145
|
+
the requested version is skipped, and the job continues to the next study.
|
|
146
|
+
|
|
147
|
+
Each study may carry its own version as ``name:version``; ``--version``
|
|
148
|
+
supplies the default for any study written bare. Examples:
|
|
149
|
+
|
|
150
|
+
g3dt delete metadata --studies "ausdiab:0.7.5,cdah:0.8.1" --env staging
|
|
151
|
+
g3dt delete metadata --studies "ausdiab:all,cdah" --version 0.9.8 --env staging
|
|
152
|
+
"""
|
|
153
|
+
fallback = (
|
|
154
|
+
_normalise_version(version, "for --version") if version is not None else None
|
|
155
|
+
)
|
|
156
|
+
specs = _parse_study_specs(studies, fallback, env)
|
|
157
|
+
versions = [v for _, v in specs]
|
|
158
|
+
|
|
159
|
+
# The typed production confirmation stays the study keys alone: short
|
|
160
|
+
# enough to retype accurately, while the per-study versions are spelled
|
|
161
|
+
# out in the action line printed directly above the prompt.
|
|
162
|
+
target = ",".join(key for key, _ in specs)
|
|
163
|
+
uniform = len(set(versions)) == 1
|
|
164
|
+
any_all = "all" in versions
|
|
165
|
+
|
|
166
|
+
if uniform and versions[0] == "all":
|
|
167
|
+
action = "deletion of ALL VERSIONS"
|
|
168
|
+
elif uniform:
|
|
169
|
+
action = f"deletion of v{versions[0]}"
|
|
170
|
+
else:
|
|
171
|
+
plan = ", ".join(f"{key}:{v}" for key, v in specs)
|
|
172
|
+
action = f"deletion of per-study versions [{plan}]"
|
|
173
|
+
|
|
174
|
+
# Deleting every version is the most destructive path: always prompt (pass
|
|
175
|
+
# assume_yes=False so --yes can't bypass it; prod still types the target).
|
|
176
|
+
# One 'all' anywhere in the list is enough to force the prompt, so an 'all'
|
|
177
|
+
# buried mid-list cannot ride along on a batch marked unattended.
|
|
178
|
+
safety.confirm_destructive(action, target, env, False if any_all else yes)
|
|
179
|
+
|
|
180
|
+
def build_args(env_name):
|
|
181
|
+
if uniform:
|
|
182
|
+
# Canonical (and historical) shape: one --version for every study.
|
|
183
|
+
# Emitting it keeps a newer CLI compatible with an older installed
|
|
184
|
+
# service script on the box, which can lag a pip upgrade.
|
|
185
|
+
a = ["--studies", target, "--env", env_name, "--version", versions[0]]
|
|
186
|
+
else:
|
|
187
|
+
a = [
|
|
188
|
+
"--studies",
|
|
189
|
+
",".join(f"{key}:{v}" for key, v in specs),
|
|
190
|
+
"--env",
|
|
191
|
+
env_name,
|
|
192
|
+
]
|
|
193
|
+
if node:
|
|
194
|
+
a += ["--node", node]
|
|
195
|
+
return a
|
|
196
|
+
|
|
197
|
+
def remote_cli(env_name):
|
|
198
|
+
# --yes: confirmation already happened locally; the remote job must
|
|
199
|
+
# not prompt (SSM has no TTY). The raw --studies string is forwarded
|
|
200
|
+
# verbatim — the remote re-entry re-parses and re-validates it.
|
|
201
|
+
a = ["delete", "metadata", "--studies", studies, "--env", env_name]
|
|
202
|
+
if version is not None:
|
|
203
|
+
a += ["--version", version]
|
|
204
|
+
a.append("--yes")
|
|
205
|
+
if node:
|
|
206
|
+
a += ["--node", node]
|
|
207
|
+
return a
|
|
208
|
+
|
|
209
|
+
dispatch.run_or_dispatch(
|
|
210
|
+
on, env, _DELETE_METADATA, build_args, "delete-metadata",
|
|
211
|
+
interpreter="bash", remote_cli=remote_cli,
|
|
212
|
+
)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""`g3dt indexd` — register S3 files with Gen3 indexd, and verify they download.
|
|
2
|
+
|
|
3
|
+
``register`` is a long data-plane op, so it supports ``--on ec2``.
|
|
4
|
+
``check-download`` is a read-only HTTP check that takes seconds and whose
|
|
5
|
+
whole value is the PASS/FAIL in your terminal, so it is local-only.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
import typer
|
|
12
|
+
|
|
13
|
+
from g3dt.cli._internal import dispatch, runner
|
|
14
|
+
from g3dt.cli._internal.dispatch import Target
|
|
15
|
+
from g3dt.cli._internal.resolve import env_of, study_of
|
|
16
|
+
|
|
17
|
+
app = typer.Typer(
|
|
18
|
+
no_args_is_help=True,
|
|
19
|
+
help="Register files with Gen3 indexd and verify download access.",
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
_REGISTER = "services/indexd/register_indexd.py"
|
|
23
|
+
_CHECK_DOWNLOAD = "services/indexd/verify_file_access.py"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@app.command()
|
|
27
|
+
def register(
|
|
28
|
+
s3_paths: List[str] = typer.Option(
|
|
29
|
+
..., "--s3-paths", help="One or more S3 prefixes to scan (repeatable)."
|
|
30
|
+
),
|
|
31
|
+
study: str = typer.Option(..., "--study", "-s", help="Study, e.g. edcad."),
|
|
32
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
33
|
+
dry_run: bool = typer.Option(
|
|
34
|
+
False, "--dry-run", help="Scan + write file_metadata only; skip indexd."
|
|
35
|
+
),
|
|
36
|
+
force: bool = typer.Option(
|
|
37
|
+
False, "--force",
|
|
38
|
+
help="Re-register files already in the registry with the same md5.",
|
|
39
|
+
),
|
|
40
|
+
on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
|
|
41
|
+
) -> None:
|
|
42
|
+
"""Scan S3 prefixes and register the files with Gen3 indexd.
|
|
43
|
+
|
|
44
|
+
Files already registered for this study at this endpoint with an unchanged
|
|
45
|
+
md5 are skipped (each re-registration would create a new indexd revision
|
|
46
|
+
and duplicate the registry). ``--force`` re-registers everything.
|
|
47
|
+
|
|
48
|
+
Examples:
|
|
49
|
+
g3dt indexd register --s3-paths s3://bucket/edcad/ --study edcad --env staging
|
|
50
|
+
g3dt indexd register --s3-paths s3://b/a/ --s3-paths s3://b/c/ --study edcad --env staging --on ec2
|
|
51
|
+
"""
|
|
52
|
+
s = study_of(study, env)
|
|
53
|
+
|
|
54
|
+
def build_args(env_name):
|
|
55
|
+
a = ["--s3-paths", *s3_paths, "--study", s.key, "--env", env_name]
|
|
56
|
+
if dry_run:
|
|
57
|
+
a.append("--dry-run")
|
|
58
|
+
if force:
|
|
59
|
+
a.append("--force")
|
|
60
|
+
return a
|
|
61
|
+
|
|
62
|
+
def remote_cli(env_name):
|
|
63
|
+
a: list = ["indexd", "register"]
|
|
64
|
+
for p in s3_paths:
|
|
65
|
+
a += ["--s3-paths", p]
|
|
66
|
+
a += ["--study", study, "--env", env_name]
|
|
67
|
+
if dry_run:
|
|
68
|
+
a.append("--dry-run")
|
|
69
|
+
if force:
|
|
70
|
+
a.append("--force")
|
|
71
|
+
return a
|
|
72
|
+
|
|
73
|
+
dispatch.run_or_dispatch(
|
|
74
|
+
on, env, _REGISTER, build_args, "indexd-register", remote_cli=remote_cli,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@app.command(name="check-download")
|
|
79
|
+
def check_download(
|
|
80
|
+
guids: Optional[List[str]] = typer.Argument(
|
|
81
|
+
None,
|
|
82
|
+
help="Object GUIDs, e.g. PREFIX/<uuid>. Omit to sample the most "
|
|
83
|
+
"recently registered objects from the indexd registry.",
|
|
84
|
+
),
|
|
85
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
86
|
+
limit: int = typer.Option(
|
|
87
|
+
25, "--limit", "-n",
|
|
88
|
+
help="How many objects to sample when no GUIDs are given.",
|
|
89
|
+
),
|
|
90
|
+
key_path: Optional[str] = typer.Option(
|
|
91
|
+
None, "--key-path",
|
|
92
|
+
help="Break-glass: local Gen3 API key JSON file, instead of the "
|
|
93
|
+
"env's secret.",
|
|
94
|
+
),
|
|
95
|
+
) -> None:
|
|
96
|
+
"""Prove registered objects are downloadable end to end.
|
|
97
|
+
|
|
98
|
+
Walks Indexd -> DRS -> Fence signed URL for each GUID and exits non-zero
|
|
99
|
+
if any object fails, so it can gate a deployment step. Read-only and
|
|
100
|
+
local-only (seconds, not a long job — there is nothing to dispatch to EC2).
|
|
101
|
+
|
|
102
|
+
The env selects the API key secret and the key's JWT selects the commons,
|
|
103
|
+
so a staging env checks staging. There is no URL to pass.
|
|
104
|
+
|
|
105
|
+
With no GUIDs, the newest --limit objects for this commons are sampled
|
|
106
|
+
from the indexd registry (latest revision per baseid). The registry may
|
|
107
|
+
live in a different AWS account than the commons — sampling needs an env
|
|
108
|
+
whose AWS profile can reach it; otherwise pass GUIDs explicitly.
|
|
109
|
+
|
|
110
|
+
Examples:
|
|
111
|
+
g3dt indexd check-download --env staging # sample 25 newest
|
|
112
|
+
g3dt indexd check-download --env staging --limit 50
|
|
113
|
+
g3dt indexd check-download --env prod PREFIX/aaa PREFIX/bbb
|
|
114
|
+
"""
|
|
115
|
+
# Validate the env before spawning a subprocess: an unknown env should
|
|
116
|
+
# fail here with the config error, not deep inside the script.
|
|
117
|
+
e = env_of(env)
|
|
118
|
+
|
|
119
|
+
args: List[str] = ["--env", e.name]
|
|
120
|
+
if key_path:
|
|
121
|
+
args += ["--key-path", key_path]
|
|
122
|
+
if guids:
|
|
123
|
+
args += list(guids)
|
|
124
|
+
else:
|
|
125
|
+
args += ["--limit", str(limit)]
|
|
126
|
+
runner.run(runner.python_script(_CHECK_DOWNLOAD, *args))
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""`g3dt metadata` — upload real study metadata to Gen3 (data-plane).
|
|
2
|
+
|
|
3
|
+
These are the multi-hour jobs, so they support ``--on ec2`` to run on the
|
|
4
|
+
env's EC2 job box via SSM Run Command (disconnect-safe) instead of the laptop.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
|
|
10
|
+
from g3dt.cli._internal import dispatch, safety
|
|
11
|
+
from g3dt.cli._internal.dispatch import Target
|
|
12
|
+
from g3dt.cli._internal.resolve import study_of
|
|
13
|
+
|
|
14
|
+
app = typer.Typer(no_args_is_help=True, help="Upload study metadata to Gen3.")
|
|
15
|
+
|
|
16
|
+
_UPLOAD = "services/upload/metadata/upload_metadata.py"
|
|
17
|
+
_UPLOAD_ALL = "services/upload/metadata/upload_all_studies.sh"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@app.command()
|
|
21
|
+
def upload(
|
|
22
|
+
study: str = typer.Option(..., "--study", "-s", help="Study, e.g. ausdiab."),
|
|
23
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
24
|
+
node: str = typer.Option(None, "--node", help="Submit only this node type."),
|
|
25
|
+
force_reupload: bool = typer.Option(
|
|
26
|
+
False, "--force-reupload",
|
|
27
|
+
help="Proceed even if this project+version was already uploaded to "
|
|
28
|
+
"this commons (uploads are additive: re-running duplicates records).",
|
|
29
|
+
),
|
|
30
|
+
on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
|
|
31
|
+
) -> None:
|
|
32
|
+
"""Upload a study's release metadata to Gen3 sheepdog.
|
|
33
|
+
|
|
34
|
+
The worker refuses (exit 2) when the audit table already records an
|
|
35
|
+
upload of the same project + version + endpoint — re-running would
|
|
36
|
+
duplicate every record. ``--force-reupload`` overrides.
|
|
37
|
+
|
|
38
|
+
Examples:
|
|
39
|
+
g3dt metadata upload --study ausdiab --env staging
|
|
40
|
+
g3dt metadata upload --study ausdiab --env staging --on ec2
|
|
41
|
+
"""
|
|
42
|
+
s = study_of(study, env)
|
|
43
|
+
|
|
44
|
+
def build_args(env_name):
|
|
45
|
+
a = ["--study", s.key, "--env", env_name]
|
|
46
|
+
if node:
|
|
47
|
+
a += ["--specific-node", node]
|
|
48
|
+
if force_reupload:
|
|
49
|
+
a.append("--force-reupload")
|
|
50
|
+
return a
|
|
51
|
+
|
|
52
|
+
def remote_cli(env_name):
|
|
53
|
+
a = ["metadata", "upload", "--study", study, "--env", env_name]
|
|
54
|
+
if node:
|
|
55
|
+
a += ["--node", node]
|
|
56
|
+
if force_reupload:
|
|
57
|
+
a.append("--force-reupload")
|
|
58
|
+
return a
|
|
59
|
+
|
|
60
|
+
dispatch.run_or_dispatch(
|
|
61
|
+
on, env, _UPLOAD, build_args, "metadata-upload", remote_cli=remote_cli,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@app.command(name="upload-all")
|
|
66
|
+
def upload_all(
|
|
67
|
+
studies: str = typer.Option(
|
|
68
|
+
..., "--studies", help="Comma-separated studies, e.g. ausdiab,caughtcad."
|
|
69
|
+
),
|
|
70
|
+
env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
|
|
71
|
+
allow_prod: bool = typer.Option(
|
|
72
|
+
False, "--allow-prod",
|
|
73
|
+
help="Allow bulk upload against production (typed confirmation required).",
|
|
74
|
+
),
|
|
75
|
+
prod_confirmed: bool = typer.Option(
|
|
76
|
+
False, "--prod-confirmed", hidden=True,
|
|
77
|
+
help="Internal: set by the remote re-entry after the typed "
|
|
78
|
+
"confirmation already happened locally. Never pass by hand.",
|
|
79
|
+
),
|
|
80
|
+
force_reupload: bool = typer.Option(
|
|
81
|
+
False, "--force-reupload",
|
|
82
|
+
help="Proceed even for project+versions the audit table says were "
|
|
83
|
+
"already uploaded to this commons.",
|
|
84
|
+
),
|
|
85
|
+
on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Upload several studies sequentially (wraps upload_all_studies.sh).
|
|
88
|
+
|
|
89
|
+
Production needs ``--allow-prod`` AND a typed confirmation of the env
|
|
90
|
+
name. The confirmation happens locally, before any EC2 dispatch (SSM has
|
|
91
|
+
no TTY, so a remote prompt would abort); the remote re-entry carries the
|
|
92
|
+
hidden ``--prod-confirmed`` marker instead of re-prompting, and
|
|
93
|
+
``--allow-prod`` is forwarded so the wrapped script's own guard passes.
|
|
94
|
+
"""
|
|
95
|
+
names = [s.strip() for s in studies.split(",") if s.strip()]
|
|
96
|
+
keys = [study_of(name, env).key for name in names]
|
|
97
|
+
|
|
98
|
+
# Prod is detected on the resolved study keys as well as on --env:
|
|
99
|
+
# `--env staging --studies ausdiab_prod` is a production write.
|
|
100
|
+
if safety.is_prod(env) or any(safety.is_prod(k) for k in keys):
|
|
101
|
+
if not allow_prod:
|
|
102
|
+
typer.secho(
|
|
103
|
+
"Refusing bulk upload against a production environment. "
|
|
104
|
+
"Re-run with --allow-prod to confirm interactively.",
|
|
105
|
+
fg=typer.colors.RED,
|
|
106
|
+
err=True,
|
|
107
|
+
)
|
|
108
|
+
raise typer.Exit(2)
|
|
109
|
+
if not prod_confirmed:
|
|
110
|
+
safety.confirm_prod_strict("bulk metadata upload", env)
|
|
111
|
+
|
|
112
|
+
def build_args(env_name):
|
|
113
|
+
a = ["--studies", ",".join(keys), "--env", env_name]
|
|
114
|
+
if allow_prod:
|
|
115
|
+
a.append("--allow-prod")
|
|
116
|
+
if force_reupload:
|
|
117
|
+
a.append("--force-reupload")
|
|
118
|
+
return a
|
|
119
|
+
|
|
120
|
+
def remote_cli(env_name):
|
|
121
|
+
a = ["metadata", "upload-all", "--studies", studies, "--env", env_name]
|
|
122
|
+
if allow_prod:
|
|
123
|
+
# The typed confirmation already happened locally above; the box
|
|
124
|
+
# has no TTY, so the re-entry must not prompt again.
|
|
125
|
+
a += ["--allow-prod", "--prod-confirmed"]
|
|
126
|
+
if force_reupload:
|
|
127
|
+
a.append("--force-reupload")
|
|
128
|
+
return a
|
|
129
|
+
|
|
130
|
+
dispatch.run_or_dispatch(
|
|
131
|
+
on, env, _UPLOAD_ALL, build_args, "metadata-upload-all",
|
|
132
|
+
interpreter="bash", remote_cli=remote_cli,
|
|
133
|
+
)
|