gen3-dataops-toolkit 2.2.0__tar.gz → 2.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/PKG-INFO +42 -1
  2. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/README.md +40 -0
  3. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/pyproject.toml +2 -1
  4. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/safety.py +0 -12
  5. gen3_dataops_toolkit-2.3.0/src/g3dt/cli/delete_cmds.py +212 -0
  6. gen3_dataops_toolkit-2.3.0/src/g3dt/cli/indexd_cmds.py +126 -0
  7. gen3_dataops_toolkit-2.3.0/src/g3dt/cli/metadata.py +133 -0
  8. gen3_dataops_toolkit-2.3.0/src/g3dt/indexd/file_access.py +274 -0
  9. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/indexd/indexd_registrar.py +95 -4
  10. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/ingest/ingest.py +117 -33
  11. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/delete/delete_metadata.sh +52 -18
  12. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/indexd/register_indexd.py +38 -0
  13. gen3_dataops_toolkit-2.3.0/src/g3dt/services/indexd/verify_file_access.py +126 -0
  14. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/upload/metadata/upload_all_studies.sh +29 -8
  15. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/upload/metadata/upload_metadata.py +34 -0
  16. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/metadata_submitter.py +81 -8
  17. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/utils/athena_utils.py +79 -0
  18. gen3_dataops_toolkit-2.2.0/src/g3dt/cli/delete_cmds.py +0 -101
  19. gen3_dataops_toolkit-2.2.0/src/g3dt/cli/indexd_cmds.py +0 -57
  20. gen3_dataops_toolkit-2.2.0/src/g3dt/cli/metadata.py +0 -76
  21. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/__init__.py +0 -0
  22. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/__init__.py +0 -0
  23. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/__init__.py +0 -0
  24. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/dispatch.py +0 -0
  25. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/registry.py +0 -0
  26. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/resolve.py +0 -0
  27. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/_internal/runner.py +0 -0
  28. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/config_cmds.py +0 -0
  29. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/dict_cmds.py +0 -0
  30. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/ec2_cmds.py +0 -0
  31. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/jobs.py +0 -0
  32. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/k8s.py +0 -0
  33. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/main.py +0 -0
  34. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/pipeline_cmds.py +0 -0
  35. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/release_cmds.py +0 -0
  36. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/cli/synth.py +0 -0
  37. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/config.py +0 -0
  38. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/indexd/__init__.py +0 -0
  39. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/resolver.py +0 -0
  40. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
  41. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
  42. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/dictionary/deploy_dd.sh +0 -0
  43. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
  44. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
  45. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +0 -0
  46. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +0 -0
  47. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +0 -0
  48. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
  49. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +0 -0
  50. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
  51. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +0 -0
  52. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
  53. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +0 -0
  54. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/__init__.py +0 -0
  55. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/metadata_deleter.py +0 -0
  56. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
  57. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/utils/dbt_utils.py +0 -0
  58. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/utils/release_writer.py +0 -0
  59. {gen3_dataops_toolkit-2.2.0 → gen3_dataops_toolkit-2.3.0}/src/g3dt/validate/validate.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gen3-dataops-toolkit
3
- Version: 2.2.0
3
+ Version: 2.3.0
4
4
  Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
5
5
  License: Apache-2.0
6
6
  Author: JoshuaHarris391
@@ -19,6 +19,7 @@ Requires-Dist: gen3 (>=4.27.4,<5.0.0)
19
19
  Requires-Dist: gen3-metadata (>=1.4.0,<2.0.0)
20
20
  Requires-Dist: gen3_validator (>=2.0.0,<3.0.0)
21
21
  Requires-Dist: numpy (<2.0.0)
22
+ Requires-Dist: openpyxl (>=3.1.0)
22
23
  Requires-Dist: pyarrow (>=14.0.0,<19.0.0)
23
24
  Requires-Dist: pyjwt (>=2.10.1,<3.0.0)
24
25
  Requires-Dist: python-dotenv
@@ -175,6 +176,46 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
175
176
  refuses a batch that doesn't match the version being uploaded (override with
176
177
  `--allow-version-mismatch`).
177
178
 
179
+ ## Verifying download access (check-download)
180
+
181
+ Registration alone does not prove a file can be downloaded. Two failure modes
182
+ are invisible until a user clicks the file in the portal: an Indexd record
183
+ with no storage URL (nothing to download, ever), and a record Fence refuses
184
+ to sign a URL for. `g3dt indexd check-download` walks the exact chain the
185
+ portal hits — Indexd record → storage URL → DRS object → access methods →
186
+ Fence signed URL — and reports PASS/FAIL per object, exiting non-zero if any
187
+ object fails so it can gate a deployment step.
188
+
189
+ Run it before a release, and after registering new files. The env selects the
190
+ API key secret and the key's JWT selects the commons, so there is no URL to
191
+ pass (and none to get wrong).
192
+
193
+ ```bash
194
+ g3dt indexd check-download --env staging # sample the 25 newest
195
+ g3dt indexd check-download --env staging --limit 50
196
+ g3dt indexd check-download --env prod PREFIX/<uuid-1> PREFIX/<uuid-2>
197
+ ```
198
+
199
+ With no GUIDs, the newest objects for the env's commons are sampled from the
200
+ indexd registry (latest revision per baseid). The registry may live in a
201
+ different AWS account than the commons being checked; if the env's AWS
202
+ profile cannot reach it, pass GUIDs explicitly.
203
+
204
+ Reading a failure:
205
+
206
+ | Symptom | Meaning |
207
+ |---|---|
208
+ | `Indexd status: 404` | the object is not registered — a registration problem, not a download one |
209
+ | `urls: []` / no access methods | registered but with no storage location; it can never download |
210
+ | `Access endpoint … 401` | authorization: the API key's user lacks `read-storage` on the record's `authz` resource — an authz gap, not a broken key |
211
+ | `Access endpoint … 500` | Fence has the permission but failed to sign — a service-side fault |
212
+
213
+ On a 401, compare what the record requires
214
+ (`https://commons.example.org/index/<did>`, the `authz` field) with what the
215
+ key's user actually holds (`https://commons.example.org/user/user`):
216
+ downloads require `read-storage` on the record's authz resource, which a user
217
+ holding only `create` does not have.
218
+
178
219
  ## Development
179
220
 
180
221
  ```bash
@@ -142,6 +142,46 @@ Synthetic data is only schema-valid against the dictionary that generated it, so
142
142
  refuses a batch that doesn't match the version being uploaded (override with
143
143
  `--allow-version-mismatch`).
144
144
 
145
+ ## Verifying download access (check-download)
146
+
147
+ Registration alone does not prove a file can be downloaded. Two failure modes
148
+ are invisible until a user clicks the file in the portal: an Indexd record
149
+ with no storage URL (nothing to download, ever), and a record Fence refuses
150
+ to sign a URL for. `g3dt indexd check-download` walks the exact chain the
151
+ portal hits — Indexd record → storage URL → DRS object → access methods →
152
+ Fence signed URL — and reports PASS/FAIL per object, exiting non-zero if any
153
+ object fails so it can gate a deployment step.
154
+
155
+ Run it before a release, and after registering new files. The env selects the
156
+ API key secret and the key's JWT selects the commons, so there is no URL to
157
+ pass (and none to get wrong).
158
+
159
+ ```bash
160
+ g3dt indexd check-download --env staging # sample the 25 newest
161
+ g3dt indexd check-download --env staging --limit 50
162
+ g3dt indexd check-download --env prod PREFIX/<uuid-1> PREFIX/<uuid-2>
163
+ ```
164
+
165
+ With no GUIDs, the newest objects for the env's commons are sampled from the
166
+ indexd registry (latest revision per baseid). The registry may live in a
167
+ different AWS account than the commons being checked; if the env's AWS
168
+ profile cannot reach it, pass GUIDs explicitly.
169
+
170
+ Reading a failure:
171
+
172
+ | Symptom | Meaning |
173
+ |---|---|
174
+ | `Indexd status: 404` | the object is not registered — a registration problem, not a download one |
175
+ | `urls: []` / no access methods | registered but with no storage location; it can never download |
176
+ | `Access endpoint … 401` | authorization: the API key's user lacks `read-storage` on the record's `authz` resource — an authz gap, not a broken key |
177
+ | `Access endpoint … 500` | Fence has the permission but failed to sign — a service-side fault |
178
+
179
+ On a 401, compare what the record requires
180
+ (`https://commons.example.org/index/<did>`, the `authz` field) with what the
181
+ key's user actually holds (`https://commons.example.org/user/user`):
182
+ downloads require `read-storage` on the record's authz resource, which a user
183
+ holding only `create` does not have.
184
+
145
185
  ## Development
146
186
 
147
187
  ```bash
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "gen3-dataops-toolkit"
3
- version = "2.2.0"
3
+ version = "2.3.0"
4
4
  description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
@@ -28,6 +28,7 @@ awswrangler = ">=3.14.0,<4.0.0"
28
28
  pyarrow = ">=14.0.0,<19.0.0"
29
29
  gen3-metadata = ">=1.4.0,<2.0.0"
30
30
  typer = ">=0.12"
31
+ openpyxl = ">=3.1.0"
31
32
 
32
33
  [tool.poetry.scripts]
33
34
  g3dt = "g3dt.cli.main:main"
@@ -33,18 +33,6 @@ def require_test_env(env: str) -> None:
33
33
  raise typer.Exit(2)
34
34
 
35
35
 
36
- def abort_if_prod(env: str) -> None:
37
- """Hard abort for bulk operations that must never touch production."""
38
- if is_prod(env):
39
- typer.secho(
40
- f"Refusing bulk operation against a production environment "
41
- f"('{env}').",
42
- fg=typer.colors.RED,
43
- err=True,
44
- )
45
- raise typer.Exit(2)
46
-
47
-
48
36
  def confirm_destructive(action: str, target: str, env: str, assume_yes: bool) -> None:
49
37
  """Gate a destructive operation with an appropriate confirmation.
50
38
 
@@ -0,0 +1,212 @@
1
+ """`g3dt delete` — destructive metadata removal (data-plane).
2
+
3
+ A single ``delete metadata`` command handles one or many studies, sequentially,
4
+ in a single job. ``--version`` is required: pass a specific version (e.g.
5
+ ``0.9.8``) to remove just that version (resolved via an Athena GUID lookup), or
6
+ ``all`` to remove every version.
7
+
8
+ Every command confirms before acting. Production always requires typing the
9
+ target id, even with ``--yes``. Deleting ALL versions always prompts, even with
10
+ ``--yes``. Confirmation happens locally before any EC2 dispatch (SSM has no
11
+ TTY), after which the remote job runs non-interactively.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import re
16
+
17
+ import typer
18
+
19
+ from g3dt.cli._internal import dispatch, safety
20
+ from g3dt.cli._internal.dispatch import Target
21
+ from g3dt.cli._internal.resolve import study_of
22
+
23
+ app = typer.Typer(no_args_is_help=True, help="Delete metadata from Gen3 (destructive).")
24
+
25
+ _DELETE_METADATA = "services/delete/delete_metadata.sh"
26
+
27
+ #: A version token in the form the Athena ``version`` column stores it. The
28
+ #: uploader writes ``group(1)`` of this same pattern (metadata_submitter's
29
+ #: ``_find_version_from_path``), i.e. WITHOUT any leading ``v``. The delete
30
+ #: query interpolates the string straight into SQL, so a ``v``-prefixed version
31
+ #: matches zero rows and is reported as "skipped" rather than as an error — a
32
+ #: silent no-op that reads as a clean run. Normalising here closes that.
33
+ _VERSION_RE = re.compile(r"^v?(\d+\.\d+\.\d+)$", re.IGNORECASE)
34
+
35
+
36
+ def _normalise_version(raw: str, where: str) -> str:
37
+ """Canonicalise one version token to the form stored in Athena.
38
+
39
+ ``all`` in any case becomes ``all``; ``v1.5.4`` and ``1.5.4`` both become
40
+ ``1.5.4``. Anything else is a usage error: the column only ever holds
41
+ three-part semver, so a truncated version like ``0.9`` would match nothing
42
+ and be counted as a skip.
43
+ """
44
+ token = raw.strip()
45
+ if token.lower() == "all":
46
+ return "all"
47
+ match = _VERSION_RE.match(token)
48
+ if not match:
49
+ typer.secho(
50
+ f"Invalid version '{raw}' {where}: expected x.y.z (e.g. 0.9.8) "
51
+ "or 'all'.",
52
+ fg=typer.colors.RED,
53
+ err=True,
54
+ )
55
+ raise typer.Exit(2)
56
+ return match.group(1)
57
+
58
+
59
+ def _parse_study_specs(studies: str, fallback, env: str):
60
+ """Turn ``--studies`` into ``[(resolved_study_key, version), ...]``.
61
+
62
+ Each comma-separated entry is ``name`` or ``name:version``. A bare name
63
+ takes *fallback* (the ``--version`` default); *fallback* is ``None`` when
64
+ ``--version`` was not given, which makes a bare name a usage error.
65
+
66
+ Every entry is validated before anything is dispatched, so a typo in the
67
+ last study cannot leave the earlier ones already deleted.
68
+ """
69
+ specs = []
70
+ for entry in studies.split(","):
71
+ entry = entry.strip()
72
+ if not entry:
73
+ continue
74
+
75
+ # partition() rather than split(), so a trailing colon ("ausdiab:") is
76
+ # distinguishable from a bare name and can be rejected instead of
77
+ # silently taking the fallback.
78
+ name, sep, raw_version = entry.partition(":")
79
+ name = name.strip()
80
+
81
+ if not name:
82
+ typer.secho(
83
+ f"Invalid --studies entry '{entry}': missing study name.",
84
+ fg=typer.colors.RED,
85
+ err=True,
86
+ )
87
+ raise typer.Exit(2)
88
+
89
+ if sep and not raw_version.strip():
90
+ typer.secho(
91
+ f"Invalid --studies entry '{entry}': ':' with no version. "
92
+ f"Use '{name}:0.9.8', '{name}:all', or a bare '{name}' to take "
93
+ "the --version default.",
94
+ fg=typer.colors.RED,
95
+ err=True,
96
+ )
97
+ raise typer.Exit(2)
98
+
99
+ if sep:
100
+ version = _normalise_version(raw_version, f"for study '{name}'")
101
+ elif fallback is not None:
102
+ version = fallback
103
+ else:
104
+ typer.secho(
105
+ f"No version for study '{name}': add ':<version>' to it "
106
+ f"(e.g. '{name}:0.9.8'), or pass --version as the default for "
107
+ "every study. Use 'all' to delete every version.",
108
+ fg=typer.colors.RED,
109
+ err=True,
110
+ )
111
+ raise typer.Exit(2)
112
+
113
+ specs.append((study_of(name, env).key, version))
114
+
115
+ if not specs:
116
+ typer.secho("--studies is empty.", fg=typer.colors.RED, err=True)
117
+ raise typer.Exit(2)
118
+ return specs
119
+
120
+
121
+ @app.command()
122
+ def metadata(
123
+ studies: str = typer.Option(
124
+ ...,
125
+ "--studies",
126
+ help="Comma-separated studies, each optionally 'name:version', "
127
+ "e.g. ausdiab:0.7.5,cdah:0.8.1,edcad.",
128
+ ),
129
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
130
+ version: str = typer.Option(
131
+ None,
132
+ "--version",
133
+ help="Default version for studies written without their own "
134
+ "':version', e.g. 0.9.8, or 'all' for every version.",
135
+ ),
136
+ node: str = typer.Option(None, "--node", help="Delete only this node type."),
137
+ yes: bool = typer.Option(
138
+ False, "--yes", "-y", help="Skip the non-prod prompt (specific-version only)."
139
+ ),
140
+ on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
141
+ ) -> None:
142
+ """Delete study metadata for one or more studies, sequentially, in one job.
143
+
144
+ Studies are processed one at a time. A study that exists but has no data at
145
+ the requested version is skipped, and the job continues to the next study.
146
+
147
+ Each study may carry its own version as ``name:version``; ``--version``
148
+ supplies the default for any study written bare. Examples:
149
+
150
+ g3dt delete metadata --studies "ausdiab:0.7.5,cdah:0.8.1" --env staging
151
+ g3dt delete metadata --studies "ausdiab:all,cdah" --version 0.9.8 --env staging
152
+ """
153
+ fallback = (
154
+ _normalise_version(version, "for --version") if version is not None else None
155
+ )
156
+ specs = _parse_study_specs(studies, fallback, env)
157
+ versions = [v for _, v in specs]
158
+
159
+ # The typed production confirmation stays the study keys alone: short
160
+ # enough to retype accurately, while the per-study versions are spelled
161
+ # out in the action line printed directly above the prompt.
162
+ target = ",".join(key for key, _ in specs)
163
+ uniform = len(set(versions)) == 1
164
+ any_all = "all" in versions
165
+
166
+ if uniform and versions[0] == "all":
167
+ action = "deletion of ALL VERSIONS"
168
+ elif uniform:
169
+ action = f"deletion of v{versions[0]}"
170
+ else:
171
+ plan = ", ".join(f"{key}:{v}" for key, v in specs)
172
+ action = f"deletion of per-study versions [{plan}]"
173
+
174
+ # Deleting every version is the most destructive path: always prompt (pass
175
+ # assume_yes=False so --yes can't bypass it; prod still types the target).
176
+ # One 'all' anywhere in the list is enough to force the prompt, so an 'all'
177
+ # buried mid-list cannot ride along on a batch marked unattended.
178
+ safety.confirm_destructive(action, target, env, False if any_all else yes)
179
+
180
+ def build_args(env_name):
181
+ if uniform:
182
+ # Canonical (and historical) shape: one --version for every study.
183
+ # Emitting it keeps a newer CLI compatible with an older installed
184
+ # service script on the box, which can lag a pip upgrade.
185
+ a = ["--studies", target, "--env", env_name, "--version", versions[0]]
186
+ else:
187
+ a = [
188
+ "--studies",
189
+ ",".join(f"{key}:{v}" for key, v in specs),
190
+ "--env",
191
+ env_name,
192
+ ]
193
+ if node:
194
+ a += ["--node", node]
195
+ return a
196
+
197
+ def remote_cli(env_name):
198
+ # --yes: confirmation already happened locally; the remote job must
199
+ # not prompt (SSM has no TTY). The raw --studies string is forwarded
200
+ # verbatim — the remote re-entry re-parses and re-validates it.
201
+ a = ["delete", "metadata", "--studies", studies, "--env", env_name]
202
+ if version is not None:
203
+ a += ["--version", version]
204
+ a.append("--yes")
205
+ if node:
206
+ a += ["--node", node]
207
+ return a
208
+
209
+ dispatch.run_or_dispatch(
210
+ on, env, _DELETE_METADATA, build_args, "delete-metadata",
211
+ interpreter="bash", remote_cli=remote_cli,
212
+ )
@@ -0,0 +1,126 @@
1
+ """`g3dt indexd` — register S3 files with Gen3 indexd, and verify they download.
2
+
3
+ ``register`` is a long data-plane op, so it supports ``--on ec2``.
4
+ ``check-download`` is a read-only HTTP check that takes seconds and whose
5
+ whole value is the PASS/FAIL in your terminal, so it is local-only.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from typing import List, Optional
10
+
11
+ import typer
12
+
13
+ from g3dt.cli._internal import dispatch, runner
14
+ from g3dt.cli._internal.dispatch import Target
15
+ from g3dt.cli._internal.resolve import env_of, study_of
16
+
17
+ app = typer.Typer(
18
+ no_args_is_help=True,
19
+ help="Register files with Gen3 indexd and verify download access.",
20
+ )
21
+
22
+ _REGISTER = "services/indexd/register_indexd.py"
23
+ _CHECK_DOWNLOAD = "services/indexd/verify_file_access.py"
24
+
25
+
26
+ @app.command()
27
+ def register(
28
+ s3_paths: List[str] = typer.Option(
29
+ ..., "--s3-paths", help="One or more S3 prefixes to scan (repeatable)."
30
+ ),
31
+ study: str = typer.Option(..., "--study", "-s", help="Study, e.g. edcad."),
32
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
33
+ dry_run: bool = typer.Option(
34
+ False, "--dry-run", help="Scan + write file_metadata only; skip indexd."
35
+ ),
36
+ force: bool = typer.Option(
37
+ False, "--force",
38
+ help="Re-register files already in the registry with the same md5.",
39
+ ),
40
+ on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
41
+ ) -> None:
42
+ """Scan S3 prefixes and register the files with Gen3 indexd.
43
+
44
+ Files already registered for this study at this endpoint with an unchanged
45
+ md5 are skipped (each re-registration would create a new indexd revision
46
+ and duplicate the registry). ``--force`` re-registers everything.
47
+
48
+ Examples:
49
+ g3dt indexd register --s3-paths s3://bucket/edcad/ --study edcad --env staging
50
+ g3dt indexd register --s3-paths s3://b/a/ --s3-paths s3://b/c/ --study edcad --env staging --on ec2
51
+ """
52
+ s = study_of(study, env)
53
+
54
+ def build_args(env_name):
55
+ a = ["--s3-paths", *s3_paths, "--study", s.key, "--env", env_name]
56
+ if dry_run:
57
+ a.append("--dry-run")
58
+ if force:
59
+ a.append("--force")
60
+ return a
61
+
62
+ def remote_cli(env_name):
63
+ a: list = ["indexd", "register"]
64
+ for p in s3_paths:
65
+ a += ["--s3-paths", p]
66
+ a += ["--study", study, "--env", env_name]
67
+ if dry_run:
68
+ a.append("--dry-run")
69
+ if force:
70
+ a.append("--force")
71
+ return a
72
+
73
+ dispatch.run_or_dispatch(
74
+ on, env, _REGISTER, build_args, "indexd-register", remote_cli=remote_cli,
75
+ )
76
+
77
+
78
+ @app.command(name="check-download")
79
+ def check_download(
80
+ guids: Optional[List[str]] = typer.Argument(
81
+ None,
82
+ help="Object GUIDs, e.g. PREFIX/<uuid>. Omit to sample the most "
83
+ "recently registered objects from the indexd registry.",
84
+ ),
85
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
86
+ limit: int = typer.Option(
87
+ 25, "--limit", "-n",
88
+ help="How many objects to sample when no GUIDs are given.",
89
+ ),
90
+ key_path: Optional[str] = typer.Option(
91
+ None, "--key-path",
92
+ help="Break-glass: local Gen3 API key JSON file, instead of the "
93
+ "env's secret.",
94
+ ),
95
+ ) -> None:
96
+ """Prove registered objects are downloadable end to end.
97
+
98
+ Walks Indexd -> DRS -> Fence signed URL for each GUID and exits non-zero
99
+ if any object fails, so it can gate a deployment step. Read-only and
100
+ local-only (seconds, not a long job — there is nothing to dispatch to EC2).
101
+
102
+ The env selects the API key secret and the key's JWT selects the commons,
103
+ so a staging env checks staging. There is no URL to pass.
104
+
105
+ With no GUIDs, the newest --limit objects for this commons are sampled
106
+ from the indexd registry (latest revision per baseid). The registry may
107
+ live in a different AWS account than the commons — sampling needs an env
108
+ whose AWS profile can reach it; otherwise pass GUIDs explicitly.
109
+
110
+ Examples:
111
+ g3dt indexd check-download --env staging # sample 25 newest
112
+ g3dt indexd check-download --env staging --limit 50
113
+ g3dt indexd check-download --env prod PREFIX/aaa PREFIX/bbb
114
+ """
115
+ # Validate the env before spawning a subprocess: an unknown env should
116
+ # fail here with the config error, not deep inside the script.
117
+ e = env_of(env)
118
+
119
+ args: List[str] = ["--env", e.name]
120
+ if key_path:
121
+ args += ["--key-path", key_path]
122
+ if guids:
123
+ args += list(guids)
124
+ else:
125
+ args += ["--limit", str(limit)]
126
+ runner.run(runner.python_script(_CHECK_DOWNLOAD, *args))
@@ -0,0 +1,133 @@
1
+ """`g3dt metadata` — upload real study metadata to Gen3 (data-plane).
2
+
3
+ These are the multi-hour jobs, so they support ``--on ec2`` to run on the
4
+ env's EC2 job box via SSM Run Command (disconnect-safe) instead of the laptop.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import typer
9
+
10
+ from g3dt.cli._internal import dispatch, safety
11
+ from g3dt.cli._internal.dispatch import Target
12
+ from g3dt.cli._internal.resolve import study_of
13
+
14
+ app = typer.Typer(no_args_is_help=True, help="Upload study metadata to Gen3.")
15
+
16
+ _UPLOAD = "services/upload/metadata/upload_metadata.py"
17
+ _UPLOAD_ALL = "services/upload/metadata/upload_all_studies.sh"
18
+
19
+
20
+ @app.command()
21
+ def upload(
22
+ study: str = typer.Option(..., "--study", "-s", help="Study, e.g. ausdiab."),
23
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
24
+ node: str = typer.Option(None, "--node", help="Submit only this node type."),
25
+ force_reupload: bool = typer.Option(
26
+ False, "--force-reupload",
27
+ help="Proceed even if this project+version was already uploaded to "
28
+ "this commons (uploads are additive: re-running duplicates records).",
29
+ ),
30
+ on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
31
+ ) -> None:
32
+ """Upload a study's release metadata to Gen3 sheepdog.
33
+
34
+ The worker refuses (exit 2) when the audit table already records an
35
+ upload of the same project + version + endpoint — re-running would
36
+ duplicate every record. ``--force-reupload`` overrides.
37
+
38
+ Examples:
39
+ g3dt metadata upload --study ausdiab --env staging
40
+ g3dt metadata upload --study ausdiab --env staging --on ec2
41
+ """
42
+ s = study_of(study, env)
43
+
44
+ def build_args(env_name):
45
+ a = ["--study", s.key, "--env", env_name]
46
+ if node:
47
+ a += ["--specific-node", node]
48
+ if force_reupload:
49
+ a.append("--force-reupload")
50
+ return a
51
+
52
+ def remote_cli(env_name):
53
+ a = ["metadata", "upload", "--study", study, "--env", env_name]
54
+ if node:
55
+ a += ["--node", node]
56
+ if force_reupload:
57
+ a.append("--force-reupload")
58
+ return a
59
+
60
+ dispatch.run_or_dispatch(
61
+ on, env, _UPLOAD, build_args, "metadata-upload", remote_cli=remote_cli,
62
+ )
63
+
64
+
65
+ @app.command(name="upload-all")
66
+ def upload_all(
67
+ studies: str = typer.Option(
68
+ ..., "--studies", help="Comma-separated studies, e.g. ausdiab,caughtcad."
69
+ ),
70
+ env: str = typer.Option(..., "--env", "-e", help="Environment, e.g. test."),
71
+ allow_prod: bool = typer.Option(
72
+ False, "--allow-prod",
73
+ help="Allow bulk upload against production (typed confirmation required).",
74
+ ),
75
+ prod_confirmed: bool = typer.Option(
76
+ False, "--prod-confirmed", hidden=True,
77
+ help="Internal: set by the remote re-entry after the typed "
78
+ "confirmation already happened locally. Never pass by hand.",
79
+ ),
80
+ force_reupload: bool = typer.Option(
81
+ False, "--force-reupload",
82
+ help="Proceed even for project+versions the audit table says were "
83
+ "already uploaded to this commons.",
84
+ ),
85
+ on: Target = typer.Option(Target.local, "--on", help="Run local or on ec2."),
86
+ ) -> None:
87
+ """Upload several studies sequentially (wraps upload_all_studies.sh).
88
+
89
+ Production needs ``--allow-prod`` AND a typed confirmation of the env
90
+ name. The confirmation happens locally, before any EC2 dispatch (SSM has
91
+ no TTY, so a remote prompt would abort); the remote re-entry carries the
92
+ hidden ``--prod-confirmed`` marker instead of re-prompting, and
93
+ ``--allow-prod`` is forwarded so the wrapped script's own guard passes.
94
+ """
95
+ names = [s.strip() for s in studies.split(",") if s.strip()]
96
+ keys = [study_of(name, env).key for name in names]
97
+
98
+ # Prod is detected on the resolved study keys as well as on --env:
99
+ # `--env staging --studies ausdiab_prod` is a production write.
100
+ if safety.is_prod(env) or any(safety.is_prod(k) for k in keys):
101
+ if not allow_prod:
102
+ typer.secho(
103
+ "Refusing bulk upload against a production environment. "
104
+ "Re-run with --allow-prod to confirm interactively.",
105
+ fg=typer.colors.RED,
106
+ err=True,
107
+ )
108
+ raise typer.Exit(2)
109
+ if not prod_confirmed:
110
+ safety.confirm_prod_strict("bulk metadata upload", env)
111
+
112
+ def build_args(env_name):
113
+ a = ["--studies", ",".join(keys), "--env", env_name]
114
+ if allow_prod:
115
+ a.append("--allow-prod")
116
+ if force_reupload:
117
+ a.append("--force-reupload")
118
+ return a
119
+
120
+ def remote_cli(env_name):
121
+ a = ["metadata", "upload-all", "--studies", studies, "--env", env_name]
122
+ if allow_prod:
123
+ # The typed confirmation already happened locally above; the box
124
+ # has no TTY, so the re-entry must not prompt again.
125
+ a += ["--allow-prod", "--prod-confirmed"]
126
+ if force_reupload:
127
+ a.append("--force-reupload")
128
+ return a
129
+
130
+ dispatch.run_or_dispatch(
131
+ on, env, _UPLOAD_ALL, build_args, "metadata-upload-all",
132
+ interpreter="bash", remote_cli=remote_cli,
133
+ )