gen3-dataops-toolkit 2.1.0__tar.gz → 2.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/PKG-INFO +1 -1
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/pyproject.toml +1 -1
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/config.py +3 -1
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/resolver.py +30 -6
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/README.md +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/__init__.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/__init__.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/__init__.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/dispatch.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/registry.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/resolve.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/runner.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/safety.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/config_cmds.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/delete_cmds.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/dict_cmds.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/ec2_cmds.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/indexd_cmds.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/jobs.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/k8s.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/main.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/metadata.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/pipeline_cmds.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/release_cmds.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/synth.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/indexd/__init__.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/indexd/indexd_registrar.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/ingest/ingest.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/delete/delete_all_metadata_for_project.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/delete/delete_metadata.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/delete/delete_metadata_by_guid.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/dictionary/deploy_dd.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/dictionary/pull_dict.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/dictionary/upload_dictionary.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/indexd/register_indexd.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/k8s_ops/argocd_restart_etl.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/k8s_ops/argocd_restart_ms.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/k8s_ops/argocd_restart_schema.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/k8s_ops/login_to_pod.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/k8s_ops/restart_etl_and_ms.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/synthetic_data/delete_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/synthetic_data/full_deploy_dd_and_synth.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/upload/metadata/upload_all_studies.sh +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/upload/metadata/upload_metadata.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/upload/__init__.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/upload/metadata_deleter.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/upload/metadata_submitter.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/upload/upload_synthdata_s3.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/utils/athena_utils.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/utils/dbt_utils.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/utils/release_writer.py +0 -0
- {gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/validate/validate.py +0 -0
|
@@ -336,7 +336,9 @@ def _studies_from_s3_cached(project: str, base_env: str, profile: Optional[str])
|
|
|
336
336
|
from g3dt import resolver
|
|
337
337
|
|
|
338
338
|
rc = resolver.resolve(project, base_env, profile=profile)
|
|
339
|
-
|
|
339
|
+
# Explicit region: botocore ignores AWS_REGION (only AWS_DEFAULT_REGION),
|
|
340
|
+
# so an ambient session on the EC2 box would have no region at all.
|
|
341
|
+
session = boto3.Session(profile_name=profile, region_name=rc.region)
|
|
340
342
|
try:
|
|
341
343
|
body = session.client("s3").get_object(
|
|
342
344
|
Bucket=rc.metadata_bucket, Key=STUDIES_S3_KEY
|
|
@@ -127,31 +127,55 @@ class ResolvedConfig:
|
|
|
127
127
|
return self.params.get(key, default)
|
|
128
128
|
|
|
129
129
|
|
|
130
|
+
def _session(profile: Optional[str], region: Optional[str]) -> boto3.Session:
|
|
131
|
+
"""A boto3 session with an EXPLICIT region.
|
|
132
|
+
|
|
133
|
+
The region always comes from the marker (file value, or its AWS_REGION
|
|
134
|
+
env-var override) rather than boto3's own chain: botocore only honours
|
|
135
|
+
AWS_DEFAULT_REGION, so on the EC2 job box — where user-data exports
|
|
136
|
+
AWS_REGION and there is no ~/.aws/config — an ambient-chain session has
|
|
137
|
+
no region at all (NoRegionError).
|
|
138
|
+
"""
|
|
139
|
+
if region is None:
|
|
140
|
+
from g3dt.config import load_marker # late import (config imports us)
|
|
141
|
+
|
|
142
|
+
region = load_marker()["region"]
|
|
143
|
+
return boto3.Session(profile_name=profile, region_name=region)
|
|
144
|
+
|
|
145
|
+
|
|
130
146
|
@functools.lru_cache(maxsize=None)
|
|
131
|
-
def resolve(
|
|
147
|
+
def resolve(
|
|
148
|
+
project: str,
|
|
149
|
+
env: str,
|
|
150
|
+
profile: Optional[str] = None,
|
|
151
|
+
region: Optional[str] = None,
|
|
152
|
+
) -> ResolvedConfig:
|
|
132
153
|
"""Resolve all deployed names for ``project``/``env`` from SSM (cached).
|
|
133
154
|
|
|
134
|
-
Cached on ``(project, env, profile)`` for the life of the process,
|
|
135
|
-
single CLI invocation makes exactly one ``get_parameters_by_path``
|
|
155
|
+
Cached on ``(project, env, profile, region)`` for the life of the process,
|
|
156
|
+
so a single CLI invocation makes exactly one ``get_parameters_by_path``
|
|
136
157
|
round-trip no matter how many commands ask for a name. Call
|
|
137
158
|
``resolve.cache_clear()`` in tests.
|
|
138
159
|
|
|
139
160
|
``profile`` is the local AWS named profile to authenticate the read (e.g.
|
|
140
161
|
``etl_test``); ``None`` uses the default credential chain — which is the
|
|
141
162
|
instance profile on the EC2 job box and the build role in CodeBuild.
|
|
163
|
+
``region`` defaults to the marker's region.
|
|
142
164
|
"""
|
|
143
|
-
session =
|
|
165
|
+
session = _session(profile, region)
|
|
144
166
|
return ResolvedConfig(
|
|
145
167
|
project=project, env=env, params=_fetch_params(project, env, session=session)
|
|
146
168
|
)
|
|
147
169
|
|
|
148
170
|
|
|
149
|
-
def list_envs(
|
|
171
|
+
def list_envs(
|
|
172
|
+
project: str, profile: Optional[str] = None, region: Optional[str] = None
|
|
173
|
+
) -> list:
|
|
150
174
|
"""Return the environment names that have an SSM tree under /{project}/.
|
|
151
175
|
|
|
152
176
|
Reads the parameter names one level down (e.g. ``/etl/test/...`` -> ``test``).
|
|
153
177
|
"""
|
|
154
|
-
session =
|
|
178
|
+
session = _session(profile, region)
|
|
155
179
|
client = session.client("ssm")
|
|
156
180
|
paginator = client.get_paginator("get_parameters_by_path")
|
|
157
181
|
envs = set()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/__init__.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/dispatch.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/cli/_internal/registry.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/indexd/indexd_registrar.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/dictionary/deploy_dd.sh
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/dictionary/pull_dict.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/services/k8s_ops/login_to_pod.sh
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/upload/metadata_deleter.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/upload/metadata_submitter.py
RENAMED
|
File without changes
|
{gen3_dataops_toolkit-2.1.0 → gen3_dataops_toolkit-2.1.1}/src/g3dt/upload/upload_synthdata_s3.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|