nci-cidc-api-modules 1.1.37__tar.gz → 1.1.39__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nci_cidc_api_modules-1.1.37/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.1.39}/PKG-INFO +5 -33
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/README.md +3 -32
- nci_cidc_api_modules-1.1.39/cidc_api/config/db.py +59 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/config/settings.py +3 -1
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/models.py +36 -8
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/file_handling.py +43 -3
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/gcloud_client.py +8 -1
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39/nci_cidc_api_modules.egg-info}/PKG-INFO +5 -33
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/nci_cidc_api_modules.egg-info/requires.txt +1 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/requirements.modules.txt +1 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/tests/test_api.py +0 -4
- nci_cidc_api_modules-1.1.37/cidc_api/config/db.py +0 -60
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/LICENSE +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/MANIFEST.in +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/config/__init__.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/config/logging.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/config/secrets.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/__init__.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/files/__init__.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/files/details.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/files/facets.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/migrations.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/schemas.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/__init__.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/auth.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/emails.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/jose.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/rest_utils.py +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/nci_cidc_api_modules.egg-info/SOURCES.txt +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/nci_cidc_api_modules.egg-info/dependency_links.txt +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/nci_cidc_api_modules.egg-info/not-zip-safe +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/nci_cidc_api_modules.egg-info/top_level.txt +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/pyproject.toml +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/setup.cfg +0 -0
- {nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/setup.py +0 -0
{nci_cidc_api_modules-1.1.37/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.1.39}/PKG-INFO
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: nci_cidc_api_modules
|
3
|
-
Version: 1.1.
|
3
|
+
Version: 1.1.39
|
4
4
|
Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
|
5
5
|
Home-page: https://github.com/NCI-CIDC/cidc-api-gae
|
6
6
|
License: MIT license
|
@@ -28,6 +28,7 @@ Requires-Dist: python-dotenv==0.10.3
|
|
28
28
|
Requires-Dist: requests==2.32.4
|
29
29
|
Requires-Dist: jinja2==3.1.6
|
30
30
|
Requires-Dist: certifi==2024.7.4
|
31
|
+
Requires-Dist: cloud-sql-python-connector[pg8000]==1.18.3
|
31
32
|
Requires-Dist: nci-cidc-schemas==0.27.27
|
32
33
|
Dynamic: description
|
33
34
|
Dynamic: description-content-type
|
@@ -206,44 +207,15 @@ FLASK_APP=cidc_api.app:app flask db upgrade
|
|
206
207
|
|
207
208
|
### Connecting to a Cloud SQL database instance
|
208
209
|
|
209
|
-
|
210
|
-
|
211
|
-
```bash
|
212
|
-
sudo curl -o /usr/local/bin/cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.darwin.amd64
|
213
|
-
sudo chmod +x /usr/local/bin/cloud-sql-proxy
|
214
|
-
mkdir ~/.cloudsql
|
215
|
-
chmod 770 ~/.cloudsql
|
216
|
-
```
|
217
|
-
|
218
|
-
Proxy to the dev Cloud SQL instance:
|
219
|
-
|
220
|
-
```bash
|
221
|
-
cloud-sql-proxy --auto-iam-authn --address 127.0.0.1 --port 5432 nih-nci-cimac-cidc-dev2:us-east4:cidc-postgresql-dev2 &
|
222
|
-
```
|
223
|
-
|
224
|
-
If you want to run the proxy alongside a postgres instance on localhost listening on 5432, change the port for the proxy to another port instead like 5433.
|
225
|
-
If you experience auth errors, make sure your google cloud sdk is authenticated.
|
210
|
+
Make sure you are authenticated to gcloud:
|
226
211
|
|
227
212
|
```bash
|
228
213
|
gcloud auth login
|
229
214
|
gcloud auth application-default login
|
230
215
|
```
|
231
216
|
|
232
|
-
|
233
|
-
|
234
|
-
If you wish to connect to the staging Cloud SQL instance via the postgres REPL, download and run the CIDC sql proxy tool (a wrapper for `cloud_sql_proxy`):
|
235
|
-
|
236
|
-
```bash
|
237
|
-
# Download the proxy
|
238
|
-
curl https://raw.githubusercontent.com/NCI-CIDC/cidc-devops/master/scripts/cidc_sql_proxy.sh -o /usr/local/bin/cidc_sql_proxy
|
239
|
-
|
240
|
-
# Prepare the proxy
|
241
|
-
chmod +x /usr/local/bin/cidc_sql_proxy
|
242
|
-
cidc_sql_proxy install
|
243
|
-
|
244
|
-
# Run the proxy
|
245
|
-
cidc_sql_proxy staging # or cidc_sql_proxy prod
|
246
|
-
```
|
217
|
+
In your .env file, comment out `POSTGRES_URI` and uncommment
|
218
|
+
`CLOUD_SQL_INSTANCE_NAME CLOUD_SQL_DB_USER CLOUD_SQL_DB_NAME` Replace `CLOUD_SQL_DB_USER` with your NIH email.
|
247
219
|
|
248
220
|
### Running database migrations
|
249
221
|
|
@@ -166,44 +166,15 @@ FLASK_APP=cidc_api.app:app flask db upgrade
|
|
166
166
|
|
167
167
|
### Connecting to a Cloud SQL database instance
|
168
168
|
|
169
|
-
|
170
|
-
|
171
|
-
```bash
|
172
|
-
sudo curl -o /usr/local/bin/cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.darwin.amd64
|
173
|
-
sudo chmod +x /usr/local/bin/cloud-sql-proxy
|
174
|
-
mkdir ~/.cloudsql
|
175
|
-
chmod 770 ~/.cloudsql
|
176
|
-
```
|
177
|
-
|
178
|
-
Proxy to the dev Cloud SQL instance:
|
179
|
-
|
180
|
-
```bash
|
181
|
-
cloud-sql-proxy --auto-iam-authn --address 127.0.0.1 --port 5432 nih-nci-cimac-cidc-dev2:us-east4:cidc-postgresql-dev2 &
|
182
|
-
```
|
183
|
-
|
184
|
-
If you want to run the proxy alongside a postgres instance on localhost listening on 5432, change the port for the proxy to another port instead like 5433.
|
185
|
-
If you experience auth errors, make sure your google cloud sdk is authenticated.
|
169
|
+
Make sure you are authenticated to gcloud:
|
186
170
|
|
187
171
|
```bash
|
188
172
|
gcloud auth login
|
189
173
|
gcloud auth application-default login
|
190
174
|
```
|
191
175
|
|
192
|
-
|
193
|
-
|
194
|
-
If you wish to connect to the staging Cloud SQL instance via the postgres REPL, download and run the CIDC sql proxy tool (a wrapper for `cloud_sql_proxy`):
|
195
|
-
|
196
|
-
```bash
|
197
|
-
# Download the proxy
|
198
|
-
curl https://raw.githubusercontent.com/NCI-CIDC/cidc-devops/master/scripts/cidc_sql_proxy.sh -o /usr/local/bin/cidc_sql_proxy
|
199
|
-
|
200
|
-
# Prepare the proxy
|
201
|
-
chmod +x /usr/local/bin/cidc_sql_proxy
|
202
|
-
cidc_sql_proxy install
|
203
|
-
|
204
|
-
# Run the proxy
|
205
|
-
cidc_sql_proxy staging # or cidc_sql_proxy prod
|
206
|
-
```
|
176
|
+
In your .env file, comment out `POSTGRES_URI` and uncommment
|
177
|
+
`CLOUD_SQL_INSTANCE_NAME CLOUD_SQL_DB_USER CLOUD_SQL_DB_NAME` Replace `CLOUD_SQL_DB_USER` with your NIH email.
|
207
178
|
|
208
179
|
### Running database migrations
|
209
180
|
|
@@ -0,0 +1,59 @@
|
|
1
|
+
from os import environ
|
2
|
+
|
3
|
+
from flask import Flask
|
4
|
+
from flask_sqlalchemy import SQLAlchemy
|
5
|
+
from flask_migrate import Migrate, upgrade
|
6
|
+
from sqlalchemy.engine.url import URL
|
7
|
+
from sqlalchemy.orm import declarative_base
|
8
|
+
from google.cloud.sql.connector import Connector, IPTypes
|
9
|
+
|
10
|
+
from .secrets import get_secrets_manager
|
11
|
+
|
12
|
+
db = SQLAlchemy()
|
13
|
+
BaseModel = declarative_base()
|
14
|
+
db.Model = BaseModel
|
15
|
+
|
16
|
+
connector = Connector()
|
17
|
+
|
18
|
+
|
19
|
+
def getconn():
|
20
|
+
return connector.connect(
|
21
|
+
environ.get("CLOUD_SQL_INSTANCE_NAME"),
|
22
|
+
"pg8000",
|
23
|
+
user=environ.get("CLOUD_SQL_DB_USER"),
|
24
|
+
password="xxxxx",
|
25
|
+
db=environ.get("CLOUD_SQL_DB_NAME"),
|
26
|
+
enable_iam_auth=True,
|
27
|
+
ip_type=IPTypes.PUBLIC,
|
28
|
+
)
|
29
|
+
|
30
|
+
|
31
|
+
def init_db(app: Flask):
|
32
|
+
"""Connect `app` to the database and run migrations"""
|
33
|
+
db.init_app(app)
|
34
|
+
db.Model = BaseModel
|
35
|
+
Migrate(app, db, app.config["MIGRATIONS_PATH"])
|
36
|
+
with app.app_context():
|
37
|
+
upgrade(app.config["MIGRATIONS_PATH"])
|
38
|
+
|
39
|
+
|
40
|
+
def get_sqlalchemy_database_uri(testing: bool = False) -> str:
|
41
|
+
"""Get the PostgreSQL DB URI from environment variables"""
|
42
|
+
|
43
|
+
db_uri = environ.get("POSTGRES_URI")
|
44
|
+
if testing:
|
45
|
+
# Connect to the test database
|
46
|
+
db_uri = environ.get("TEST_POSTGRES_URI", "fake-conn-string")
|
47
|
+
elif not db_uri:
|
48
|
+
db_uri = f"postgresql+pg8000://{environ.get('CLOUD_SQL_DB_USER')}:xxx@/{environ.get('CLOUD_SQL_DB_NAME')}"
|
49
|
+
assert db_uri
|
50
|
+
|
51
|
+
return db_uri
|
52
|
+
|
53
|
+
|
54
|
+
# Use SQLALCHEMY_ENGINE_OPTIONS to connect to the cloud but use uri for local db
|
55
|
+
def cloud_connector(testing: bool = False):
|
56
|
+
if not testing and not environ.get("POSTGRES_URI"):
|
57
|
+
return {"creator": getconn}
|
58
|
+
else:
|
59
|
+
return {}
|
@@ -10,7 +10,7 @@ from os import environ, path, mkdir
|
|
10
10
|
|
11
11
|
from dotenv import load_dotenv
|
12
12
|
|
13
|
-
from .db import get_sqlalchemy_database_uri
|
13
|
+
from .db import get_sqlalchemy_database_uri, cloud_connector
|
14
14
|
from .secrets import get_secrets_manager
|
15
15
|
|
16
16
|
load_dotenv()
|
@@ -54,6 +54,7 @@ else:
|
|
54
54
|
|
55
55
|
### Configure Flask-SQLAlchemy ###
|
56
56
|
SQLALCHEMY_DATABASE_URI = get_sqlalchemy_database_uri(TESTING)
|
57
|
+
SQLALCHEMY_ENGINE_OPTIONS = cloud_connector(TESTING)
|
57
58
|
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
58
59
|
SQLALCHEMY_ECHO = False # Set to True to emit all compiled sql statements
|
59
60
|
|
@@ -81,6 +82,7 @@ GOOGLE_PATIENT_SAMPLE_TOPIC = environ["GOOGLE_PATIENT_SAMPLE_TOPIC"]
|
|
81
82
|
GOOGLE_EMAILS_TOPIC = environ["GOOGLE_EMAILS_TOPIC"]
|
82
83
|
GOOGLE_ARTIFACT_UPLOAD_TOPIC = environ["GOOGLE_ARTIFACT_UPLOAD_TOPIC"]
|
83
84
|
GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC = environ["GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC"]
|
85
|
+
GOOGLE_HL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_HL_CLINICAL_VALIDATION_TOPIC"]
|
84
86
|
GOOGLE_AND_OPERATOR = " && "
|
85
87
|
GOOGLE_OR_OPERATOR = " || "
|
86
88
|
|
@@ -80,6 +80,7 @@ from sqlalchemy import (
|
|
80
80
|
Boolean,
|
81
81
|
CheckConstraint,
|
82
82
|
Column,
|
83
|
+
Date,
|
83
84
|
DateTime,
|
84
85
|
Enum,
|
85
86
|
ForeignKey,
|
@@ -357,6 +358,7 @@ class CIDCRole(EnumBaseClass):
|
|
357
358
|
NCI_BIOBANK_USER = "nci-biobank-user"
|
358
359
|
NETWORK_VIEWER = "network-viewer"
|
359
360
|
PACT_USER = "pact-user"
|
361
|
+
CLINICAL_TRIAL_USER = "clinical-trial-user"
|
360
362
|
|
361
363
|
|
362
364
|
ROLES = [role.value for role in CIDCRole]
|
@@ -3215,7 +3217,7 @@ def upload_manifest_json(
|
|
3215
3217
|
* The updated trial metadata object is updated in the `TrialMetadata` table.
|
3216
3218
|
"""
|
3217
3219
|
try:
|
3218
|
-
TrialMetadata.patch_manifest(trial_id, md_patch, session=session, commit=
|
3220
|
+
TrialMetadata.patch_manifest(trial_id, md_patch, session=session, commit=True)
|
3219
3221
|
except ValidationError as e:
|
3220
3222
|
raise BadRequest(json_validation.format_validation_error(e)) from e
|
3221
3223
|
except ValidationMultiError as e:
|
@@ -3353,16 +3355,28 @@ class PreprocessedFiles(CommonColumns):
|
|
3353
3355
|
.all()
|
3354
3356
|
)
|
3355
3357
|
|
3356
|
-
# TODO: logic for pending vs current files after high level validation
|
3357
3358
|
@classmethod
|
3358
3359
|
@with_default_session
|
3359
|
-
def
|
3360
|
-
|
3360
|
+
def get_latest_non_admin_files(cls, job_id: int, session: Session) -> list["PreprocessedFiles"]:
|
3361
|
+
"""Return the most recently uploaded file for each non-admin file category for the given job_id."""
|
3362
|
+
# Subquery to get latest _created per file_category
|
3363
|
+
latest_subquery = (
|
3364
|
+
session.query(cls.file_category, func.max(cls._created).label("latest_created"))
|
3365
|
+
.filter(cls.job_id == job_id, cls.file_category.notin_(ADMIN_FILE_CATEGORIES))
|
3366
|
+
.group_by(cls.file_category)
|
3367
|
+
.subquery()
|
3368
|
+
)
|
3369
|
+
# Join main table on file_category and latest _created to get full records of latest files
|
3370
|
+
latest_files = (
|
3361
3371
|
session.query(cls)
|
3362
|
-
.
|
3363
|
-
|
3372
|
+
.join(
|
3373
|
+
latest_subquery,
|
3374
|
+
(cls.file_category == latest_subquery.c.file_category)
|
3375
|
+
& (cls._created == latest_subquery.c.latest_created),
|
3376
|
+
)
|
3364
3377
|
.all()
|
3365
3378
|
)
|
3379
|
+
return latest_files
|
3366
3380
|
|
3367
3381
|
@classmethod
|
3368
3382
|
def add_job_filter(cls, query, job_id):
|
@@ -3412,11 +3426,13 @@ class IngestionJobs(CommonColumns):
|
|
3412
3426
|
status = Column("status", Enum(*INGESTION_JOB_STATUSES, name="status"), nullable=False)
|
3413
3427
|
trial_id = Column(String, nullable=False)
|
3414
3428
|
version = Column(Integer, nullable=False)
|
3429
|
+
pending = Column(Boolean, nullable=False, default=False)
|
3430
|
+
start_date = Column(Date, nullable=True)
|
3415
3431
|
|
3416
3432
|
@staticmethod
|
3417
3433
|
@with_default_session
|
3418
|
-
def create(trial_id: str, status: str, version: int, session: Session = None):
|
3419
|
-
new_job = IngestionJobs(trial_id=trial_id, status=status, version=version)
|
3434
|
+
def create(trial_id: str, status: str, version: int, pending: Boolean = False, session: Session = None):
|
3435
|
+
new_job = IngestionJobs(trial_id=trial_id, status=status, version=version, pending=pending)
|
3420
3436
|
new_job.insert(session=session)
|
3421
3437
|
return new_job
|
3422
3438
|
|
@@ -3443,6 +3459,18 @@ class IngestionJobs(CommonColumns):
|
|
3443
3459
|
headers_ended = True
|
3444
3460
|
return categories
|
3445
3461
|
|
3462
|
+
@classmethod
|
3463
|
+
@with_default_session
|
3464
|
+
def atomic_set_job_as_pending(cls, job_id: int, session: Session) -> Boolean:
|
3465
|
+
# Preventing rare race condition where multiple people try and submit a job for validation
|
3466
|
+
return bool(
|
3467
|
+
session.execute(
|
3468
|
+
update(IngestionJobs)
|
3469
|
+
.where(and_(IngestionJobs.id == job_id, IngestionJobs.pending == False))
|
3470
|
+
.values(pending=True)
|
3471
|
+
).rowcount
|
3472
|
+
)
|
3473
|
+
|
3446
3474
|
@classmethod
|
3447
3475
|
@with_default_session
|
3448
3476
|
def get_jobs_by_trial(cls, trial_id: str, session: Session = None) -> list["IngestionJobs"]:
|
{nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/file_handling.py
RENAMED
@@ -1,10 +1,15 @@
|
|
1
|
+
from pathlib import Path
|
2
|
+
|
1
3
|
from werkzeug.datastructures import FileStorage
|
2
|
-
from werkzeug.exceptions import BadRequest
|
4
|
+
from werkzeug.exceptions import BadRequest, InternalServerError
|
3
5
|
|
6
|
+
from ..config.logging import get_logger
|
4
7
|
from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
|
5
8
|
from ..models import PreprocessedFiles
|
6
9
|
from ..shared.auth import get_current_user
|
7
|
-
from ..shared.gcloud_client import upload_file_to_gcs
|
10
|
+
from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
|
11
|
+
|
12
|
+
logger = get_logger(__name__)
|
8
13
|
|
9
14
|
|
10
15
|
def set_current_file(file: FileStorage, file_category: str, gcs_folder: str, job_id: int = None) -> PreprocessedFiles:
|
@@ -22,7 +27,7 @@ def create_file(
|
|
22
27
|
) -> PreprocessedFiles:
|
23
28
|
"""Upload file to GCS and create corresponding metadata record in the database."""
|
24
29
|
status = "pending" if gcs_folder.endswith("pending/") else "current"
|
25
|
-
# only need timestamp for current/
|
30
|
+
# only need timestamp for current/versioned files
|
26
31
|
append_timestamp = status == "current"
|
27
32
|
# create file in GCS
|
28
33
|
gcs_file_path = upload_file_to_gcs(file, GOOGLE_CLINICAL_DATA_BUCKET, gcs_folder, append_timestamp=append_timestamp)
|
@@ -54,3 +59,38 @@ def format_common_preprocessed_file_response(file: PreprocessedFiles):
|
|
54
59
|
"uploader_email": file.uploader_email,
|
55
60
|
"date": file._created.isoformat(),
|
56
61
|
}
|
62
|
+
|
63
|
+
|
64
|
+
def version_pending_file(pending_file: PreprocessedFiles):
|
65
|
+
"""Transitions an existing pending file to be a current versioned file."""
|
66
|
+
original_filename = pending_file.file_name
|
67
|
+
pending_gcs_path = pending_file.object_url
|
68
|
+
try:
|
69
|
+
versioned_gcs_folder = strip_filename_and_pending_folder(pending_gcs_path)
|
70
|
+
new_gcs_path = move_gcs_file(GOOGLE_CLINICAL_DATA_BUCKET, pending_gcs_path, versioned_gcs_folder)
|
71
|
+
except Exception as e:
|
72
|
+
logger.error(str(e))
|
73
|
+
raise InternalServerError(str(e))
|
74
|
+
# Move any 'current' file(s) to 'archived' status
|
75
|
+
latest_version = PreprocessedFiles.archive_current_files(pending_file.file_category, pending_file.job_id)
|
76
|
+
# Insert new current/versioned DB record
|
77
|
+
PreprocessedFiles.create(
|
78
|
+
file_name=original_filename,
|
79
|
+
object_url=new_gcs_path,
|
80
|
+
file_category=pending_file.file_category,
|
81
|
+
uploader_email=get_current_user().email,
|
82
|
+
status="current",
|
83
|
+
job_id=pending_file.job_id,
|
84
|
+
version=latest_version + 1,
|
85
|
+
)
|
86
|
+
# Delete pending record
|
87
|
+
pending_file.delete()
|
88
|
+
return new_gcs_path
|
89
|
+
|
90
|
+
|
91
|
+
def strip_filename_and_pending_folder(path_str):
|
92
|
+
"""Returns the file path above the 'pending' folder to be used for versioned files."""
|
93
|
+
path = Path(path_str)
|
94
|
+
if path.parent.name != "pending":
|
95
|
+
raise ValueError("Expected 'pending' folder above file")
|
96
|
+
return str(path.parent.parent)
|
{nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/shared/gcloud_client.py
RENAMED
@@ -25,8 +25,8 @@ from typing import (
|
|
25
25
|
)
|
26
26
|
|
27
27
|
import googleapiclient.discovery
|
28
|
-
import requests
|
29
28
|
import pandas as pd
|
29
|
+
import requests
|
30
30
|
from cidc_schemas.prism.constants import ASSAY_TO_FILEPATH
|
31
31
|
from google.api_core.client_options import ClientOptions
|
32
32
|
from google.api_core.iam import Policy
|
@@ -54,6 +54,7 @@ from ..config.settings import (
|
|
54
54
|
GOOGLE_PATIENT_SAMPLE_TOPIC,
|
55
55
|
GOOGLE_ARTIFACT_UPLOAD_TOPIC,
|
56
56
|
GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC,
|
57
|
+
GOOGLE_HL_CLINICAL_VALIDATION_TOPIC,
|
57
58
|
TESTING,
|
58
59
|
ENV,
|
59
60
|
IS_EMAIL_ON,
|
@@ -973,6 +974,12 @@ def publish_artifact_upload(file_id: int) -> None:
|
|
973
974
|
report.result()
|
974
975
|
|
975
976
|
|
977
|
+
def publish_hl_clinical_validation(job_id: int) -> None:
|
978
|
+
"""Publish to the high_level_clinical_validation topic that a job's files are ready to be validated."""
|
979
|
+
# Start validation asynchronously
|
980
|
+
_report = _encode_and_publish(str(job_id), GOOGLE_HL_CLINICAL_VALIDATION_TOPIC)
|
981
|
+
|
982
|
+
|
976
983
|
def send_email(to_emails: List[str], subject: str, html_content: str, **kw) -> None:
|
977
984
|
"""
|
978
985
|
Publish an email-to-send to the emails topic.
|
{nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39/nci_cidc_api_modules.egg-info}/PKG-INFO
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: nci_cidc_api_modules
|
3
|
-
Version: 1.1.
|
3
|
+
Version: 1.1.39
|
4
4
|
Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
|
5
5
|
Home-page: https://github.com/NCI-CIDC/cidc-api-gae
|
6
6
|
License: MIT license
|
@@ -28,6 +28,7 @@ Requires-Dist: python-dotenv==0.10.3
|
|
28
28
|
Requires-Dist: requests==2.32.4
|
29
29
|
Requires-Dist: jinja2==3.1.6
|
30
30
|
Requires-Dist: certifi==2024.7.4
|
31
|
+
Requires-Dist: cloud-sql-python-connector[pg8000]==1.18.3
|
31
32
|
Requires-Dist: nci-cidc-schemas==0.27.27
|
32
33
|
Dynamic: description
|
33
34
|
Dynamic: description-content-type
|
@@ -206,44 +207,15 @@ FLASK_APP=cidc_api.app:app flask db upgrade
|
|
206
207
|
|
207
208
|
### Connecting to a Cloud SQL database instance
|
208
209
|
|
209
|
-
|
210
|
-
|
211
|
-
```bash
|
212
|
-
sudo curl -o /usr/local/bin/cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.darwin.amd64
|
213
|
-
sudo chmod +x /usr/local/bin/cloud-sql-proxy
|
214
|
-
mkdir ~/.cloudsql
|
215
|
-
chmod 770 ~/.cloudsql
|
216
|
-
```
|
217
|
-
|
218
|
-
Proxy to the dev Cloud SQL instance:
|
219
|
-
|
220
|
-
```bash
|
221
|
-
cloud-sql-proxy --auto-iam-authn --address 127.0.0.1 --port 5432 nih-nci-cimac-cidc-dev2:us-east4:cidc-postgresql-dev2 &
|
222
|
-
```
|
223
|
-
|
224
|
-
If you want to run the proxy alongside a postgres instance on localhost listening on 5432, change the port for the proxy to another port instead like 5433.
|
225
|
-
If you experience auth errors, make sure your google cloud sdk is authenticated.
|
210
|
+
Make sure you are authenticated to gcloud:
|
226
211
|
|
227
212
|
```bash
|
228
213
|
gcloud auth login
|
229
214
|
gcloud auth application-default login
|
230
215
|
```
|
231
216
|
|
232
|
-
|
233
|
-
|
234
|
-
If you wish to connect to the staging Cloud SQL instance via the postgres REPL, download and run the CIDC sql proxy tool (a wrapper for `cloud_sql_proxy`):
|
235
|
-
|
236
|
-
```bash
|
237
|
-
# Download the proxy
|
238
|
-
curl https://raw.githubusercontent.com/NCI-CIDC/cidc-devops/master/scripts/cidc_sql_proxy.sh -o /usr/local/bin/cidc_sql_proxy
|
239
|
-
|
240
|
-
# Prepare the proxy
|
241
|
-
chmod +x /usr/local/bin/cidc_sql_proxy
|
242
|
-
cidc_sql_proxy install
|
243
|
-
|
244
|
-
# Run the proxy
|
245
|
-
cidc_sql_proxy staging # or cidc_sql_proxy prod
|
246
|
-
```
|
217
|
+
In your .env file, comment out `POSTGRES_URI` and uncommment
|
218
|
+
`CLOUD_SQL_INSTANCE_NAME CLOUD_SQL_DB_USER CLOUD_SQL_DB_NAME` Replace `CLOUD_SQL_DB_USER` with your NIH email.
|
247
219
|
|
248
220
|
### Running database migrations
|
249
221
|
|
@@ -411,10 +411,6 @@ def test_endpoint_urls(cidc_api):
|
|
411
411
|
"""
|
412
412
|
expected_endpoints = {
|
413
413
|
"/",
|
414
|
-
"/clinical_data/files/master_appendix_a/current",
|
415
|
-
"/clinical_data/files/master_appendix_a/pending",
|
416
|
-
"/clinical_data/files/master_appendix_a/versions",
|
417
|
-
"/clinical_data/files/master_appendix_a/versions/<int:version>",
|
418
414
|
"/downloadable_files/",
|
419
415
|
"/downloadable_files/filelist",
|
420
416
|
"/downloadable_files/compressed_batch",
|
@@ -1,60 +0,0 @@
|
|
1
|
-
from os import environ
|
2
|
-
|
3
|
-
from flask import Flask
|
4
|
-
from flask_sqlalchemy import SQLAlchemy
|
5
|
-
from flask_migrate import Migrate, upgrade
|
6
|
-
from sqlalchemy.engine.url import URL
|
7
|
-
from sqlalchemy.orm import declarative_base
|
8
|
-
|
9
|
-
from .secrets import get_secrets_manager
|
10
|
-
|
11
|
-
db = SQLAlchemy()
|
12
|
-
BaseModel = declarative_base()
|
13
|
-
db.Model = BaseModel
|
14
|
-
|
15
|
-
|
16
|
-
def init_db(app: Flask):
|
17
|
-
"""Connect `app` to the database and run migrations"""
|
18
|
-
db.init_app(app)
|
19
|
-
db.Model = BaseModel
|
20
|
-
Migrate(app, db, app.config["MIGRATIONS_PATH"])
|
21
|
-
with app.app_context():
|
22
|
-
upgrade(app.config["MIGRATIONS_PATH"])
|
23
|
-
|
24
|
-
|
25
|
-
def get_sqlalchemy_database_uri(testing: bool = False) -> str:
|
26
|
-
"""Get the PostgreSQL DB URI from environment variables"""
|
27
|
-
|
28
|
-
db_uri = environ.get("POSTGRES_URI")
|
29
|
-
if testing:
|
30
|
-
# Connect to the test database
|
31
|
-
db_uri = environ.get("TEST_POSTGRES_URI", "fake-conn-string")
|
32
|
-
elif not db_uri:
|
33
|
-
secrets = get_secrets_manager(testing)
|
34
|
-
|
35
|
-
# If POSTGRES_URI env variable is not set,
|
36
|
-
# we're connecting to a Cloud SQL instance.
|
37
|
-
|
38
|
-
config: dict = {
|
39
|
-
"drivername": "postgresql",
|
40
|
-
"username": environ.get("CLOUD_SQL_DB_USER"),
|
41
|
-
"password": secrets.get(environ.get("CLOUD_SQL_DB_PASS_ID")),
|
42
|
-
"database": environ.get("CLOUD_SQL_DB_NAME"),
|
43
|
-
}
|
44
|
-
|
45
|
-
if environ.get("CLOUD_SQL_INSTANCE_NAME"):
|
46
|
-
socket_dir = environ.get("CLOUD_SQL_SOCKET_DIR", "/cloudsql/")
|
47
|
-
|
48
|
-
# If CLOUD_SQL_INSTANCE_NAME is defined, we're connecting
|
49
|
-
# via a unix socket from inside App Engine.
|
50
|
-
config["query"] = {"host": f'{socket_dir}{environ.get("CLOUD_SQL_INSTANCE_NAME")}'}
|
51
|
-
else:
|
52
|
-
raise RuntimeError(
|
53
|
-
"Either POSTGRES_URI or CLOUD_SQL_INSTANCE_NAME must be defined to connect " + "to a database."
|
54
|
-
)
|
55
|
-
|
56
|
-
db_uri = str(URL.create(**config).render_as_string(hide_password=False))
|
57
|
-
|
58
|
-
assert db_uri
|
59
|
-
|
60
|
-
return db_uri
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/files/__init__.py
RENAMED
File without changes
|
{nci_cidc_api_modules-1.1.37 → nci_cidc_api_modules-1.1.39}/cidc_api/models/files/details.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|