nci-cidc-api-modules 1.1.38__py3-none-any.whl → 1.1.40__py3-none-any.whl

cidc_api/config/db.py

@@ -5,6 +5,7 @@ from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate, upgrade
 from sqlalchemy.engine.url import URL
 from sqlalchemy.orm import declarative_base
+from google.cloud.sql.connector import Connector, IPTypes
 
 from .secrets import get_secrets_manager
 
@@ -12,6 +13,20 @@ db = SQLAlchemy()
 BaseModel = declarative_base()
 db.Model = BaseModel
 
+connector = Connector()
+
+
+def getconn():
+    return connector.connect(
+        environ.get("CLOUD_SQL_INSTANCE_NAME"),
+        "pg8000",
+        user=environ.get("CLOUD_SQL_DB_USER"),
+        password="xxxxx",
+        db=environ.get("CLOUD_SQL_DB_NAME"),
+        enable_iam_auth=True,
+        ip_type=IPTypes.PUBLIC,
+    )
+
 
 def init_db(app: Flask):
     """Connect `app` to the database and run migrations"""
@@ -30,6 +45,9 @@ def get_sqlalchemy_database_uri(testing: bool = False) -> str:
         # Connect to the test database
         db_uri = environ.get("TEST_POSTGRES_URI", "fake-conn-string")
     elif not db_uri:
+        # TODO: figure out why IAM authentication causes an issue with the huge filter query
+        # db_uri = f"postgresql+pg8000://{environ.get('CLOUD_SQL_DB_USER')}:xxx@/{environ.get('CLOUD_SQL_DB_NAME')}"
+
         secrets = get_secrets_manager(testing)
 
         # If POSTGRES_URI env variable is not set,
@@ -58,3 +76,13 @@ def get_sqlalchemy_database_uri(testing: bool = False) -> str:
     assert db_uri
 
     return db_uri
+
+
+# Use SQLALCHEMY_ENGINE_OPTIONS to connect to the cloud but use uri for local db
+def cloud_connector(testing: bool = False):
+    return {}
+    # TODO: figure out IAM authentication
+    # if not testing and not environ.get("POSTGRES_URI"):
+    #    return {"creator": getconn}
+    # else:
+    #    return {}

cidc_api/config/settings.py

@@ -10,7 +10,7 @@ from os import environ, path, mkdir
 
 from dotenv import load_dotenv
 
-from .db import get_sqlalchemy_database_uri
+from .db import get_sqlalchemy_database_uri, cloud_connector
 from .secrets import get_secrets_manager
 
 load_dotenv()
@@ -54,6 +54,7 @@ else:
 
 ### Configure Flask-SQLAlchemy ###
 SQLALCHEMY_DATABASE_URI = get_sqlalchemy_database_uri(TESTING)
+SQLALCHEMY_ENGINE_OPTIONS = cloud_connector(TESTING)
 SQLALCHEMY_TRACK_MODIFICATIONS = False
 SQLALCHEMY_ECHO = False  # Set to True to emit all compiled sql statements
 
@@ -81,6 +82,7 @@ GOOGLE_PATIENT_SAMPLE_TOPIC = environ["GOOGLE_PATIENT_SAMPLE_TOPIC"]
 GOOGLE_EMAILS_TOPIC = environ["GOOGLE_EMAILS_TOPIC"]
 GOOGLE_ARTIFACT_UPLOAD_TOPIC = environ["GOOGLE_ARTIFACT_UPLOAD_TOPIC"]
 GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC = environ["GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC"]
+GOOGLE_HL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_HL_CLINICAL_VALIDATION_TOPIC"]
 GOOGLE_AND_OPERATOR = " && "
 GOOGLE_OR_OPERATOR = " || "
 

cidc_api/models/models.py

@@ -80,6 +80,7 @@ from sqlalchemy import (
     Boolean,
     CheckConstraint,
     Column,
+    Date,
     DateTime,
     Enum,
     ForeignKey,
@@ -3354,16 +3355,28 @@ class PreprocessedFiles(CommonColumns):
             .all()
         )
 
-    # TODO: logic for pending vs current files after high level validation
     @classmethod
     @with_default_session
-    def get_pending_non_admin_files(cls, job_id: int, session: Session) -> list["PreprocessedFiles"]:
-        return (
+    def get_latest_non_admin_files(cls, job_id: int, session: Session) -> list["PreprocessedFiles"]:
+        """Return the most recently uploaded file for each non-admin file category for the given job_id."""
+        # Subquery to get latest _created per file_category
+        latest_subquery = (
+            session.query(cls.file_category, func.max(cls._created).label("latest_created"))
+            .filter(cls.job_id == job_id, cls.file_category.notin_(ADMIN_FILE_CATEGORIES))
+            .group_by(cls.file_category)
+            .subquery()
+        )
+        # Join main table on file_category and latest _created to get full records of latest files
+        latest_files = (
             session.query(cls)
-            .filter(cls.job_id == job_id)
-            .filter(cls.status == "pending", cls.file_category.notin_(ADMIN_FILE_CATEGORIES))
+            .join(
+                latest_subquery,
+                (cls.file_category == latest_subquery.c.file_category)
+                & (cls._created == latest_subquery.c.latest_created),
+            )
             .all()
         )
+        return latest_files
 
     @classmethod
     def add_job_filter(cls, query, job_id):
@@ -3413,11 +3426,13 @@ class IngestionJobs(CommonColumns):
     status = Column("status", Enum(*INGESTION_JOB_STATUSES, name="status"), nullable=False)
     trial_id = Column(String, nullable=False)
     version = Column(Integer, nullable=False)
+    pending = Column(Boolean, nullable=False, default=False)
+    start_date = Column(Date, nullable=True)
 
     @staticmethod
     @with_default_session
-    def create(trial_id: str, status: str, version: int, session: Session = None):
-        new_job = IngestionJobs(trial_id=trial_id, status=status, version=version)
+    def create(trial_id: str, status: str, version: int, pending: Boolean = False, session: Session = None):
+        new_job = IngestionJobs(trial_id=trial_id, status=status, version=version, pending=pending)
         new_job.insert(session=session)
         return new_job
 
@@ -3444,6 +3459,20 @@ class IngestionJobs(CommonColumns):
                 headers_ended = True
         return categories
 
+    @classmethod
+    @with_default_session
+    def atomic_set_job_as_pending(cls, job_id: int, session: Session) -> Boolean:
+        # Preventing rare race condition where multiple people try and submit a job for validation
+        result = bool(
+            session.execute(
+                update(IngestionJobs)
+                .where(and_(IngestionJobs.id == job_id, IngestionJobs.pending == False))
+                .values(pending=True)
+            ).rowcount
+        )
+        session.commit()
+        return result
+
     @classmethod
     @with_default_session
     def get_jobs_by_trial(cls, trial_id: str, session: Session = None) -> list["IngestionJobs"]:

cidc_api/shared/file_handling.py

@@ -1,10 +1,15 @@
+from pathlib import Path
+
 from werkzeug.datastructures import FileStorage
-from werkzeug.exceptions import BadRequest
+from werkzeug.exceptions import BadRequest, InternalServerError
 
+from ..config.logging import get_logger
 from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
 from ..models import PreprocessedFiles
 from ..shared.auth import get_current_user
-from ..shared.gcloud_client import upload_file_to_gcs
+from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
+
+logger = get_logger(__name__)
 
 
 def set_current_file(file: FileStorage, file_category: str, gcs_folder: str, job_id: int = None) -> PreprocessedFiles:
@@ -22,7 +27,7 @@ def create_file(
 ) -> PreprocessedFiles:
     """Upload file to GCS and create corresponding metadata record in the database."""
     status = "pending" if gcs_folder.endswith("pending/") else "current"
-    # only need timestamp for current/approved files
+    # only need timestamp for current/versioned files
     append_timestamp = status == "current"
     # create file in GCS
     gcs_file_path = upload_file_to_gcs(file, GOOGLE_CLINICAL_DATA_BUCKET, gcs_folder, append_timestamp=append_timestamp)
@@ -56,35 +61,36 @@ def format_common_preprocessed_file_response(file: PreprocessedFiles):
     }
 
 
-# TODO Below functions approve_pending_file and delete_pending_files were copied from deleted clinical_data.py
-# Consider re-implementing with pending files in clinical data file uploads, or remove
-# def approve_pending_file(pending_file: FileStorage):
-#     original_filename = pending_file.file_name
-#     pending_gcs_path = pending_file.object_url
-#     try:
-#         new_gcs_path = gcloud_client.move_gcs_file(
-#             GOOGLE_CLINICAL_DATA_BUCKET, pending_gcs_path, f"{MASTER_APPENDIX_A}/"
-#         )
-#     except Exception as e:
-#         logger.error(str(e))
-#         raise InternalServerError(str(e))
-#     # Move any 'current' file(s) to 'archived' status
-#     latest_version = PreprocessedFiles.archive_current_files(MASTER_APPENDIX_A)
-#     # Insert new "approved" DB record
-#     PreprocessedFiles.create(
-#         file_name=original_filename,
-#         object_url=new_gcs_path,
-#         file_category=MASTER_APPENDIX_A,
-#         uploader_email=get_current_user().email,
-#         status="current",
-#         version=latest_version + 1,
-#     )
-#     # Delete pending record
-#     pending_file.delete()
-#     return new_gcs_path
-#
-#
-# def delete_pending_files(pending_folder: str, file_category: str):
-#     """Deletes specified pending file(s) from GCS and associated db record(s)."""
-#     gcloud_client.delete_items_from_folder(GOOGLE_CLINICAL_DATA_BUCKET, pending_folder)
-#     PreprocessedFiles.delete_pending_files_by_category(file_category)
+def version_pending_file(pending_file: PreprocessedFiles):
+    """Transitions an existing pending file to be a current versioned file."""
+    original_filename = pending_file.file_name
+    pending_gcs_path = pending_file.object_url
+    try:
+        versioned_gcs_folder = strip_filename_and_pending_folder(pending_gcs_path)
+        new_gcs_path = move_gcs_file(GOOGLE_CLINICAL_DATA_BUCKET, pending_gcs_path, versioned_gcs_folder)
+    except Exception as e:
+        logger.error(str(e))
+        raise InternalServerError(str(e))
+    # Move any 'current' file(s) to 'archived' status
+    latest_version = PreprocessedFiles.archive_current_files(pending_file.file_category, pending_file.job_id)
+    # Insert new current/versioned DB record
+    PreprocessedFiles.create(
+        file_name=original_filename,
+        object_url=new_gcs_path,
+        file_category=pending_file.file_category,
+        uploader_email=get_current_user().email,
+        status="current",
+        job_id=pending_file.job_id,
+        version=latest_version + 1,
+    )
+    # Delete pending record
+    pending_file.delete()
+    return new_gcs_path
+
+
+def strip_filename_and_pending_folder(path_str):
+    """Returns the file path above the 'pending' folder to be used for versioned files."""
+    path = Path(path_str)
+    if path.parent.name != "pending":
+        raise ValueError("Expected 'pending' folder above file")
+    return str(path.parent.parent)

cidc_api/shared/gcloud_client.py

@@ -25,8 +25,8 @@ from typing import (
 )
 
 import googleapiclient.discovery
-import requests
 import pandas as pd
+import requests
 from cidc_schemas.prism.constants import ASSAY_TO_FILEPATH
 from google.api_core.client_options import ClientOptions
 from google.api_core.iam import Policy
@@ -54,6 +54,7 @@ from ..config.settings import (
     GOOGLE_PATIENT_SAMPLE_TOPIC,
     GOOGLE_ARTIFACT_UPLOAD_TOPIC,
     GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC,
+    GOOGLE_HL_CLINICAL_VALIDATION_TOPIC,
     TESTING,
     ENV,
     IS_EMAIL_ON,
@@ -973,6 +974,12 @@ def publish_artifact_upload(file_id: int) -> None:
         report.result()
 
 
+def publish_hl_clinical_validation(job_id: int) -> None:
+    """Publish to the high_level_clinical_validation topic that a job's files are ready to be validated."""
+    # Start validation asynchronously
+    _report = _encode_and_publish(str(job_id), GOOGLE_HL_CLINICAL_VALIDATION_TOPIC)
+
+
 def send_email(to_emails: List[str], subject: str, html_content: str, **kw) -> None:
     """
     Publish an email-to-send to the emails topic.

{nci_cidc_api_modules-1.1.38.dist-info → nci_cidc_api_modules-1.1.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.38
+Version: 1.1.40
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -28,7 +28,8 @@ Requires-Dist: python-dotenv==0.10.3
 Requires-Dist: requests==2.32.4
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: certifi==2024.7.4
-Requires-Dist: nci-cidc-schemas==0.27.27
+Requires-Dist: cloud-sql-python-connector[pg8000]==1.18.3
+Requires-Dist: nci-cidc-schemas==0.27.28
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
@@ -206,44 +207,15 @@ FLASK_APP=cidc_api.app:app flask db upgrade
 
 ### Connecting to a Cloud SQL database instance
 
-Install the [Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/quickstart-proxy-test):
-
-```bash
-sudo curl -o /usr/local/bin/cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.darwin.amd64
-sudo chmod +x /usr/local/bin/cloud-sql-proxy
-mkdir ~/.cloudsql
-chmod 770 ~/.cloudsql
-```
-
-Proxy to the dev Cloud SQL instance:
-
-```bash
-cloud-sql-proxy --auto-iam-authn --address 127.0.0.1 --port 5432 nih-nci-cimac-cidc-dev2:us-east4:cidc-postgresql-dev2 &
-```
-
-If you want to run the proxy alongside a postgres instance on localhost listening on 5432, change the port for the proxy to another port instead like 5433.
-If you experience auth errors, make sure your google cloud sdk is authenticated.
+Make sure you are authenticated to gcloud:
 
 ```bash
 gcloud auth login
 gcloud auth application-default login
 ```
 
-To point an API running on localhost to the remote Postgres database, edit your `.env` file and comment out `POSTGRES_URI` and uncomment all environment variables prefixed with `CLOUD_SQL_`. Change CLOUD_SQL_SOCKET_DIR to contain a reference to your home directory. Restart your local API instance, and it will connect to the staging Cloud SQL instance via the local proxy.
-
-If you wish to connect to the staging Cloud SQL instance via the postgres REPL, download and run the CIDC sql proxy tool (a wrapper for `cloud_sql_proxy`):
-
-```bash
-# Download the proxy
-curl https://raw.githubusercontent.com/NCI-CIDC/cidc-devops/master/scripts/cidc_sql_proxy.sh -o /usr/local/bin/cidc_sql_proxy
-
-# Prepare the proxy
-chmod +x /usr/local/bin/cidc_sql_proxy
-cidc_sql_proxy install
-
-# Run the proxy
-cidc_sql_proxy staging # or cidc_sql_proxy prod
-```
+In your .env file, comment out `POSTGRES_URI` and uncommment
+`CLOUD_SQL_INSTANCE_NAME CLOUD_SQL_DB_USER CLOUD_SQL_DB_NAME` Replace `CLOUD_SQL_DB_USER` with your NIH email.
 
 ### Running database migrations
 

{nci_cidc_api_modules-1.1.38.dist-info → nci_cidc_api_modules-1.1.40.dist-info}/RECORD

@@ -1,11 +1,11 @@
 cidc_api/config/__init__.py,sha256=5mX8GAPxUKV84iS-aGOoE-4m68LsOCGCDptXNdlgvj0,148
-cidc_api/config/db.py,sha256=5rf7kIowkiJIqJj2_JtO1cY9L55IjJBonJ-vThA4cGo,1960
+cidc_api/config/db.py,sha256=kbCemCDYv_zyczw-V7H2JvLCa_XeEcvXkzvF28FxADw,2862
 cidc_api/config/logging.py,sha256=abhVYtn8lfhIt0tyV2WHFgSmp_s2eeJh7kodB6LH4J0,1149
 cidc_api/config/secrets.py,sha256=jRFj7W43pWuPf9DZQLCKF7WPXf5cUv-BAaS3ASqhV_Q,1481
-cidc_api/config/settings.py,sha256=mA-4r7oB60uFepYtl5abbPigjwX8aBz__qCJXdcWWbs,4272
+cidc_api/config/settings.py,sha256=NsJbqW6Vqcz2f79xcAbk4th5tHne_I-RPCbKq_3hpz0,4427
 cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
 cidc_api/models/migrations.py,sha256=gp9vtkYbA9FFy2s-7woelAmsvQbJ41LO2_DY-YkFIrQ,11464
-cidc_api/models/models.py,sha256=D4GmcQSLKGBi0k3-w3TZk7J46zKR80JViN2_vTBZ1ZQ,141678
+cidc_api/models/models.py,sha256=eRZrUcQYIe_V5hMwD3ONc175vPQpUTe_KPcmGdYexJc,142944
 cidc_api/models/schemas.py,sha256=6IE2dJoEMcMbi0Vr1V3cYKnPKU0hv9vRKBixOZHe88s,2766
 cidc_api/models/files/__init__.py,sha256=8BMTnUSHzUbz0lBeEQY6NvApxDD3GMWMduoVMos2g4Y,213
 cidc_api/models/files/details.py,sha256=sZkGM7iEV4-J6IDQCdiMV6KBDLbPxCOqUMaU3aY9rX8,65153
@@ -13,12 +13,12 @@ cidc_api/models/files/facets.py,sha256=WqjfqtYJgY2tBnZ598Yc0eJdQUo2slFNLyTDaqPx_
 cidc_api/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cidc_api/shared/auth.py,sha256=PHqmVGkqDjbmUofytVFwD_9ssgUomESl3fFtFHPwZYQ,9062
 cidc_api/shared/emails.py,sha256=HQIixEUsR8yyu7Iv8S81RjtvEQeGuzQHzBfGsWIfP7k,4961
-cidc_api/shared/file_handling.py,sha256=lbdY4XH-otpNjWDY1g6EQoVWtEYM_9j95OlQXeeFkhE,3808
-cidc_api/shared/gcloud_client.py,sha256=tgi6Ja31EUQcJueAIYHc3VyrMchoMZCdui1eruakCLg,36351
+cidc_api/shared/file_handling.py,sha256=l4wiRkVJLL7QbCoODsLx-uki6Km8QoMmUlRVnUV9vkk,3894
+cidc_api/shared/gcloud_client.py,sha256=cNtzCRsD_StBQRAzWJ8BimW-ecEGOqywvlM7kip8CrE,36681
 cidc_api/shared/jose.py,sha256=-qzGzEDAlokEp9E7WtBtQkXyyfPWTYXlwYpCqVJWmqM,1830
 cidc_api/shared/rest_utils.py,sha256=RwR30WOUAYCxL7V-i2totEyeriG30GbBDvBcpLXhM9w,6594
-nci_cidc_api_modules-1.1.38.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
-nci_cidc_api_modules-1.1.38.dist-info/METADATA,sha256=Vlhx5ZliL-b3-3Umhqm1q22BztzUG-mc2dTIyyHzZ24,41284
-nci_cidc_api_modules-1.1.38.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nci_cidc_api_modules-1.1.38.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
-nci_cidc_api_modules-1.1.38.dist-info/RECORD,,
+nci_cidc_api_modules-1.1.40.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
+nci_cidc_api_modules-1.1.40.dist-info/METADATA,sha256=IZwg1tt0j0d6j8ve8d3yYrfck4Hnp_GKo2By-9bb0vM,39967
+nci_cidc_api_modules-1.1.40.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nci_cidc_api_modules-1.1.40.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
+nci_cidc_api_modules-1.1.40.dist-info/RECORD,,