nci-cidc-api-modules 1.1.30__py3-none-any.whl → 1.1.32__py3-none-any.whl

cidc_api/config/settings.py

@@ -70,6 +70,7 @@ GOOGLE_INTAKE_BUCKET = environ["GOOGLE_INTAKE_BUCKET"]
 GOOGLE_UPLOAD_BUCKET = environ["GOOGLE_UPLOAD_BUCKET"]
 GOOGLE_UPLOAD_TOPIC = environ["GOOGLE_UPLOAD_TOPIC"]
 GOOGLE_ACL_DATA_BUCKET = environ["GOOGLE_ACL_DATA_BUCKET"]
+GOOGLE_CLINICAL_DATA_BUCKET = environ["GOOGLE_CLINICAL_DATA_BUCKET"]
 GOOGLE_EPHEMERAL_BUCKET = environ["GOOGLE_EPHEMERAL_BUCKET"]
 GOOGLE_UPLOAD_ROLE = environ["GOOGLE_UPLOAD_ROLE"]
 GOOGLE_LISTER_ROLE = environ["GOOGLE_LISTER_ROLE"]

cidc_api/models/models.py

@@ -22,6 +22,7 @@ __all__ = [
     "Users",
     "ValidationMultiError",
     "with_default_session",
+    "PreprocessedFiles",
 ]
 
 import hashlib
@@ -3184,3 +3185,73 @@ def upload_manifest_json(
     # Publish that a manifest upload has been received
     publish_patient_sample_update(manifest_upload.id)
     return manifest_upload.id
+
+
+class PreprocessedFiles(CommonColumns):
+    __tablename__ = "preprocessed_files"
+
+    file_name = Column(String)
+    object_url = Column(String)
+    trial_id = Column(String)
+    file_category = Column(String)
+    uploader_email = Column(String)
+    status = Column(String)
+    version = Column(Integer)
+    released_version = Column(String)
+
+    @staticmethod
+    @with_default_session
+    def create(
+        file_name: str,
+        object_url: str,
+        file_category: str,
+        uploader_email: str,
+        status: str = "pending",
+        trial_id: str = None,
+        version: int = None,
+        released_version: str = None,
+        session: Session = None,
+    ):
+        """Create and insert a new PreprocessedFiles record."""
+        new_file = PreprocessedFiles(
+            file_name=file_name,
+            object_url=object_url,
+            file_category=file_category,
+            uploader_email=uploader_email,
+            status=status,
+            trial_id=trial_id,
+            version=version,
+            released_version=released_version,
+        )
+        new_file.insert(session=session)
+        return new_file
+
+    @classmethod
+    @with_default_session
+    def archive_current_files(cls, file_category: str, session: Session = None):
+        """Update any 'current' files in the given category to 'archived'. Returns latest existing version number."""
+        current_version = 0
+        current_files = cls.get_files_by_category_and_status(file_category, "current", session=session)
+        for file in current_files:
+            file.status = "archived"
+            file._updated = datetime.now()
+            current_version = file.version if file.version > current_version else current_version
+        session.commit()
+        return current_version
+
+    @classmethod
+    @with_default_session
+    def delete_pending_files_by_category(cls, file_category: str, session: Session = None):
+        """Delete all pending files matching given file_category."""
+        records = cls.get_files_by_category_and_status(file_category, "pending", session=session)
+        for record in records:
+            session.delete(record)
+        session.commit()
+
+    @classmethod
+    @with_default_session
+    def get_files_by_category_and_status(
+        cls, file_category: str, status: str, session: Session = None
+    ) -> list["PreprocessedFiles"]:
+        """Return all files matching given file_category and status."""
+        return session.query(cls).filter_by(file_category=file_category, status=status).all()

cidc_api/shared/gcloud_client.py

@@ -2,15 +2,16 @@
 
 # pylint: disable=logging-fstring-interpolation
 
-import json
-import os
-from os import environ
 import base64
 import datetime
-import warnings
 import hashlib
+import io
+import json
+import os
+import warnings
 from collections import namedtuple
 from concurrent.futures import Future
+from os import environ
 from typing import (
     Any,
     BinaryIO,
@@ -23,19 +24,20 @@ from typing import (
     Union,
 )
 
-from werkzeug.datastructures import FileStorage
-from sqlalchemy.orm.session import Session
+import googleapiclient.discovery
+import requests
+from cidc_schemas.prism.constants import ASSAY_TO_FILEPATH
+from google.api_core.client_options import ClientOptions
+from google.api_core.iam import Policy
 from google.cloud import storage, pubsub, bigquery
 from google.cloud.bigquery.enums import EntityTypes
 from google.oauth2.service_account import Credentials
-from google.api_core.iam import Policy
-from google.api_core.client_options import ClientOptions
-import googleapiclient.discovery
-import requests
+from sqlalchemy.orm.session import Session
+from werkzeug.datastructures import FileStorage
+from werkzeug.utils import secure_filename
 
-from cidc_schemas.prism.constants import ASSAY_TO_FILEPATH
 from cidc_api.config.secrets import get_secrets_manager
-
+from ..config.logging import get_logger
 from ..config.settings import (
     DEV_USE_GCS,
     GOOGLE_INTAKE_ROLE,
@@ -57,7 +59,6 @@ from ..config.settings import (
     DEV_CFUNCTIONS_SERVER,
     INACTIVE_USER_DAYS,
 )
-from ..config.logging import get_logger
 
 os.environ["TZ"] = "UTC"
 logger = get_logger(__name__)
@@ -216,6 +217,68 @@ def upload_xlsx_to_gcs(
     return final_object
 
 
+def upload_file_to_gcs(file: FileStorage, bucket_name: str, gcs_folder: str) -> str:
+    """Upload a file to the specified GCS folder and return the GCS path from the bucket."""
+    # Secure the filename and prepare file
+    filename = secure_filename(file.filename)
+    gcs_file_path = os.path.join(gcs_folder, filename)
+    binary_file = io.BytesIO(file.read())
+
+    if ENV == "dev" and not DEV_USE_GCS:
+        logger.info(f"Would've saved {gcs_file_path} to {bucket_name}")
+        return gcs_file_path
+
+    # Upload to GCS
+    blob = _get_bucket(bucket_name).blob(gcs_file_path)
+    blob.upload_from_file(binary_file, content_type=file.content_type)
+
+    return gcs_file_path
+
+
+def move_gcs_file(bucket_name: str, existing_path: str, to_folder: str, append_timestamp: bool = True) -> str:
+    """Move a file within a GCS bucket to a new folder, optionally appending a timestamp to the filename."""
+    filename = os.path.basename(existing_path)
+    if append_timestamp:
+        filename = _append_iso_timestamp_to_filename(filename)
+    # Ensure trailing slash on folder
+    if not to_folder.endswith("/"):
+        to_folder += "/"
+    new_gcs_file_path = f"{to_folder}{filename}"
+
+    if ENV == "dev" and not DEV_USE_GCS:
+        logger.info(f"Would've moved {existing_path} to {new_gcs_file_path} in {bucket_name}")
+        return new_gcs_file_path
+
+    bucket = _get_bucket(bucket_name)
+    source_blob = bucket.blob(existing_path)
+    if not source_blob.exists():
+        raise Exception("Expected file not found in GCS")
+    new_blob = bucket.blob(new_gcs_file_path)
+    # GCS move = rewrite + delete
+    new_blob.rewrite(source_blob)
+    source_blob.delete()
+
+    return new_gcs_file_path
+
+
+def delete_items_from_folder(bucket_name: str, folder: str):
+    """Deletes all blobs from the specified folder in the specified bucket."""
+    if ENV == "dev" and not DEV_USE_GCS:
+        logger.info(f"Would've deleted file(s) from {folder} in {bucket_name}")
+        return
+    bucket = _get_bucket(bucket_name)
+    existing_blobs = bucket.list_blobs(prefix=folder)
+    for blob in existing_blobs:
+        blob.delete()
+
+
+def _append_iso_timestamp_to_filename(filename: str) -> str:
+    """Append an ISO 8601 timestamp to a filename, preserving its extension."""
+    base, ext = os.path.splitext(filename)
+    timestamp = datetime.datetime.now().isoformat(timespec="milliseconds").replace(":", "-")
+    return f"{base}_{timestamp}{ext}"
+
+
 def grant_lister_access(user_email: str) -> None:
     """
     Grant a user list access to the GOOGLE_ACL_DATA_BUCKET. List access is

{nci_cidc_api_modules-1.1.30.dist-info → nci_cidc_api_modules-1.1.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.30
+Version: 1.1.32
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -25,10 +25,10 @@ Requires-Dist: pyarrow==14.0.1
 Requires-Dist: numpy<2,>=1.16.5
 Requires-Dist: pandas==1.5.3
 Requires-Dist: python-dotenv==0.10.3
-Requires-Dist: requests==2.32.3
+Requires-Dist: requests==2.32.4
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: certifi==2024.7.4
-Requires-Dist: nci-cidc-schemas==0.27.20
+Requires-Dist: nci-cidc-schemas==0.27.22
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

{nci_cidc_api_modules-1.1.30.dist-info → nci_cidc_api_modules-1.1.32.dist-info}/RECORD

@@ -2,10 +2,10 @@ cidc_api/config/__init__.py,sha256=5mX8GAPxUKV84iS-aGOoE-4m68LsOCGCDptXNdlgvj0,1
 cidc_api/config/db.py,sha256=cyWhWtmXha4OsrwUf6ez8aKSfm7tPSmPDE9JVSBx3Fk,1935
 cidc_api/config/logging.py,sha256=abhVYtn8lfhIt0tyV2WHFgSmp_s2eeJh7kodB6LH4J0,1149
 cidc_api/config/secrets.py,sha256=jRFj7W43pWuPf9DZQLCKF7WPXf5cUv-BAaS3ASqhV_Q,1481
-cidc_api/config/settings.py,sha256=fJQIaCfxsuooEi1pAO8FhHurN0BjP6FZKX8jl7uHGZM,4203
+cidc_api/config/settings.py,sha256=mA-4r7oB60uFepYtl5abbPigjwX8aBz__qCJXdcWWbs,4272
 cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
 cidc_api/models/migrations.py,sha256=gp9vtkYbA9FFy2s-7woelAmsvQbJ41LO2_DY-YkFIrQ,11464
-cidc_api/models/models.py,sha256=HBXb5228CeUInaaKOXYBcPz-T9pfwULz_7BaSyJmNDI,129427
+cidc_api/models/models.py,sha256=WU9nCAyYJEi1_TWyjSCrJ8XX1btPNa2xO7tpDWGccdU,131992
 cidc_api/models/schemas.py,sha256=7tDYtmULuzTt2kg7RorWhte06ffalgpQKrFiDRGcPEQ,2711
 cidc_api/models/files/__init__.py,sha256=8BMTnUSHzUbz0lBeEQY6NvApxDD3GMWMduoVMos2g4Y,213
 cidc_api/models/files/details.py,sha256=WrWPxJqlsteinoNbGTaQ3fcxgvChqLGJ9vY7H829jtk,62842
@@ -13,11 +13,11 @@ cidc_api/models/files/facets.py,sha256=JqCmwcjYYSz7XK4bAokSE9i71C8t9EQ4Jtbv7npth
 cidc_api/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cidc_api/shared/auth.py,sha256=PHqmVGkqDjbmUofytVFwD_9ssgUomESl3fFtFHPwZYQ,9062
 cidc_api/shared/emails.py,sha256=GY-l0EkoVU_3hjV0g-xo7N9d1iyCdluyq_arftEPPe0,4989
-cidc_api/shared/gcloud_client.py,sha256=i4ZZLoDC_pEwKaMS8218uUJ0fsIi0DKwd-hzGHGQw7g,33139
+cidc_api/shared/gcloud_client.py,sha256=oD7Y3Glp1ZrPYkonv7DvX1koGtF30lgm3ENXYQ7G5cI,35634
 cidc_api/shared/jose.py,sha256=-qzGzEDAlokEp9E7WtBtQkXyyfPWTYXlwYpCqVJWmqM,1830
 cidc_api/shared/rest_utils.py,sha256=RwR30WOUAYCxL7V-i2totEyeriG30GbBDvBcpLXhM9w,6594
-nci_cidc_api_modules-1.1.30.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
-nci_cidc_api_modules-1.1.30.dist-info/METADATA,sha256=s378K4KxeTTwVZuypJg2yiWUSO3jBN9zqeMwsuQ06sU,41285
-nci_cidc_api_modules-1.1.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nci_cidc_api_modules-1.1.30.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
-nci_cidc_api_modules-1.1.30.dist-info/RECORD,,
+nci_cidc_api_modules-1.1.32.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
+nci_cidc_api_modules-1.1.32.dist-info/METADATA,sha256=K5SKeS-yZim2JWbhfsbrFo7G2vSegE4uYk5xy3lgtIA,41285
+nci_cidc_api_modules-1.1.32.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nci_cidc_api_modules-1.1.32.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
+nci_cidc_api_modules-1.1.32.dist-info/RECORD,,