nci-cidc-api-modules 1.1.29__tar.gz → 1.1.31__tar.gz

{nci_cidc_api_modules-1.1.29/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.29
+Version: 1.1.31
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -28,7 +28,7 @@ Requires-Dist: python-dotenv==0.10.3
 Requires-Dist: requests==2.32.3
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: certifi==2024.7.4
-Requires-Dist: nci-cidc-schemas==0.27.19
+Requires-Dist: nci-cidc-schemas==0.27.21
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/cidc_api/config/settings.py

@@ -70,6 +70,7 @@ GOOGLE_INTAKE_BUCKET = environ["GOOGLE_INTAKE_BUCKET"]
 GOOGLE_UPLOAD_BUCKET = environ["GOOGLE_UPLOAD_BUCKET"]
 GOOGLE_UPLOAD_TOPIC = environ["GOOGLE_UPLOAD_TOPIC"]
 GOOGLE_ACL_DATA_BUCKET = environ["GOOGLE_ACL_DATA_BUCKET"]
+GOOGLE_CLINICAL_DATA_BUCKET = environ["GOOGLE_CLINICAL_DATA_BUCKET"]
 GOOGLE_EPHEMERAL_BUCKET = environ["GOOGLE_EPHEMERAL_BUCKET"]
 GOOGLE_UPLOAD_ROLE = environ["GOOGLE_UPLOAD_ROLE"]
 GOOGLE_LISTER_ROLE = environ["GOOGLE_LISTER_ROLE"]

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/cidc_api/models/models.py

@@ -22,6 +22,7 @@ __all__ = [
     "Users",
     "ValidationMultiError",
     "with_default_session",
+    "PreprocessedFiles",
 ]
 
 import hashlib
@@ -3184,3 +3185,70 @@ def upload_manifest_json(
     # Publish that a manifest upload has been received
     publish_patient_sample_update(manifest_upload.id)
     return manifest_upload.id
+
+
+class PreprocessedFiles(CommonColumns):
+    __tablename__ = "preprocessed_files"
+
+    file_name = Column(String)
+    object_url = Column(String)
+    trial_id = Column(String)
+    file_category = Column(String)
+    uploader_email = Column(String)
+    status = Column(String)
+    version = Column(Integer)
+    released_version = Column(String)
+
+    @staticmethod
+    @with_default_session
+    def create(
+        file_name: str,
+        object_url: str,
+        file_category: str,
+        uploader_email: str,
+        status: str = "pending",
+        trial_id: str = None,
+        version: int = None,
+        released_version: str = None,
+        session: Session = None,
+    ):
+        """Create and insert a new PreprocessedFiles record."""
+        new_file = PreprocessedFiles(
+            file_name=file_name,
+            object_url=object_url,
+            file_category=file_category,
+            uploader_email=uploader_email,
+            status=status,
+            trial_id=trial_id,
+            version=version,
+            released_version=released_version,
+        )
+        new_file.insert(session=session)
+        return new_file
+
+    @classmethod
+    @with_default_session
+    def archive_current_files(cls, file_category: str, session: Session = None):
+        """Update any 'current' files in the given category to 'archived'."""
+        current_files = cls.get_files_by_category_and_status(file_category, "current", session=session)
+        for file in current_files:
+            file.status = "archived"
+            file._updated = datetime.now()
+        session.commit()
+
+    @classmethod
+    @with_default_session
+    def delete_pending_files_by_category(cls, file_category: str, session: Session = None):
+        """Delete all pending files matching given file_category."""
+        records = cls.get_files_by_category_and_status(file_category, "pending", session=session)
+        for record in records:
+            session.delete(record)
+        session.commit()
+
+    @classmethod
+    @with_default_session
+    def get_files_by_category_and_status(
+        cls, file_category: str, status: str, session: Session = None
+    ) -> list["PreprocessedFiles"]:
+        """Return all files matching given file_category and status."""
+        return session.query(cls).filter_by(file_category=file_category, status=status).all()

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/cidc_api/shared/gcloud_client.py

@@ -2,15 +2,16 @@
 
 # pylint: disable=logging-fstring-interpolation
 
-import json
-import os
-from os import environ
 import base64
 import datetime
-import warnings
 import hashlib
+import io
+import json
+import os
+import warnings
 from collections import namedtuple
 from concurrent.futures import Future
+from os import environ
 from typing import (
     Any,
     BinaryIO,
@@ -23,19 +24,20 @@ from typing import (
     Union,
 )
 
-from werkzeug.datastructures import FileStorage
-from sqlalchemy.orm.session import Session
+import googleapiclient.discovery
+import requests
+from cidc_schemas.prism.constants import ASSAY_TO_FILEPATH
+from google.api_core.client_options import ClientOptions
+from google.api_core.iam import Policy
 from google.cloud import storage, pubsub, bigquery
 from google.cloud.bigquery.enums import EntityTypes
 from google.oauth2.service_account import Credentials
-from google.api_core.iam import Policy
-from google.api_core.client_options import ClientOptions
-import googleapiclient.discovery
-import requests
+from sqlalchemy.orm.session import Session
+from werkzeug.datastructures import FileStorage
+from werkzeug.utils import secure_filename
 
-from cidc_schemas.prism.constants import ASSAY_TO_FILEPATH
 from cidc_api.config.secrets import get_secrets_manager
-
+from ..config.logging import get_logger
 from ..config.settings import (
     DEV_USE_GCS,
     GOOGLE_INTAKE_ROLE,
@@ -57,7 +59,6 @@ from ..config.settings import (
     DEV_CFUNCTIONS_SERVER,
     INACTIVE_USER_DAYS,
 )
-from ..config.logging import get_logger
 
 os.environ["TZ"] = "UTC"
 logger = get_logger(__name__)
@@ -216,6 +217,68 @@ def upload_xlsx_to_gcs(
     return final_object
 
 
+def upload_file_to_gcs(file: FileStorage, bucket_name: str, gcs_folder: str) -> str:
+    """Upload a file to the specified GCS folder and return the GCS path from the bucket."""
+    # Secure the filename and prepare file
+    filename = secure_filename(file.filename)
+    gcs_file_path = os.path.join(gcs_folder, filename)
+    binary_file = io.BytesIO(file.read())
+
+    if ENV == "dev" and not DEV_USE_GCS:
+        logger.info(f"Would've saved {gcs_file_path} to {bucket_name}")
+        return gcs_file_path
+
+    # Upload to GCS
+    blob = _get_bucket(bucket_name).blob(gcs_file_path)
+    blob.upload_from_file(binary_file, content_type=file.content_type)
+
+    return gcs_file_path
+
+
+def move_gcs_file(bucket_name: str, existing_path: str, to_folder: str, append_timestamp: bool = True) -> str:
+    """Move a file within a GCS bucket to a new folder, optionally appending a timestamp to the filename."""
+    bucket = _get_bucket(bucket_name)
+    filename = os.path.basename(existing_path)
+    if append_timestamp:
+        filename = _append_iso_timestamp_to_filename(filename)
+    # Ensure trailing slash on folder
+    if not to_folder.endswith("/"):
+        to_folder += "/"
+    new_gcs_file_path = f"{to_folder}{filename}"
+
+    if ENV == "dev" and not DEV_USE_GCS:
+        logger.info(f"Would've moved {existing_path} to {new_gcs_file_path} in {bucket_name}")
+        return new_gcs_file_path
+
+    source_blob = bucket.blob(existing_path)
+    if not source_blob.exists():
+        raise Exception("Expected file not found in GCS")
+    new_blob = bucket.blob(new_gcs_file_path)
+    # GCS move = rewrite + delete
+    new_blob.rewrite(source_blob)
+    source_blob.delete()
+
+    return new_gcs_file_path
+
+
+def delete_items_from_folder(bucket_name: str, folder: str):
+    """Deletes all blobs from the specified folder in the specified bucket."""
+    bucket = _get_bucket(bucket_name)
+    if ENV == "dev" and not DEV_USE_GCS:
+        logger.info(f"Would've deleted file(s) from {folder} in {bucket_name}")
+        return
+    existing_blobs = bucket.list_blobs(prefix=folder)
+    for blob in existing_blobs:
+        blob.delete()
+
+
+def _append_iso_timestamp_to_filename(filename: str) -> str:
+    """Append an ISO 8601 timestamp to a filename, preserving its extension."""
+    base, ext = os.path.splitext(filename)
+    timestamp = datetime.datetime.now().isoformat(timespec="milliseconds").replace(":", "-")
+    return f"{base}_{timestamp}{ext}"
+
+
 def grant_lister_access(user_email: str) -> None:
     """
     Grant a user list access to the GOOGLE_ACL_DATA_BUCKET. List access is

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.29
+Version: 1.1.31
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -28,7 +28,7 @@ Requires-Dist: python-dotenv==0.10.3
 Requires-Dist: requests==2.32.3
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: certifi==2024.7.4
-Requires-Dist: nci-cidc-schemas==0.27.19
+Requires-Dist: nci-cidc-schemas==0.27.21
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/nci_cidc_api_modules.egg-info/SOURCES.txt

@@ -9,10 +9,7 @@ cidc_api/config/db.py
 cidc_api/config/logging.py
 cidc_api/config/secrets.py
 cidc_api/config/settings.py
-cidc_api/csms/__init__.py
-cidc_api/csms/auth.py
 cidc_api/models/__init__.py
-cidc_api/models/csms_api.py
 cidc_api/models/migrations.py
 cidc_api/models/models.py
 cidc_api/models/schemas.py

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/nci_cidc_api_modules.egg-info/requires.txt

@@ -19,4 +19,4 @@ python-dotenv==0.10.3
 requests==2.32.3
 jinja2==3.1.6
 certifi==2024.7.4
-nci-cidc-schemas==0.27.19
+nci-cidc-schemas==0.27.21

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/requirements.modules.txt

@@ -19,4 +19,4 @@ python-dotenv==0.10.3
 requests==2.32.3
 jinja2==3.1.6
 certifi==2024.7.4
-nci-cidc-schemas==0.27.19
+nci-cidc-schemas==0.27.21

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/setup.py

@@ -18,7 +18,6 @@ setup(
     license="MIT license",
     packages=[
         "cidc_api.config",
-        "cidc_api.csms",
         "cidc_api.models",
         "cidc_api.shared",
         "cidc_api.models.files",

{nci_cidc_api_modules-1.1.29 → nci_cidc_api_modules-1.1.31}/tests/test_api.py

@@ -410,7 +410,8 @@ def test_endpoint_urls(cidc_api):
     """
     expected_endpoints = {
         "/",
-        "/admin/test_csms",
+        "/clinical_data/files/master_appendix_a/pending",
+        "/clinical_data/jobs",
         "/downloadable_files/",
         "/downloadable_files/filelist",
         "/downloadable_files/compressed_batch",
@@ -437,7 +438,6 @@ def test_endpoint_urls(cidc_api):
         "/permissions/<int:permission>",
         "/samples/",
         "/trial_metadata/",
-        "/trial_metadata/new_manifest",
         "/trial_metadata/summaries",
         "/trial_metadata/<string:trial>",
         "/trial_metadata/<string:trial>/refresh_trial",

nci_cidc_api_modules-1.1.29/cidc_api/csms/__init__.py

@@ -1 +0,0 @@
-from .auth import *

nci_cidc_api_modules-1.1.29/cidc_api/csms/auth.py

@@ -1,105 +0,0 @@
-__all__ = ["get_token", "get_with_authorization", "get_with_paging"]
-
-import os
-from datetime import datetime, timedelta
-from typing import Any, Dict, Iterator
-
-import requests
-
-from ..config.settings import (
-    CSMS_BASE_URL,
-    CSMS_CLIENT_ID,
-    CSMS_CLIENT_SECRET,
-    CSMS_TOKEN_URL,
-)
-
-os.environ["TZ"] = "UTC"
-
-TIMEOUT_IN_SECONDS = 20
-_TOKEN, _TOKEN_EXPIRY = None, datetime.now()
-
-
-def get_token():
-    global _TOKEN, _TOKEN_EXPIRY
-    if not _TOKEN or datetime.now() >= _TOKEN_EXPIRY:
-        res, time = (
-            requests.post(
-                CSMS_TOKEN_URL,
-                headers={"Content-Type": "application/x-www-form-urlencoded"},
-                data={
-                    "grant_type": "client_credentials",
-                    "client_id": CSMS_CLIENT_ID,
-                    "client_secret": CSMS_CLIENT_SECRET,
-                },
-                timeout=TIMEOUT_IN_SECONDS,
-            ).json(),
-            datetime.now(),
-        )
-
-        # res definition from https://developer.okta.com/docs/reference/api/oidc/#response-example-error-7
-        if "errorCode" in res:
-            raise RuntimeError(res["errorCode"] + ": " + res.get("errorSummary"))
-
-        _TOKEN = res["access_token"]
-        _TOKEN_EXPIRY = time + timedelta(seconds=res["expires_in"])
-
-    return _TOKEN
-
-
-def get_with_authorization(url: str, **kwargs) -> requests.Response:
-    """url should be fully valid or begin with `/` to be prefixed with CSMS_BASE_URL"""
-    token = get_token()
-    headers = {
-        **kwargs.get("headers", {}),
-        "Authorization": f"Bearer {token}",
-        "accept": "*/*",
-    }
-    kwargs["headers"] = headers
-    if not url.startswith(CSMS_BASE_URL):
-        url = CSMS_BASE_URL + url
-    return requests.get(
-        url,
-        **kwargs,
-        timeout=TIMEOUT_IN_SECONDS,
-    )
-
-
-def get_with_paging(url: str, limit: int = None, offset: int = 0, **kwargs) -> Iterator[Dict[str, Any]]:
-    """
-    Return an iterator of entries via get_with_authorization with handling for CSMS paging
-
-    Parameters
-    ----------
-    url: str
-        url should be fully valid or begin with `/` to be prefixed with CSMS_BASE_URL
-    limit: int = None
-        the number of records to return on each page
-        default: 5000 for samples, 50 for manifests, 1 otherwise
-    offset: int = 0
-        which page to return, 0-indexed
-        increments as needed to continue returning
-
-    Raises
-    ------
-    requests.exceptions.HTTPError
-        via res.raise_for_status()
-        https://docs.python-requests.org/en/master/user/quickstart/#response-status-codes
-    """
-    if not limit:
-        if "samples" in url:
-            limit = 5000
-        elif "manifests" in url:
-            limit = 50
-        else:
-            limit = 1
-
-    kwargs.update({"limit": limit, "offset": offset})
-
-    res = get_with_authorization(url, params=kwargs)
-    while res.status_code < 300 and len(res.json().get("data", [])) > 0:
-        # if there's not an error and we're still returning
-        yield from res.json()["data"]
-        kwargs["offset"] += 1  # get the next page
-        res = get_with_authorization(url, params=kwargs)
-
-    res.raise_for_status()