nci-cidc-api-modules 1.2.34__tar.gz → 1.2.36__tar.gz

{nci_cidc_api_modules-1.2.34/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.34
+Version: 1.2.36
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -33,6 +33,14 @@ Requires-Dist: requests>=2.32.5
 Requires-Dist: sqlalchemy>=2.0.44
 Requires-Dist: sqlalchemy-mixins~=2.0.5
 Requires-Dist: werkzeug>=3.1.4
+Requires-Dist: opentelemetry-api>=1.38.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.38.0
+Requires-Dist: opentelemetry-sdk>=1.38.0
+Requires-Dist: opentelemetry-instrumentation-flask>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
+Requires-Dist: opentelemetry-exporter-gcp-trace>=1.11.0
+Requires-Dist: opentelemetry-propagator-gcp>=1.11.0
 Requires-Dist: nci-cidc-schemas==0.28.10
 Dynamic: description
 Dynamic: description-content-type

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36}/cidc_api/models/models.py

@@ -35,7 +35,8 @@ __all__ = [
     "ADMIN_FILE_CATEGORIES",
     "FINAL_JOB_STATUS",
     "INGESTION_JOB_STATUSES",
-    "INGESTION_JOB_COLORS",
+    "ASSAY_JOB_COLORS",
+    "CLINICAL_JOB_COLORS",
 ]
 
 import hashlib
@@ -3436,7 +3437,7 @@ INGESTION_JOB_STATUSES = [
 ]
 
 # Business decision to pass hex codes from the backend though that should be done by the front end...
-INGESTION_JOB_COLORS = {
+CLINICAL_JOB_COLORS = {
     "DRAFT": "",
     "INITIAL SUBMISSION": "#ACCAD7",
     "VALIDATION REVIEW": "#DABE90",
@@ -3444,6 +3445,13 @@ INGESTION_JOB_COLORS = {
     "INGESTION": "#8FCEC7",
     "PUBLISHED": "#90D9E6",
 }
+ASSAY_JOB_COLORS = {
+    "INITIAL SUBMISSION": "#43807E",
+    "VALIDATION REVIEW": "#906F3F",
+    "REVISION SUBMISSION": "#95358A",
+    "INGESTION": "#542C88",
+    "PUBLISHED": "#1C81A0",
+}
 # TODO If have "CANCELLED" concept or other final status, add here
 FINAL_JOB_STATUS = ["PUBLISHED"]
 TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER = "Data Category"
@@ -3465,11 +3473,41 @@ class IngestionJobs(CommonColumns):
     pending = Column(Boolean, nullable=False, default=False)
     start_date = Column(DateTime, nullable=True)
     error_status = Column(String, nullable=True)
+    job_type = Column(String, nullable=False, default="clinical")
+    assay_type = Column(String, nullable=True)
+    batch_id = Column(String, nullable=True)
+    submission_id = Column(String, nullable=True)
+    intake_path = Column(String, nullable=True)
 
     @staticmethod
     @with_default_session
-    def create(trial_id: str, status: str, version: int, pending: Boolean = False, session: Session = None):
-        new_job = IngestionJobs(trial_id=trial_id, status=status, version=version, pending=pending)
+    def create(
+        trial_id: str,
+        status: str,
+        version: int,
+        error_status: str = None,
+        pending: Boolean = False,
+        job_type: str = "clinical",
+        assay_type: str = None,
+        batch_id: str = None,
+        submission_id: str = None,
+        intake_path: str = None,
+        start_date: datetime = None,
+        session: Session = None,
+    ):
+        new_job = IngestionJobs(
+            trial_id=trial_id,
+            status=status,
+            error_status=error_status,
+            version=version,
+            pending=pending,
+            job_type=job_type,
+            assay_type=assay_type,
+            batch_id=batch_id,
+            submission_id=submission_id,
+            intake_path=intake_path,
+            start_date=start_date,
+        )
         new_job.insert(session=session)
         return new_job
 
@@ -3494,29 +3532,43 @@ class IngestionJobs(CommonColumns):
 
     @classmethod
     @with_default_session
-    def get_jobs_by_trial(cls, trial_id: str, session: Session = None) -> list["IngestionJobs"]:
-        return session.query(cls).filter(cls.trial_id == trial_id).order_by(cls.version.desc()).all()
+    def get_jobs_by_trial(
+        cls, trial_id: str, job_type: str = "clinical", session: Session = None
+    ) -> list["IngestionJobs"]:
+        return (
+            session.query(cls)
+            .filter(cls.trial_id == trial_id, cls.job_type == job_type)
+            .order_by(cls.version.desc())
+            .all()
+        )
 
     @classmethod
     @with_default_session
-    def get_open_job_by_trial(cls, trial_id: str, session: Session = None) -> Optional["IngestionJobs"]:
+    def get_open_job_by_trial(
+        cls, trial_id: str, job_type: str = "clinical", session: Session = None
+    ) -> Optional["IngestionJobs"]:
         """Return the open job for a given trial if it exists."""
         return (
             session.query(cls)
             .filter(
                 cls.trial_id == trial_id,
+                cls.job_type == job_type,
                 cls.status.notin_(FINAL_JOB_STATUS),
             )
             .order_by(cls._created.desc())
             .first()
         )
 
+    @classmethod
+    def get_jobs_for_user(cls, user: Users, job_type: str = None) -> list["IngestionJobs"]:
+        return cls.get_assay_jobs_for_user(user) if job_type == "assay" else cls.get_clinical_jobs_for_user(user)
+
     @classmethod
     @with_default_session
-    def get_open_jobs_for_user(cls, user: Users, session: Session = None) -> list["IngestionJobs"]:
+    def get_clinical_jobs_for_user(cls, user: Users, session: Session = None) -> list["IngestionJobs"]:
         if user.role not in [CIDCRole.ADMIN.value, CIDCRole.CLINICAL_TRIAL_USER.value]:
             return []
-        job_query = session.query(cls).filter(cls.status.notin_(["DRAFT"]))
+        job_query = session.query(cls).filter(cls.status.notin_(["DRAFT"]), cls.job_type == "clinical")
         if (
             user.role != CIDCRole.ADMIN.value
             and not session.query(Permissions)
@@ -3539,6 +3591,81 @@ class IngestionJobs(CommonColumns):
             job_query = job_query.filter(cls.trial_id.in_(map(lambda x: x.trial_id, authorized_trials)))
         return job_query.order_by(cls._created.desc()).all()
 
+    @classmethod
+    @with_default_session
+    def get_assay_jobs_for_user(cls, user: Users, session: Session = None) -> list["IngestionJobs"]:
+        # TODO allow more than just Admin role and get authorized trials based on permissions
+        if user.role not in [CIDCRole.ADMIN.value]:
+            return []
+        return session.query(cls).filter(cls.job_type == "assay").order_by(cls._created.desc()).all()
+
+    @classmethod
+    @with_default_session
+    def get_unique_assay_job(
+        cls,
+        trial_id: str,
+        assay_type: str,
+        batch_id: str,
+        session: Session = None,
+    ) -> Optional["IngestionJobs"]:
+        """Look for unique assay job with matching trial_id/assay_type/batch_id combination."""
+        return (
+            session.query(cls)
+            .filter(
+                cls.job_type == "assay",
+                cls.trial_id == trial_id,
+                cls.assay_type == assay_type,
+                cls.batch_id == batch_id,
+            )
+            .first()
+        )
+
+    @classmethod
+    @with_default_session
+    def next_assay_submission_id(cls, trial_id: str, assay_type: str, session: Session = None) -> str:
+        """
+        Generate the next CIDC Submission ID for an assay job.
+
+        Format:
+          <trial_id>-<assay_type>-<yyyymmdd>        (first submission of the day)
+          <trial_id>-<assay_type>-<yyyymmdd>-<#>    (subsequent submissions on same day)
+
+        Uses only the most recent matching submission_id to determine the next suffix.
+        """
+        today_str = datetime.now().strftime("%Y%m%d")
+        base_submission_id = f"{trial_id}-{assay_type}-{today_str}"
+
+        # Get the most recent submission_id matching this prefix
+        latest = (
+            session.query(cls.submission_id)
+            .filter(
+                cls.trial_id == trial_id,
+                cls.assay_type == assay_type,
+                cls.submission_id.like(f"{base_submission_id}%"),
+            )
+            .order_by(cls._created.desc())
+            .first()
+        )
+
+        # No existing submission for this prefix -> start at 1
+        if not latest or not latest[0]:
+            return base_submission_id
+
+        last_id = latest[0]
+        # Case 1: the latest is exactly the prefix (i.e., first submission today)
+        if last_id == base_submission_id:
+            return f"{base_submission_id}-2"
+
+        # Case 2: latest already has a suffix
+        try:
+            _, last_suffix = last_id.rsplit("-", 1)
+            n = int(last_suffix)
+            return f"{base_submission_id}-{n + 1}"
+        except Exception as e:
+            # If malformed, restart numbering for safety
+            logger.error("Unexpected error parsing Submission ID in next_assay_submission_id: %s", e)
+            return f"{base_submission_id}-2"
+
 
 class JobFileCategories(CommonColumns):
     __tablename__ = "job_file_categories"
@@ -3613,6 +3740,7 @@ class CategoryDataElements(CommonColumns):
     name = Column(String, nullable=False)
     is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
     element_type = Column(String, nullable=False)
+    data_type = Column(String, nullable=True)
     cardinality = Column(String, nullable=True)
 
     @classmethod

nci_cidc_api_modules-1.2.36/cidc_api/shared/assay_handling.py

@@ -0,0 +1,68 @@
+from datetime import datetime
+from urllib.parse import quote
+
+from werkzeug.exceptions import BadRequest
+
+from cidc_api.models import IngestionJobs
+from . import gcloud_client
+from ..shared.auth import get_current_user
+
+JOB_TYPE_ASSAY = "assay"
+JOB_TYPE_CLINICAL = "clinical"
+ALLOWED_JOB_TYPES = {JOB_TYPE_CLINICAL, JOB_TYPE_ASSAY}
+
+
+def resolve_job_type_and_assay_fields(data: dict) -> tuple[str, str | None, str | None]:
+    """Decide job_type and gather assay_type/batch_id from request JSON."""
+    assay_type = data.get("assay_type")
+    # If job_type is assay or assay_type is present, treat this as an assay job.
+    job_type = data.get("job_type") or (JOB_TYPE_ASSAY if assay_type else JOB_TYPE_CLINICAL)
+
+    if job_type not in ALLOWED_JOB_TYPES:
+        raise BadRequest("Invalid job_type. Allowed values are 'clinical' or 'assay'.")
+
+    if job_type == JOB_TYPE_ASSAY and (not assay_type or not isinstance(assay_type, str)):
+        raise BadRequest("assay_type must be provided for job_type='assay'.")
+
+    assay_type = assay_type.strip() if assay_type else None
+    batch_id = data.get("batch_id").strip() if isinstance(data.get("batch_id"), str) else None
+
+    return job_type, assay_type, batch_id
+
+
+def prepare_assay_job(trial_id: str, assay_type: str, batch_id: str) -> tuple[str, str, str, datetime, int, str]:
+    """
+    Validate assay job uniqueness and generate submission_id, start_date, version, and the trial’s GCS intake path.
+    """
+    if not assay_type:
+        raise BadRequest("assay_type must be provided for job_type='assay'.")
+
+    # Enforce uniqueness of (trial_id, assay_type, batch_id) when batch_id is present.
+    if batch_id:
+        existing_job = IngestionJobs.get_unique_assay_job(trial_id, assay_type, batch_id)
+        if existing_job:
+            raise BadRequest(
+                f"Assay job {existing_job.id} already exists for this exact trial_id/assay_type/batch_id combination."
+            )
+
+    submission_id = IngestionJobs.next_assay_submission_id(trial_id, assay_type)
+    job_status = "INITIAL SUBMISSION"
+    error_status = "Upload Incomplete"  # job starts with 'Incomplete' notifier
+    start_date = datetime.now()
+    version = 1
+
+    # Create or retrieve intake bucket corresponding to the trial
+    intake_bucket = gcloud_client.create_intake_bucket(get_current_user().email, trial_id=trial_id)
+    gcs_path = f"{intake_bucket.name}/{assay_type}/{submission_id}"
+
+    return submission_id, job_status, error_status, start_date, version, gcs_path
+
+
+def get_google_links(intake_path: str) -> tuple[str, str]:
+    """Build the GCS URI and GCS Console URL corresponding to the intake path."""
+    gcs_uri = f"gs://{intake_path}"
+    # Encode path to ensure link opens correctly
+    encoded_path = quote(intake_path)
+    console_url = f"https://console.cloud.google.com/storage/browser/{encoded_path}"
+
+    return gcs_uri, console_url

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36}/cidc_api/shared/auth.py

@@ -1,16 +1,14 @@
 from functools import wraps
 from typing import List
 
-from packaging import version
-
 from flask import g, request, current_app as app, Flask
+from packaging import version
 from werkzeug.exceptions import Unauthorized, BadRequest, PreconditionFailed
 
-from ..models import Users, UserSchema
-
 from ..config.logging import get_logger
-
+from ..models import Users, UserSchema
 from ..shared.jose import decode_id_token
+from ..telemetry import trace_
 
 logger = get_logger(__name__)
 
@@ -144,6 +142,7 @@ def get_current_user() -> Users:
 _user_schema = UserSchema()
 
 
+@trace_()
 def authenticate() -> Users:
     id_token = _extract_token()
     token_payload = decode_id_token(id_token)
@@ -172,6 +171,7 @@ def _extract_token() -> str:
 
 
 ### Authorization logic ###
+@trace_()
 def authorize(user: Users, allowed_roles: List[str], resource: str, method: str) -> bool:
     """Check if the current user is authorized to act on the current request's resource.
     Raises Unauthorized

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36}/cidc_api/shared/file_handling.py

@@ -10,10 +10,12 @@ from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
 from ..models import PreprocessedFiles, TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER
 from ..shared.auth import get_current_user
 from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
+from ..telemetry import trace_
 
 logger = get_logger(__name__)
 
 
+@trace_()
 def set_current_file(
     file: FileStorage, file_category: str, gcs_folder: str, session: Session, uploader_email: str, job_id: int = None
 ) -> PreprocessedFiles:
@@ -26,6 +28,7 @@ def set_current_file(
     return latest_file
 
 
+@trace_()
 def create_file(
     file: FileStorage,
     gcs_folder: str,

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36}/cidc_api/shared/gcloud_client.py

@@ -1,6 +1,6 @@
 """Utilities for interacting with the Google Cloud Platform APIs."""
 
-# pylint: disable=logging-fstring-interpolation,too-many-lines
+# pylint: disable=logging-fstring-interpolation,too-many-lines, broad-exception-raised
 
 import base64
 import datetime
@@ -8,6 +8,7 @@ import hashlib
 import io
 import json
 import os
+import re
 import warnings
 from collections import namedtuple
 from concurrent.futures import Future
@@ -361,15 +362,34 @@ def get_intake_bucket_name(user_email: str) -> str:
     return bucket_name
 
 
-def create_intake_bucket(user_email: str) -> storage.Bucket:
+def get_trial_intake_bucket_name(trial_id: str) -> str:
     """
-    Create a new data intake bucket for this user, or get the existing one.
+    Return a sanitized GCS bucket name for a given trial_id.
+
+    Produces:  <GOOGLE_INTAKE_BUCKET>-<sanitized_trial_id>
+    where the trial_id segment is lowercased and restricted to [a-z0-9-].
+    """
+    # Replace non-allowed bucket chars with "-"
+    sanitized_id = re.sub(r"[^a-z0-9-]", "-", trial_id.lower())
+    # Collapse repeated "-" and trim from both ends
+    sanitized_id = re.sub(r"-+", "-", sanitized_id).strip("-")
+
+    return f"{GOOGLE_INTAKE_BUCKET}-{sanitized_id}"
+
+
+def create_intake_bucket(user_email: str, trial_id: str = None) -> storage.Bucket:
+    """
+    Create (or retrieve) the appropriate data intake bucket.
+    If a trial_id is provided, a trial-specific bucket is used;
+    otherwise a user-specific intake bucket is used.
+
     Grant the user GCS object admin permissions on the bucket, or refresh those
     permissions if they've already been granted.
     Created with uniform bucket-level IAM access, so expiring permission.
     """
     storage_client = _get_storage_client()
-    bucket_name = get_intake_bucket_name(user_email)
+    # Get trial-specific bucket name if trial_id is given, otherwise a user-specific bucket name.
+    bucket_name = get_trial_intake_bucket_name(trial_id) if trial_id else get_intake_bucket_name(user_email)
     bucket = storage_client.bucket(bucket_name)
 
     if not bucket.exists():
@@ -614,6 +634,7 @@ def _build_trial_upload_prefixes(
     trial_set: Set[str] = set()
     upload_set: Set[str] = set()
     if not trial_id:
+        # import is here becasue of circular import
         from ..models.models import TrialMetadata
 
         trial_set = {str(t.trial_id) for t in session.query(TrialMetadata).add_columns(TrialMetadata.trial_id)}
@@ -920,7 +941,8 @@ def get_signed_url(
 
 def _encode_and_publish(content: str, topic: str) -> Future:
     """Convert `content` to bytes and publish it to `topic`."""
-    pubsub_publisher = pubsub.PublisherClient()
+    publisher_options = pubsub.types.PublisherOptions(enable_open_telemetry_tracing=ENV == "dev-int")
+    pubsub_publisher = pubsub.PublisherClient(publisher_options=publisher_options)
     topic = pubsub_publisher.topic_path(GOOGLE_CLOUD_PROJECT, topic)
     data = bytes(content, "utf-8")
 

nci_cidc_api_modules-1.2.36/cidc_api/shared/utils.py

@@ -0,0 +1,19 @@
+from cidc_api.telemetry import trace_
+
+
+def _stripper(x):
+    if x and isinstance(x, str):
+        return x.strip()
+    else:
+        return x
+
+
+@trace_("sheet")
+def strip_whitespaces(df, sheet=None):
+    if sheet:
+        df = df[sheet]
+
+    df.rename(columns=_stripper, inplace=True)
+    df = df.map(_stripper)
+
+    return df

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.34
+Version: 1.2.36
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -33,6 +33,14 @@ Requires-Dist: requests>=2.32.5
 Requires-Dist: sqlalchemy>=2.0.44
 Requires-Dist: sqlalchemy-mixins~=2.0.5
 Requires-Dist: werkzeug>=3.1.4
+Requires-Dist: opentelemetry-api>=1.38.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.38.0
+Requires-Dist: opentelemetry-sdk>=1.38.0
+Requires-Dist: opentelemetry-instrumentation-flask>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
+Requires-Dist: opentelemetry-exporter-gcp-trace>=1.11.0
+Requires-Dist: opentelemetry-propagator-gcp>=1.11.0
 Requires-Dist: nci-cidc-schemas==0.28.10
 Dynamic: description
 Dynamic: description-content-type

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36}/nci_cidc_api_modules.egg-info/SOURCES.txt

@@ -100,6 +100,7 @@ cidc_api/reference/icd10cm.py
 cidc_api/reference/icdo3.py
 cidc_api/reference/uberon.py
 cidc_api/shared/__init__.py
+cidc_api/shared/assay_handling.py
 cidc_api/shared/auth.py
 cidc_api/shared/email_layout.html
 cidc_api/shared/emails.py

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36}/nci_cidc_api_modules.egg-info/requires.txt

@@ -24,4 +24,12 @@ requests>=2.32.5
 sqlalchemy>=2.0.44
 sqlalchemy-mixins~=2.0.5
 werkzeug>=3.1.4
+opentelemetry-api>=1.38.0
+opentelemetry-exporter-otlp-proto-grpc>=1.38.0
+opentelemetry-sdk>=1.38.0
+opentelemetry-instrumentation-flask>=0.59b0
+opentelemetry-instrumentation-requests>=0.59b0
+opentelemetry-instrumentation-sqlalchemy>=0.59b0
+opentelemetry-exporter-gcp-trace>=1.11.0
+opentelemetry-propagator-gcp>=1.11.0
 nci-cidc-schemas==0.28.10

{nci_cidc_api_modules-1.2.34 → nci_cidc_api_modules-1.2.36}/requirements.modules.txt

@@ -25,5 +25,19 @@ sqlalchemy>=2.0.44
 sqlalchemy-mixins~=2.0.5
 werkzeug>=3.1.4
 
+# open telemetry sdk
+opentelemetry-api >= 1.38.0
+opentelemetry-exporter-otlp-proto-grpc >= 1.38.0
+opentelemetry-sdk >= 1.38.0
+
+# open telemetry instrumentation
+opentelemetry-instrumentation-flask >= 0.59b0
+opentelemetry-instrumentation-requests >= 0.59b0
+opentelemetry-instrumentation-sqlalchemy >= 0.59b0
+
+# open telemetry gcp
+opentelemetry-exporter-gcp-trace >= 1.11.0
+opentelemetry-propagator-gcp >= 1.11.0
+
 # cidc deps
 nci-cidc-schemas==0.28.10

nci_cidc_api_modules-1.2.34/cidc_api/shared/utils.py

@@ -1,11 +0,0 @@
-def strip_whitespaces(df):
-    def stripper(x):
-        if x and isinstance(x, str):
-            return x.strip()
-        else:
-            return x
-
-    df.rename(columns=stripper, inplace=True)
-    df = df.map(stripper)
-
-    return df