nci-cidc-api-modules 1.2.33__tar.gz → 1.2.35__tar.gz

{nci_cidc_api_modules-1.2.33/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.33
+Version: 1.2.35
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -14,24 +14,25 @@ Requires-Dist: flask-migrate>=4.1.0
 Requires-Dist: flask-sqlalchemy>=3.1.1
 Requires-Dist: flask-talisman>=0.7.0
 Requires-Dist: google-auth==2.43.0
-Requires-Dist: google-api-python-client>=2.185.0
+Requires-Dist: google-api-python-client>=2.187.0
 Requires-Dist: google-cloud-bigquery>=3.38.0
 Requires-Dist: google-cloud-pubsub>=2.33.0
 Requires-Dist: google-cloud-secret-manager>=2.25.0
 Requires-Dist: google-cloud-storage>=3.6.0
 Requires-Dist: jinja2>=3.1.6
+Requires-Dist: joserfc>=1.5.0
 Requires-Dist: marshmallow>=4.1.0
 Requires-Dist: marshmallow-sqlalchemy>=1.4.2
 Requires-Dist: numpy>=2.3.5
 Requires-Dist: packaging>=25.0
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: pyarrow>=22.0.0
-Requires-Dist: pydantic~=2.12.4
+Requires-Dist: pydantic~=2.12.5
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: requests>=2.32.5
 Requires-Dist: sqlalchemy>=2.0.44
 Requires-Dist: sqlalchemy-mixins~=2.0.5
-Requires-Dist: werkzeug>=3.1.3
+Requires-Dist: werkzeug>=3.1.4
 Requires-Dist: nci-cidc-schemas==0.28.10
 Dynamic: description
 Dynamic: description-content-type

{nci_cidc_api_modules-1.2.33 → nci_cidc_api_modules-1.2.35}/cidc_api/models/models.py

@@ -3465,11 +3465,41 @@ class IngestionJobs(CommonColumns):
     pending = Column(Boolean, nullable=False, default=False)
     start_date = Column(DateTime, nullable=True)
     error_status = Column(String, nullable=True)
+    job_type = Column(String, nullable=False, default="clinical")
+    assay_type = Column(String, nullable=True)
+    batch_id = Column(String, nullable=True)
+    submission_id = Column(String, nullable=True)
+    intake_path = Column(String, nullable=True)
 
     @staticmethod
     @with_default_session
-    def create(trial_id: str, status: str, version: int, pending: Boolean = False, session: Session = None):
-        new_job = IngestionJobs(trial_id=trial_id, status=status, version=version, pending=pending)
+    def create(
+        trial_id: str,
+        status: str,
+        version: int,
+        error_status: str = None,
+        pending: Boolean = False,
+        job_type: str = "clinical",
+        assay_type: str = None,
+        batch_id: str = None,
+        submission_id: str = None,
+        intake_path: str = None,
+        start_date: datetime = None,
+        session: Session = None,
+    ):
+        new_job = IngestionJobs(
+            trial_id=trial_id,
+            status=status,
+            error_status=error_status,
+            version=version,
+            pending=pending,
+            job_type=job_type,
+            assay_type=assay_type,
+            batch_id=batch_id,
+            submission_id=submission_id,
+            intake_path=intake_path,
+            start_date=start_date,
+        )
         new_job.insert(session=session)
         return new_job
 
@@ -3494,29 +3524,43 @@ class IngestionJobs(CommonColumns):
 
     @classmethod
     @with_default_session
-    def get_jobs_by_trial(cls, trial_id: str, session: Session = None) -> list["IngestionJobs"]:
-        return session.query(cls).filter(cls.trial_id == trial_id).order_by(cls.version.desc()).all()
+    def get_jobs_by_trial(
+        cls, trial_id: str, job_type: str = "clinical", session: Session = None
+    ) -> list["IngestionJobs"]:
+        return (
+            session.query(cls)
+            .filter(cls.trial_id == trial_id, cls.job_type == job_type)
+            .order_by(cls.version.desc())
+            .all()
+        )
 
     @classmethod
     @with_default_session
-    def get_open_job_by_trial(cls, trial_id: str, session: Session = None) -> Optional["IngestionJobs"]:
+    def get_open_job_by_trial(
+        cls, trial_id: str, job_type: str = "clinical", session: Session = None
+    ) -> Optional["IngestionJobs"]:
         """Return the open job for a given trial if it exists."""
         return (
             session.query(cls)
             .filter(
                 cls.trial_id == trial_id,
+                cls.job_type == job_type,
                 cls.status.notin_(FINAL_JOB_STATUS),
             )
             .order_by(cls._created.desc())
             .first()
         )
 
+    @classmethod
+    def get_jobs_for_user(cls, user: Users, job_type: str = None) -> list["IngestionJobs"]:
+        return cls.get_assay_jobs_for_user(user) if job_type == "assay" else cls.get_clinical_jobs_for_user(user)
+
     @classmethod
     @with_default_session
-    def get_open_jobs_for_user(cls, user: Users, session: Session = None) -> list["IngestionJobs"]:
+    def get_clinical_jobs_for_user(cls, user: Users, session: Session = None) -> list["IngestionJobs"]:
         if user.role not in [CIDCRole.ADMIN.value, CIDCRole.CLINICAL_TRIAL_USER.value]:
             return []
-        job_query = session.query(cls).filter(cls.status.notin_(["DRAFT"]))
+        job_query = session.query(cls).filter(cls.status.notin_(["DRAFT"]), cls.job_type == "clinical")
         if (
             user.role != CIDCRole.ADMIN.value
             and not session.query(Permissions)
@@ -3539,6 +3583,81 @@ class IngestionJobs(CommonColumns):
             job_query = job_query.filter(cls.trial_id.in_(map(lambda x: x.trial_id, authorized_trials)))
         return job_query.order_by(cls._created.desc()).all()
 
+    @classmethod
+    @with_default_session
+    def get_assay_jobs_for_user(cls, user: Users, session: Session = None) -> list["IngestionJobs"]:
+        # TODO allow more than just Admin role and get authorized trials based on permissions
+        if user.role not in [CIDCRole.ADMIN.value]:
+            return []
+        return session.query(cls).filter(cls.job_type == "assay").order_by(cls._created.desc()).all()
+
+    @classmethod
+    @with_default_session
+    def get_unique_assay_job(
+        cls,
+        trial_id: str,
+        assay_type: str,
+        batch_id: str,
+        session: Session = None,
+    ) -> Optional["IngestionJobs"]:
+        """Look for unique assay job with matching trial_id/assay_type/batch_id combination."""
+        return (
+            session.query(cls)
+            .filter(
+                cls.job_type == "assay",
+                cls.trial_id == trial_id,
+                cls.assay_type == assay_type,
+                cls.batch_id == batch_id,
+            )
+            .first()
+        )
+
+    @classmethod
+    @with_default_session
+    def next_assay_submission_id(cls, trial_id: str, assay_type: str, session: Session = None) -> str:
+        """
+        Generate the next CIDC Submission ID for an assay job.
+
+        Format:
+          <trial_id>-<assay_type>-<yyyymmdd>        (first submission of the day)
+          <trial_id>-<assay_type>-<yyyymmdd>-<#>    (subsequent submissions on same day)
+
+        Uses only the most recent matching submission_id to determine the next suffix.
+        """
+        today_str = datetime.now().strftime("%Y%m%d")
+        base_submission_id = f"{trial_id}-{assay_type}-{today_str}"
+
+        # Get the most recent submission_id matching this prefix
+        latest = (
+            session.query(cls.submission_id)
+            .filter(
+                cls.trial_id == trial_id,
+                cls.assay_type == assay_type,
+                cls.submission_id.like(f"{base_submission_id}%"),
+            )
+            .order_by(cls._created.desc())
+            .first()
+        )
+
+        # No existing submission for this prefix -> start at 1
+        if not latest or not latest[0]:
+            return base_submission_id
+
+        last_id = latest[0]
+        # Case 1: the latest is exactly the prefix (i.e., first submission today)
+        if last_id == base_submission_id:
+            return f"{base_submission_id}-2"
+
+        # Case 2: latest already has a suffix
+        try:
+            _, last_suffix = last_id.rsplit("-", 1)
+            n = int(last_suffix)
+            return f"{base_submission_id}-{n + 1}"
+        except Exception as e:
+            # If malformed, restart numbering for safety
+            logger.error("Unexpected error parsing Submission ID in next_assay_submission_id: %s", e)
+            return f"{base_submission_id}-2"
+
 
 class JobFileCategories(CommonColumns):
     __tablename__ = "job_file_categories"
@@ -3613,6 +3732,7 @@ class CategoryDataElements(CommonColumns):
     name = Column(String, nullable=False)
     is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
     element_type = Column(String, nullable=False)
+    data_type = Column(String, nullable=True)
     cardinality = Column(String, nullable=True)
 
     @classmethod

nci_cidc_api_modules-1.2.35/cidc_api/shared/assay_handling.py

@@ -0,0 +1,68 @@
+from datetime import datetime
+from urllib.parse import quote
+
+from werkzeug.exceptions import BadRequest
+
+from cidc_api.models import IngestionJobs
+from . import gcloud_client
+from ..shared.auth import get_current_user
+
+JOB_TYPE_ASSAY = "assay"
+JOB_TYPE_CLINICAL = "clinical"
+ALLOWED_JOB_TYPES = {JOB_TYPE_CLINICAL, JOB_TYPE_ASSAY}
+
+
+def resolve_job_type_and_assay_fields(data: dict) -> tuple[str, str | None, str | None]:
+    """Decide job_type and gather assay_type/batch_id from request JSON."""
+    assay_type = data.get("assay_type")
+    # If job_type is assay or assay_type is present, treat this as an assay job.
+    job_type = data.get("job_type") or (JOB_TYPE_ASSAY if assay_type else JOB_TYPE_CLINICAL)
+
+    if job_type not in ALLOWED_JOB_TYPES:
+        raise BadRequest("Invalid job_type. Allowed values are 'clinical' or 'assay'.")
+
+    if job_type == JOB_TYPE_ASSAY and (not assay_type or not isinstance(assay_type, str)):
+        raise BadRequest("assay_type must be provided for job_type='assay'.")
+
+    assay_type = assay_type.strip() if assay_type else None
+    batch_id = data.get("batch_id").strip() if isinstance(data.get("batch_id"), str) else None
+
+    return job_type, assay_type, batch_id
+
+
+def prepare_assay_job(trial_id: str, assay_type: str, batch_id: str) -> tuple[str, str, str, datetime, int, str]:
+    """
+    Validate assay job uniqueness and generate submission_id, start_date, version, and the trial’s GCS intake path.
+    """
+    if not assay_type:
+        raise BadRequest("assay_type must be provided for job_type='assay'.")
+
+    # Enforce uniqueness of (trial_id, assay_type, batch_id) when batch_id is present.
+    if batch_id:
+        existing_job = IngestionJobs.get_unique_assay_job(trial_id, assay_type, batch_id)
+        if existing_job:
+            raise BadRequest(
+                f"Assay job {existing_job.id} already exists for this exact trial_id/assay_type/batch_id combination."
+            )
+
+    submission_id = IngestionJobs.next_assay_submission_id(trial_id, assay_type)
+    job_status = "INITIAL SUBMISSION"
+    error_status = "Upload Incomplete"  # job starts with 'Incomplete' notifier
+    start_date = datetime.now()
+    version = 1
+
+    # Create or retrieve intake bucket corresponding to the trial
+    intake_bucket = gcloud_client.create_intake_bucket(get_current_user().email, trial_id=trial_id)
+    gcs_path = f"{intake_bucket.name}/{trial_id}/{assay_type}"
+
+    return submission_id, job_status, error_status, start_date, version, gcs_path
+
+
+def get_google_links(intake_path: str) -> tuple[str, str]:
+    """Build the GCS URI and GCS Console URL corresponding to the intake path."""
+    gcs_uri = f"gs://{intake_path}"
+    # Encode path to ensure link opens correctly
+    encoded_path = quote(intake_path)
+    console_url = f"https://console.cloud.google.com/storage/browser/{encoded_path}"
+
+    return gcs_uri, console_url

{nci_cidc_api_modules-1.2.33 → nci_cidc_api_modules-1.2.35}/cidc_api/shared/gcloud_client.py

@@ -8,6 +8,7 @@ import hashlib
 import io
 import json
 import os
+import re
 import warnings
 from collections import namedtuple
 from concurrent.futures import Future
@@ -361,15 +362,34 @@ def get_intake_bucket_name(user_email: str) -> str:
     return bucket_name
 
 
-def create_intake_bucket(user_email: str) -> storage.Bucket:
+def get_trial_intake_bucket_name(trial_id: str) -> str:
     """
-    Create a new data intake bucket for this user, or get the existing one.
+    Return a sanitized GCS bucket name for a given trial_id.
+
+    Produces:  <GOOGLE_INTAKE_BUCKET>-<sanitized_trial_id>
+    where the trial_id segment is lowercased and restricted to [a-z0-9-].
+    """
+    # Replace non-allowed bucket chars with "-"
+    sanitized_id = re.sub(r"[^a-z0-9-]", "-", trial_id.lower())
+    # Collapse repeated "-" and trim from both ends
+    sanitized_id = re.sub(r"-+", "-", sanitized_id).strip("-")
+
+    return f"{GOOGLE_INTAKE_BUCKET}-{sanitized_id}"
+
+
+def create_intake_bucket(user_email: str, trial_id: str = None) -> storage.Bucket:
+    """
+    Create (or retrieve) the appropriate data intake bucket.
+    If a trial_id is provided, a trial-specific bucket is used;
+    otherwise a user-specific intake bucket is used.
+
     Grant the user GCS object admin permissions on the bucket, or refresh those
     permissions if they've already been granted.
     Created with uniform bucket-level IAM access, so expiring permission.
     """
     storage_client = _get_storage_client()
-    bucket_name = get_intake_bucket_name(user_email)
+    # Get trial-specific bucket name if trial_id is given, otherwise a user-specific bucket name.
+    bucket_name = get_trial_intake_bucket_name(trial_id) if trial_id else get_intake_bucket_name(user_email)
     bucket = storage_client.bucket(bucket_name)
 
     if not bucket.exists():

{nci_cidc_api_modules-1.2.33 → nci_cidc_api_modules-1.2.35/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.33
+Version: 1.2.35
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -14,24 +14,25 @@ Requires-Dist: flask-migrate>=4.1.0
 Requires-Dist: flask-sqlalchemy>=3.1.1
 Requires-Dist: flask-talisman>=0.7.0
 Requires-Dist: google-auth==2.43.0
-Requires-Dist: google-api-python-client>=2.185.0
+Requires-Dist: google-api-python-client>=2.187.0
 Requires-Dist: google-cloud-bigquery>=3.38.0
 Requires-Dist: google-cloud-pubsub>=2.33.0
 Requires-Dist: google-cloud-secret-manager>=2.25.0
 Requires-Dist: google-cloud-storage>=3.6.0
 Requires-Dist: jinja2>=3.1.6
+Requires-Dist: joserfc>=1.5.0
 Requires-Dist: marshmallow>=4.1.0
 Requires-Dist: marshmallow-sqlalchemy>=1.4.2
 Requires-Dist: numpy>=2.3.5
 Requires-Dist: packaging>=25.0
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: pyarrow>=22.0.0
-Requires-Dist: pydantic~=2.12.4
+Requires-Dist: pydantic~=2.12.5
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: requests>=2.32.5
 Requires-Dist: sqlalchemy>=2.0.44
 Requires-Dist: sqlalchemy-mixins~=2.0.5
-Requires-Dist: werkzeug>=3.1.3
+Requires-Dist: werkzeug>=3.1.4
 Requires-Dist: nci-cidc-schemas==0.28.10
 Dynamic: description
 Dynamic: description-content-type

{nci_cidc_api_modules-1.2.33 → nci_cidc_api_modules-1.2.35}/nci_cidc_api_modules.egg-info/SOURCES.txt

@@ -100,6 +100,7 @@ cidc_api/reference/icd10cm.py
 cidc_api/reference/icdo3.py
 cidc_api/reference/uberon.py
 cidc_api/shared/__init__.py
+cidc_api/shared/assay_handling.py
 cidc_api/shared/auth.py
 cidc_api/shared/email_layout.html
 cidc_api/shared/emails.py

{nci_cidc_api_modules-1.2.33 → nci_cidc_api_modules-1.2.35}/nci_cidc_api_modules.egg-info/requires.txt

@@ -5,22 +5,23 @@ flask-migrate>=4.1.0
 flask-sqlalchemy>=3.1.1
 flask-talisman>=0.7.0
 google-auth==2.43.0
-google-api-python-client>=2.185.0
+google-api-python-client>=2.187.0
 google-cloud-bigquery>=3.38.0
 google-cloud-pubsub>=2.33.0
 google-cloud-secret-manager>=2.25.0
 google-cloud-storage>=3.6.0
 jinja2>=3.1.6
+joserfc>=1.5.0
 marshmallow>=4.1.0
 marshmallow-sqlalchemy>=1.4.2
 numpy>=2.3.5
 packaging>=25.0
 pandas>=2.3.3
 pyarrow>=22.0.0
-pydantic~=2.12.4
+pydantic~=2.12.5
 python-dotenv>=1.2.1
 requests>=2.32.5
 sqlalchemy>=2.0.44
 sqlalchemy-mixins~=2.0.5
-werkzeug>=3.1.3
+werkzeug>=3.1.4
 nci-cidc-schemas==0.28.10

{nci_cidc_api_modules-1.2.33 → nci_cidc_api_modules-1.2.35}/requirements.modules.txt

@@ -4,25 +4,26 @@ flask>=3.1.2
 flask-migrate>=4.1.0
 flask-sqlalchemy>=3.1.1
 flask-talisman>=0.7.0
-google-auth==2.43.0 # There is a bug in 2.42.X that causes local to fail when connecting to dev, if still there downgrade to 2.41.1
-google-api-python-client>=2.185.0
+google-auth==2.43.0
+google-api-python-client>=2.187.0
 google-cloud-bigquery>=3.38.0
 google-cloud-pubsub>=2.33.0
 google-cloud-secret-manager>=2.25.0
 google-cloud-storage>=3.6.0
 jinja2>=3.1.6
+joserfc>=1.5.0
 marshmallow>=4.1.0
 marshmallow-sqlalchemy>=1.4.2
 numpy>=2.3.5
 packaging>=25.0
 pandas>=2.3.3
 pyarrow>=22.0.0
-pydantic~=2.12.4
+pydantic~=2.12.5
 python-dotenv>=1.2.1
 requests>=2.32.5
 sqlalchemy>=2.0.44
 sqlalchemy-mixins~=2.0.5
-werkzeug>=3.1.3
+werkzeug>=3.1.4
 
 # cidc deps
 nci-cidc-schemas==0.28.10