nci-cidc-api-modules 1.2.34__py3-none-any.whl → 1.2.45__py3-none-any.whl

cidc_api/models/pydantic/stage2/prior_treatment.py

@@ -2,8 +2,8 @@ from typing import Self, Annotated, List
 
 from pydantic import NonPositiveInt, NegativeInt, model_validator, BeforeValidator
 
-from .base import Base
-from cidc_api.models.types import PriorTreatmentType, ConditioningRegimenType, StemCellDonorType
+from cidc_api.models.pydantic.base import Base
+from cidc_api.models.types import ConditioningRegimenType, StemCellDonorType
 
 
 class PriorTreatment(Base):
@@ -24,12 +24,9 @@ class PriorTreatment(Base):
     # the treatment modality.
     prior_treatment_days_to_end: NonPositiveInt | None = None
 
-    # Specifies the category or kind of prior treatment modality a participant received.
-    prior_treatment_type: Annotated[List[PriorTreatmentType], BeforeValidator(Base.split_list)]
-
     # Description of the prior treatment such as its full generic name if it is a type of therapy agent,
     # radiotherapy procedure name and location, or surgical procedure name and location.
-    prior_treatment_description: str | None = None
+    prior_treatment_description: str
 
     # Best response from any response assessment system to the prior treatment if available or applicable.
     prior_treatment_best_response: str | None = None
@@ -41,12 +38,6 @@ class PriorTreatment(Base):
     # If prior treatment is "Stem cell transplant", indicates what stem cell donor type used.
     prior_treatment_stem_cell_donor_type: StemCellDonorType | None = None
 
-    # If prior treatment is "Stem cell transplant", indicates the number of days from the transplant
-    # date to the start of the current treatment.
-    prior_treatment_days_to_prior_transplant: NegativeInt | None = None
-
-    @model_validator(mode="after")
-    def validate_description_cr(self) -> Self:
-        if "Other therapy" in self.prior_treatment_type and not self.prior_treatment_description:
-            raise ValueError('If type is "Other therapy", please provide description.')
-        return self
+    # If prior treatment is "Stem cell transplant", indicates the number of days from enrollment
+    # to the prior transplant. This must be a negative number.
+    prior_treatment_days_from_transplant_to_treatment_initiation: NegativeInt | None = None

cidc_api/models/pydantic/stage2/publication.py

@@ -1,4 +1,4 @@
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 
 
 class Publication(Base):
@@ -6,7 +6,7 @@ class Publication(Base):
     publication_id: int | None = None
 
     # The unique internal identifier for the associated Trial record
-    trial_id: int | None = None
+    trial_id: str | None = None
 
     # The version number of the trial dataset
     version: str | None = None

cidc_api/models/pydantic/stage2/radiotherapy_dose.py

@@ -2,7 +2,7 @@ from typing import Self
 
 from pydantic import NonNegativeInt, NonNegativeFloat, model_validator
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import (
     YNU,
     RadiotherapyProcedure,

cidc_api/models/pydantic/stage2/response.py

@@ -2,7 +2,7 @@ from typing import Self
 
 from pydantic import NonNegativeInt, model_validator
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import SurvivalStatus, YNUNA, CauseOfDeath
 
 
@@ -44,7 +44,7 @@ class Response(Base):
     evaluable_for_efficacy: bool
 
     # Days from enrollment date to the last time the patient's vital status was verified.
-    days_to_last_vital_status: NonNegativeInt | None = None
+    days_to_last_vital_status: NonNegativeInt | None = None  # TODO: Needs CR check
 
     @model_validator(mode="after")
     def validate_cause_of_death_cr(self) -> Self:

cidc_api/models/pydantic/stage2/response_by_system.py

@@ -2,7 +2,7 @@ from typing import Self
 
 from pydantic import PositiveInt, model_validator, NonNegativeInt
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import ResponseSystem, ResponseSystemVersion, BestOverallResponse, YNUNA
 
 

cidc_api/models/pydantic/stage2/shipment.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import AssayPriority, AssayType, Courier, ShipmentCondition, ShipmentQuality
 
 
@@ -12,7 +12,7 @@ class Shipment(Base):
     institution_id: int | None = None
 
     # The unique internal identifier for the associated trial.
-    trial_id: int | None = None
+    trial_id: str | None = None
 
     # The version number of the trial dataset
     version: str | None = None

cidc_api/models/pydantic/stage2/shipment_specimen.py

@@ -1,4 +1,4 @@
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 
 
 class ShipmentSpecimen(Base):

cidc_api/models/pydantic/stage2/specimen.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import (
     UberonAnatomicalTerm,
     ICDO3MorphologicalCode,
@@ -66,7 +66,7 @@ class Specimen(Base):
     # Categorical description of timepoint at which the sample was taken.
     # CDE: https://cadsr.cancer.gov/onedata/dmdirect/NIH/NCI/CO/CDEDD?filter=CDEDD.ITEM_ID=5899851%20and%20ver_nr=1
     # Note: CIDC doesn't conform to this CDE's PVs
-    collection_event_name: str | None = None
+    collection_event_name: str
 
     # The type of the specimen
     specimen_type: SpecimenType | None = None
@@ -208,4 +208,7 @@ class Specimen(Base):
     date_ingested: datetime | None = None
 
     # Days from enrollment date to date specimen was collected.
-    days_to_specimen_collection: int | None = None
+    days_to_specimen_collection: int
+
+    # The location within the body from which a specimen was originally obtained as captured in the Uberon anatomical term.
+    organ_site_of_collection: UberonAnatomicalTerm

cidc_api/models/pydantic/stage2/stem_cell_transplant.py

@@ -1,5 +1,5 @@
 from pydantic import NonNegativeInt
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import (
     StemCellDonorType,
     AllogeneicDonorType,
@@ -26,7 +26,7 @@ class StemCellTransplant(Base):
     allogeneic_donor_type: AllogeneicDonorType | None = None
 
     # Source of the stem cells used for transplant.
-    stem_cells_source: StemCellSource
+    stem_cell_source: StemCellSource
 
     # Days from the enrollment date to the date of the stem cell transplant.
     days_to_transplant: NonNegativeInt

cidc_api/models/pydantic/stage2/surgery.py

@@ -2,7 +2,7 @@ from typing import Self
 
 from pydantic import NonNegativeInt, model_validator
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import SurgicalProcedure, UberonAnatomicalTerm, YNU
 
 

cidc_api/models/pydantic/stage2/therapy_agent_dose.py

@@ -2,7 +2,7 @@ from typing import Self
 
 from pydantic import NonNegativeInt, NonNegativeFloat, PositiveFloat, model_validator
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import YNU, TherapyAgentDoseUnits
 
 

cidc_api/models/pydantic/stage2/treatment.py

@@ -2,7 +2,7 @@ from typing import Self
 
 from pydantic import model_validator
 
-from .base import Base
+from cidc_api.models.pydantic.base import Base
 from cidc_api.models.types import YNU, OffTreatmentReason
 
 

cidc_api/models/pydantic/stage2/trial.py

@@ -2,8 +2,8 @@ from datetime import datetime
 from pydantic import BeforeValidator
 from typing import List, Annotated
 
-from .base import Base
-from cidc_api.models.types import TrialOrganization, TrialFundingAgency, AssayType, AgeGroup
+from cidc_api.models.pydantic.base import Base
+from cidc_api.models.types import TrialOrganization, TrialFundingAgency, AssayType, AgeGroup, PrimaryPurposeType
 
 
 class Trial(Base):
@@ -12,7 +12,7 @@ class Trial(Base):
 
     # The unique identifier for the clinical trial. e.g. "GU16-287","BACCI"
     # CDE: https://cadsr.cancer.gov/onedata/dmdirect/NIH/NCI/CO/CDEDD?filter=CDEDD.ITEM_ID=5054234%20and%20ver_nr=1
-    trial_id: str | None = None  # TODO: Fix stage2 trial_id to not be nullable, once stage 1 models are complete
+    trial_id: str | None = None
 
     # The version number of the trial dataset. e.g. "1.0"
     version: str | None = None
@@ -69,6 +69,10 @@ class Trial(Base):
     # CDE: https://cadsr.cancer.gov/onedata/dmdirect/NIH/NCI/CO/CDEDD?filter=CDEDD.ITEM_ID=16333703%20and%20ver_nr=1
     dates_of_conduct_end: datetime | None = None
 
+    # A classification of the study based upon the primary intent of the study's activities.
+    # CDE: https://cadsr.cancer.gov/onedata/dmdirect/NIH/NCI/CO/CDEDD?filter=CDEDD.ITEM_ID=11160683%20and%20ver_nr=1
+    primary_purpose_type: PrimaryPurposeType
+
     # The image of the trial data schema
     schema_file_id: int | None = None
 
@@ -81,11 +85,5 @@ class Trial(Base):
     # The list of assays that CIDC expects to receive for this trial.
     expected_assays: List[AssayType] = []
 
-    # Is the cancer studying a liquid tumor type?
-    is_liquid_tumor_trial: bool = False
-
     # The dbgap study accession number associated with the trial.
-    dbgap_study_accession: str | None = None
-
-    # The internal version identifier for this specific trial dataset.
-    version: str
+    dbgap_study_accession: str

cidc_api/models/types.py

@@ -26,6 +26,7 @@ AgeGroup = Literal[
     "Pediatric",
 ]
 
+
 TrialOrganization = Literal[
     "ECOG-ACRIN",
     "SWOG",
@@ -59,6 +60,34 @@ TrialFundingAgency = Literal[
 ]
 
 
+PrimaryPurposeType = Literal[
+    "Adverse Effect Mitigation Study",
+    "Ancillary Study",
+    "Basic Science  Research ",
+    "Correlative Study",
+    "Cure Study",
+    "Device Feasibility Study",
+    "Diagnosis Study",
+    "Disease Modifying Treatment Study",
+    "Early Detection Study",
+    "Education Training Clinical Study",
+    "Epidemiology  Research ",
+    "Genomics Research",
+    "Health Services Research",
+    "Imaging Research",
+    "Interventional Study",
+    "Observational Study",
+    "Outcomes Research",
+    "Prevention Study",
+    "Proteomic Research",
+    "Rehabilitation Clinical Study ",
+    "Screening Study",
+    "Supportive Care Study",
+    "Transcriptomics Research",
+    "Treatment Study",
+]
+
+
 AssayType = Literal[
     "Olink",
     "WES",
@@ -84,6 +113,7 @@ AssayType = Literal[
     "snRNA-Seq",
     "Visium",
     "Olink HT",
+    "TCRseq RNA",
 ]
 
 
@@ -285,7 +315,6 @@ CancerStageAJCC = Literal[
 
 
 CancerStageFIGO = Literal[
-    "value",
     "Stage I",
     "Stage IA",
     "Stage IA1",
@@ -1016,21 +1045,6 @@ GVHDDiagnosisChronicGlobalSeverity = Literal["Mild", "Moderate", "Severe"]
 GVHDOrganChronicScore = Literal["0", "1", "2", "3"]
 
 
-PriorTreatmentType = Literal[
-    "Surgery",
-    "Radiotherapy",
-    "Immunotherapy",
-    "Chemotherapy",
-    "Targeted therapy",
-    "Other therapy",
-    "Radiopharmaceutical",
-    "Stem cell transplant",
-    "Immunosuppressive therapy/GVHD prophylaxis for transplant",
-    "Conditioning therapy",
-    "Post-transplant salvage therapy",
-]
-
-
 ConditioningRegimenType = Literal["Myeloablative", "Reduced-intensity", "Non-myeloablative", "Other"]
 
 StemCellDonorType = Literal["Autologous", "Allogeneic"]

cidc_api/shared/assay_handling.py

@@ -0,0 +1,68 @@
+from datetime import datetime
+from urllib.parse import quote
+
+from werkzeug.exceptions import BadRequest
+
+from cidc_api.models import IngestionJobs
+from . import gcloud_client
+from ..shared.auth import get_current_user
+
+JOB_TYPE_ASSAY = "assay"
+JOB_TYPE_CLINICAL = "clinical"
+ALLOWED_JOB_TYPES = {JOB_TYPE_CLINICAL, JOB_TYPE_ASSAY}
+
+
+def resolve_job_type_and_assay_fields(data: dict) -> tuple[str, str | None, str | None]:
+    """Decide job_type and gather assay_type/batch_id from request JSON."""
+    assay_type = data.get("assay_type")
+    # If job_type is assay or assay_type is present, treat this as an assay job.
+    job_type = data.get("job_type") or (JOB_TYPE_ASSAY if assay_type else JOB_TYPE_CLINICAL)
+
+    if job_type not in ALLOWED_JOB_TYPES:
+        raise BadRequest("Invalid job_type. Allowed values are 'clinical' or 'assay'.")
+
+    if job_type == JOB_TYPE_ASSAY and (not assay_type or not isinstance(assay_type, str)):
+        raise BadRequest("assay_type must be provided for job_type='assay'.")
+
+    assay_type = assay_type.strip() if assay_type else None
+    batch_id = data.get("batch_id").strip() if isinstance(data.get("batch_id"), str) else None
+
+    return job_type, assay_type, batch_id
+
+
+def prepare_assay_job(trial_id: str, assay_type: str, batch_id: str) -> tuple[str, str, str, datetime, int, str]:
+    """
+    Validate assay job uniqueness and generate submission_id, start_date, version, and the trial’s GCS intake path.
+    """
+    if not assay_type:
+        raise BadRequest("assay_type must be provided for job_type='assay'.")
+
+    # Enforce uniqueness of (trial_id, assay_type, batch_id) when batch_id is present.
+    if batch_id:
+        existing_job = IngestionJobs.get_unique_assay_job(trial_id, assay_type, batch_id)
+        if existing_job:
+            raise BadRequest(
+                f"Assay job {existing_job.id} already exists for this exact trial_id/assay_type/batch_id combination."
+            )
+
+    submission_id = IngestionJobs.next_assay_submission_id(trial_id, assay_type)
+    job_status = "INITIAL SUBMISSION"
+    error_status = "Upload Incomplete"  # job starts with 'Incomplete' notifier
+    start_date = datetime.now()
+    version = 1
+
+    # Create or retrieve intake bucket corresponding to the trial
+    intake_bucket = gcloud_client.create_intake_bucket(get_current_user().email, trial_id=trial_id)
+    gcs_path = f"{intake_bucket.name}/{assay_type}/{submission_id}"
+
+    return submission_id, job_status, error_status, start_date, version, gcs_path
+
+
+def get_google_links(intake_path: str) -> tuple[str, str]:
+    """Build the GCS URI and GCS Console URL corresponding to the intake path."""
+    gcs_uri = f"gs://{intake_path}"
+    # Encode path to ensure link opens correctly
+    encoded_path = quote(intake_path)
+    console_url = f"https://console.cloud.google.com/storage/browser/{encoded_path}"
+
+    return gcs_uri, console_url

cidc_api/shared/auth.py

@@ -1,16 +1,14 @@
 from functools import wraps
 from typing import List
 
-from packaging import version
-
 from flask import g, request, current_app as app, Flask
+from packaging import version
 from werkzeug.exceptions import Unauthorized, BadRequest, PreconditionFailed
 
-from ..models import Users, UserSchema
-
 from ..config.logging import get_logger
-
+from ..models import Users, UserSchema
 from ..shared.jose import decode_id_token
+from ..telemetry import trace_
 
 logger = get_logger(__name__)
 
@@ -144,6 +142,7 @@ def get_current_user() -> Users:
 _user_schema = UserSchema()
 
 
+@trace_()
 def authenticate() -> Users:
     id_token = _extract_token()
     token_payload = decode_id_token(id_token)
@@ -172,6 +171,7 @@ def _extract_token() -> str:
 
 
 ### Authorization logic ###
+@trace_()
 def authorize(user: Users, allowed_roles: List[str], resource: str, method: str) -> bool:
     """Check if the current user is authorized to act on the current request's resource.
     Raises Unauthorized

cidc_api/shared/file_handling.py

@@ -10,22 +10,33 @@ from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
 from ..models import PreprocessedFiles, TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER
 from ..shared.auth import get_current_user
 from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
+from ..telemetry import trace_
 
 logger = get_logger(__name__)
 
 
+@trace_()
 def set_current_file(
-    file: FileStorage, file_category: str, gcs_folder: str, session: Session, uploader_email: str, job_id: int = None
+    file: FileStorage,
+    file_category: str,
+    gcs_folder: str,
+    session: Session,
+    uploader_email: str,
+    job_id: int = None,
+    append_timestamp: bool = None,
 ) -> PreprocessedFiles:
     """
     Archives any existing 'current' files for the given category and job,
     then uploads the new file as the latest 'current' version.
     """
     latest_version = PreprocessedFiles.archive_current_files(file_category, job_id=job_id, session=session)
-    latest_file = create_file(file, gcs_folder, file_category, session, uploader_email, job_id, latest_version + 1)
+    latest_file = create_file(
+        file, gcs_folder, file_category, session, uploader_email, job_id, latest_version + 1, append_timestamp
+    )
     return latest_file
 
 
+@trace_()
 def create_file(
     file: FileStorage,
     gcs_folder: str,
@@ -34,11 +45,12 @@ def create_file(
     uploader_email: str,
     job_id: int = None,
     version: int = None,
+    append_timestamp: bool = None,
 ) -> PreprocessedFiles:
     """Upload file to GCS and create corresponding metadata record in the database."""
     status = "pending" if gcs_folder.endswith("pending/") else "current"
-    # only need timestamp for current/versioned files
-    append_timestamp = status == "current"
+    # only need timestamp for current/versioned files, if not specified otherwise
+    append_timestamp = append_timestamp if append_timestamp is not None else (status == "current")
     # create file in GCS
     gcs_file_path = upload_file_to_gcs(file, GOOGLE_CLINICAL_DATA_BUCKET, gcs_folder, append_timestamp=append_timestamp)
     # create corresponding record in db

cidc_api/shared/gcloud_client.py

@@ -1,13 +1,15 @@
 """Utilities for interacting with the Google Cloud Platform APIs."""
 
-# pylint: disable=logging-fstring-interpolation,too-many-lines
+# pylint: disable=logging-fstring-interpolation,too-many-lines, broad-exception-raised
 
+import asyncio
 import base64
 import datetime
 import hashlib
 import io
 import json
 import os
+import re
 import warnings
 from collections import namedtuple
 from concurrent.futures import Future
@@ -25,6 +27,8 @@ from typing import (
 )
 
 import googleapiclient.discovery
+from gcloud.aio.storage import Storage
+from pandas.core.frame import DataFrame
 import pandas as pd
 import requests
 from cidc_schemas.prism.constants import ASSAY_TO_FILEPATH
@@ -56,6 +60,7 @@ from ..config.settings import (
     GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC,
     GOOGLE_HL_CLINICAL_VALIDATION_TOPIC,
     GOOGLE_DL_CLINICAL_VALIDATION_TOPIC,
+    GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC,
     TESTING,
     ENV,
     IS_EMAIL_ON,
@@ -361,15 +366,34 @@ def get_intake_bucket_name(user_email: str) -> str:
     return bucket_name
 
 
-def create_intake_bucket(user_email: str) -> storage.Bucket:
+def get_trial_intake_bucket_name(trial_id: str) -> str:
     """
-    Create a new data intake bucket for this user, or get the existing one.
+    Return a sanitized GCS bucket name for a given trial_id.
+
+    Produces:  <GOOGLE_INTAKE_BUCKET>-<sanitized_trial_id>
+    where the trial_id segment is lowercased and restricted to [a-z0-9-].
+    """
+    # Replace non-allowed bucket chars with "-"
+    sanitized_id = re.sub(r"[^a-z0-9-]", "-", trial_id.lower())
+    # Collapse repeated "-" and trim from both ends
+    sanitized_id = re.sub(r"-+", "-", sanitized_id).strip("-")
+
+    return f"{GOOGLE_INTAKE_BUCKET}-{sanitized_id}"
+
+
+def create_intake_bucket(user_email: str, trial_id: str = None) -> storage.Bucket:
+    """
+    Create (or retrieve) the appropriate data intake bucket.
+    If a trial_id is provided, a trial-specific bucket is used;
+    otherwise a user-specific intake bucket is used.
+
     Grant the user GCS object admin permissions on the bucket, or refresh those
     permissions if they've already been granted.
     Created with uniform bucket-level IAM access, so expiring permission.
     """
     storage_client = _get_storage_client()
-    bucket_name = get_intake_bucket_name(user_email)
+    # Get trial-specific bucket name if trial_id is given, otherwise a user-specific bucket name.
+    bucket_name = get_trial_intake_bucket_name(trial_id) if trial_id else get_intake_bucket_name(user_email)
     bucket = storage_client.bucket(bucket_name)
 
     if not bucket.exists():
@@ -423,25 +447,50 @@ def upload_xlsx_to_intake_bucket(user_email: str, trial_id: str, upload_type: st
     return f"https://console.cloud.google.com/storage/browser/_details/{bucket_name}/{blob_name}"
 
 
-def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
-    """Reads an XLSX or CSV file from Google Cloud Storage into a Pandas DataFrame."""
-    temp_file = get_file_bytes_from_gcs(bucket_name, blob_name)
-
-    # TODO: specify sheet in xlsx file and/or accept tsv and xls files
-    if blob_name[-3:] == "csv":
-        return strip_whitespaces(pd.read_csv(temp_file))
-    elif blob_name[-4:] == "xlsx":
-        return strip_whitespaces(pd.read_excel(temp_file))
+def prepare_dataframe(extension, bytes) -> DataFrame:
+    if extension == "csv":
+        return strip_whitespaces(pd.read_csv(bytes, dtype=str, keep_default_na=False))
+    elif extension == "xlsx":
+        return strip_whitespaces(pd.read_excel(bytes, dtype=str, keep_default_na=False))
     else:
         raise Exception("Can only read csv or xlsx files")
 
 
+def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str) -> DataFrame:
+    """Reads an XLSX or CSV file from Google Cloud Storage into a Pandas DataFrame."""
+    contents = get_file_bytes_from_gcs(bucket_name, blob_name)
+    extension = blob_name.split(".")[-1]
+    return prepare_dataframe(extension, contents)
+
+
 def get_file_bytes_from_gcs(bucket_name: str, blob_name: str) -> io.BytesIO:
     """Reads a file from Google Cloud Storage and returns it as BytesIO."""
     sheet_data = storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
     return io.BytesIO(sheet_data)
 
 
+async def async_gcs_files_to_pandas_dataframes(bucket_name: str, blob_names: List[str]) -> List[DataFrame]:
+    """Async reads a XLSX or CSV files from Google Cloud Storage into a list of Pandas DataFrames."""
+
+    all_contents = await asyncio.gather(
+        *[async_get_file_bytes_from_gcs(bucket_name, blob_name) for blob_name in blob_names]
+    )
+    dataframes = []
+
+    for blob_name, contents in zip(blob_names, all_contents):
+        extension = blob_name.split(".")[-1]
+        dataframes.append(prepare_dataframe(extension, contents))
+    return dataframes
+
+
+async def async_get_file_bytes_from_gcs(bucket_name: str, blob_name: str) -> io.BytesIO:
+    """Async reads a file from Google Cloud Storage and returns it as BytesIO."""
+
+    async with Storage() as client:
+        sheet_data = await client.download(bucket_name, blob_name)
+    return io.BytesIO(sheet_data)
+
+
 def _execute_multiblob_acl_change(
     user_email_list: List[str],
     blob_list: List[storage.Blob],
@@ -614,6 +663,7 @@ def _build_trial_upload_prefixes(
     trial_set: Set[str] = set()
     upload_set: Set[str] = set()
     if not trial_id:
+        # import is here becasue of circular import
         from ..models.models import TrialMetadata
 
         trial_set = {str(t.trial_id) for t in session.query(TrialMetadata).add_columns(TrialMetadata.trial_id)}
@@ -886,6 +936,7 @@ def get_signed_url(
     bucket_name: str = GOOGLE_ACL_DATA_BUCKET,
     method: str = "GET",
     expiry_mins: int = 30,
+    use_short_filename: bool = False,
 ) -> str:
     """
     Generate a signed URL for `object_name` to give a client temporary access.
@@ -900,7 +951,11 @@ def get_signed_url(
 
     # Generate the signed URL, allowing a client to use `method` for `expiry_mins` minutes
     expiration = datetime.timedelta(minutes=expiry_mins)
-    full_filename = object_name.replace("/", "_").replace('"', "_").replace(" ", "_")
+    if use_short_filename:
+        filename = os.path.basename(object_name)
+    else:
+        # full filename with path included
+        filename = object_name.replace("/", "_").replace('"', "_").replace(" ", "_")
     other_kwargs = {}
     if os.environ.get("DEV_GOOGLE_STORAGE", None):
         other_kwargs["api_access_endpoint"] = (os.environ.get("DEV_GOOGLE_STORAGE") or "") + (
@@ -910,7 +965,7 @@ def get_signed_url(
         version="v2",
         expiration=expiration,
         method=method,
-        response_disposition=f'attachment; filename="{full_filename}"',
+        response_disposition=f'attachment; filename="{filename}"',
         **other_kwargs,
     )
     logger.info(f"generated signed URL for {object_name}: {url}")
@@ -920,7 +975,8 @@ def get_signed_url(
 
 def _encode_and_publish(content: str, topic: str) -> Future:
     """Convert `content` to bytes and publish it to `topic`."""
-    pubsub_publisher = pubsub.PublisherClient()
+    publisher_options = pubsub.types.PublisherOptions(enable_open_telemetry_tracing=ENV == "dev-int")
+    pubsub_publisher = pubsub.PublisherClient(publisher_options=publisher_options)
     topic = pubsub_publisher.topic_path(GOOGLE_CLOUD_PROJECT, topic)
     data = bytes(content, "utf-8")
 
@@ -994,6 +1050,12 @@ def publish_detailed_validation(job_id: int) -> None:
     _report = _encode_and_publish(str(job_id), GOOGLE_DL_CLINICAL_VALIDATION_TOPIC)
 
 
+def publish_assay_metadata_validation(job_id: int) -> None:
+    """Publish to the assay_metadata_validation topic that a job's assay metadata file is ready to be validated."""
+    # Start validation asynchronously
+    _report = _encode_and_publish(str(job_id), GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC)
+
+
 def send_email(to_emails: List[str], subject: str, html_content: str, **kw) -> None:
     """
     Publish an email-to-send to the emails topic.