nci-cidc-api-modules 1.2.49__tar.gz → 1.2.51__tar.gz

{nci_cidc_api_modules-1.2.49/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.49
+Version: 1.2.51
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -43,7 +43,7 @@ Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
 Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
 Requires-Dist: opentelemetry-exporter-gcp-trace>=1.11.0
 Requires-Dist: opentelemetry-propagator-gcp>=1.11.0
-Requires-Dist: nci-cidc-schemas==0.28.11
+Requires-Dist: nci-cidc-schemas==0.28.12
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

nci_cidc_api_modules-1.2.51/cidc_api/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.2.51"

{nci_cidc_api_modules-1.2.49 → nci_cidc_api_modules-1.2.51}/cidc_api/config/settings.py

@@ -78,6 +78,7 @@ GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC = environ["GOOGLE_GRANT_DOWNLOAD_PERMISS
 GOOGLE_HL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_HL_CLINICAL_VALIDATION_TOPIC"]
 GOOGLE_DL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_DL_CLINICAL_VALIDATION_TOPIC"]
 GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC = environ["GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC"]
+GOOGLE_CLINICAL_DATA_INGESTION_PROCESSING_TOPIC = "ingestion"
 GOOGLE_AND_OPERATOR = " && "
 GOOGLE_OR_OPERATOR = " || "
 

{nci_cidc_api_modules-1.2.49 → nci_cidc_api_modules-1.2.51}/cidc_api/models/models.py

@@ -36,6 +36,7 @@ __all__ = [
     "ADMIN_FILE_CATEGORIES",
     "FINAL_JOB_STATUS",
     "INGESTION_JOB_STATUSES",
+    "INGESTION_PHASE_STATUSES",
     "ASSAY_JOB_COLORS",
     "CLINICAL_JOB_COLORS",
 ]
@@ -1048,11 +1049,11 @@ class Permissions(CommonColumns):
             user_email_list.append(user.email)
             grant_lister_access(user.email)
 
-        if upload.upload_type in prism.SUPPORTED_SHIPPING_MANIFESTS:
+        if upload.upload_type in prism.SUPPORTED_MANIFESTS:
             # Passed with empty user email list because they will be queried for in CFn
             grant_download_access([], upload.trial_id, "participants info")
             grant_download_access([], upload.trial_id, "samples info")
-        elif upload.upload_type not in prism.SUPPORTED_WEIRD_MANIFESTS:
+        else:
             grant_download_access(user_email_list, upload.trial_id, upload.upload_type)
 
     @staticmethod
@@ -3448,6 +3449,8 @@ INGESTION_JOB_STATUSES = [
     "PUBLISHED",
 ]
 
+INGESTION_PHASE_STATUSES = ["NOT STARTED", "SUCCESS", "FAILED"]
+
 # Business decision to pass hex codes from the backend though that should be done by the front end...
 CLINICAL_JOB_COLORS = {
     "DRAFT": "",
@@ -3493,6 +3496,12 @@ class IngestionJobs(CommonColumns):
     uploader_email = Column(String, nullable=True)
     is_template_downloaded = Column(Boolean, nullable=True)
     master_aa_version = Column(Integer, nullable=True)
+    ingestion_phase_status = Column(
+        "ingestion_phase_status",
+        Enum(*INGESTION_PHASE_STATUSES, name="ingestion_phase_status"),
+        nullable=False,
+        default=INGESTION_PHASE_STATUSES[0],
+    )
 
     @staticmethod
     @with_default_session
@@ -3503,6 +3512,7 @@ class IngestionJobs(CommonColumns):
         error_status: str = None,
         pending: Boolean = False,
         job_type: str = "clinical",
+        ingestion_phase_status=INGESTION_PHASE_STATUSES[0],
         assay_type: str = None,
         batch_id: str = None,
         submission_id: str = None,
@@ -3524,6 +3534,7 @@ class IngestionJobs(CommonColumns):
             intake_path=intake_path,
             start_date=start_date,
             uploader_email=uploader_email,
+            ingestion_phase_status=ingestion_phase_status,
         )
         new_job.insert(session=session)
         return new_job

{nci_cidc_api_modules-1.2.49 → nci_cidc_api_modules-1.2.51}/cidc_api/shared/gcloud_client.py

@@ -61,6 +61,7 @@ from ..config.settings import (
     GOOGLE_HL_CLINICAL_VALIDATION_TOPIC,
     GOOGLE_DL_CLINICAL_VALIDATION_TOPIC,
     GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC,
+    GOOGLE_CLINICAL_DATA_INGESTION_PROCESSING_TOPIC,
     TESTING,
     ENV,
     IS_EMAIL_ON,
@@ -486,9 +487,17 @@ async def async_gcs_files_to_pandas_dataframes(bucket_name: str, blob_names: Lis
     return dataframes
 
 
+async def async_gcs_files_to_bytes(bucket_name: str, blob_names: List[str]) -> List[DataFrame]:
+    """Async reads a XLSX or CSV files from Google Cloud Storage into a list of raw bytes"""
+
+    all_contents = await asyncio.gather(
+        *[async_get_file_bytes_from_gcs(bucket_name, blob_name) for blob_name in blob_names]
+    )
+    return all_contents
+
+
 async def async_get_file_bytes_from_gcs(bucket_name: str, blob_name: str) -> io.BytesIO:
     """Async reads a file from Google Cloud Storage and returns it as BytesIO."""
-
     async with Storage() as client:
         sheet_data = await client.download(bucket_name, blob_name)
     return io.BytesIO(sheet_data)
@@ -1059,6 +1068,12 @@ def publish_assay_metadata_validation(job_id: int) -> None:
     _report = _encode_and_publish(str(job_id), GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC)
 
 
+def publish_clinical_data_ingestion(job_id: int) -> None:
+    """Start ingestion of clinical data job"""
+    # Start asynchronously
+    _report = _encode_and_publish(str(job_id), GOOGLE_CLINICAL_DATA_INGESTION_PROCESSING_TOPIC)
+
+
 def send_email(to_emails: List[str], subject: str, html_content: str, **kw) -> None:
     """
     Publish an email-to-send to the emails topic.

{nci_cidc_api_modules-1.2.49 → nci_cidc_api_modules-1.2.51/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.49
+Version: 1.2.51
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -43,7 +43,7 @@ Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
 Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
 Requires-Dist: opentelemetry-exporter-gcp-trace>=1.11.0
 Requires-Dist: opentelemetry-propagator-gcp>=1.11.0
-Requires-Dist: nci-cidc-schemas==0.28.11
+Requires-Dist: nci-cidc-schemas==0.28.12
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

{nci_cidc_api_modules-1.2.49 → nci_cidc_api_modules-1.2.51}/nci_cidc_api_modules.egg-info/requires.txt

@@ -34,4 +34,4 @@ opentelemetry-instrumentation-requests>=0.59b0
 opentelemetry-instrumentation-sqlalchemy>=0.59b0
 opentelemetry-exporter-gcp-trace>=1.11.0
 opentelemetry-propagator-gcp>=1.11.0
-nci-cidc-schemas==0.28.11
+nci-cidc-schemas==0.28.12

{nci_cidc_api_modules-1.2.49 → nci_cidc_api_modules-1.2.51}/requirements.modules.txt

@@ -42,4 +42,4 @@ opentelemetry-exporter-gcp-trace >= 1.11.0
 opentelemetry-propagator-gcp >= 1.11.0
 
 # cidc deps
-nci-cidc-schemas==0.28.11
+nci-cidc-schemas==0.28.12

nci_cidc_api_modules-1.2.49/cidc_api/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.2.49"