nci-cidc-api-modules 1.2.49__py3-none-any.whl → 1.2.51__py3-none-any.whl

cidc_api/__init__.py

@@ -1 +1 @@
-__version__ = "1.2.49"
+__version__ = "1.2.51"

cidc_api/config/settings.py

@@ -78,6 +78,7 @@ GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC = environ["GOOGLE_GRANT_DOWNLOAD_PERMISS
 GOOGLE_HL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_HL_CLINICAL_VALIDATION_TOPIC"]
 GOOGLE_DL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_DL_CLINICAL_VALIDATION_TOPIC"]
 GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC = environ["GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC"]
+GOOGLE_CLINICAL_DATA_INGESTION_PROCESSING_TOPIC = "ingestion"
 GOOGLE_AND_OPERATOR = " && "
 GOOGLE_OR_OPERATOR = " || "
 

cidc_api/models/models.py

@@ -36,6 +36,7 @@ __all__ = [
     "ADMIN_FILE_CATEGORIES",
     "FINAL_JOB_STATUS",
     "INGESTION_JOB_STATUSES",
+    "INGESTION_PHASE_STATUSES",
     "ASSAY_JOB_COLORS",
     "CLINICAL_JOB_COLORS",
 ]
@@ -1048,11 +1049,11 @@ class Permissions(CommonColumns):
             user_email_list.append(user.email)
             grant_lister_access(user.email)
 
-        if upload.upload_type in prism.SUPPORTED_SHIPPING_MANIFESTS:
+        if upload.upload_type in prism.SUPPORTED_MANIFESTS:
             # Passed with empty user email list because they will be queried for in CFn
             grant_download_access([], upload.trial_id, "participants info")
             grant_download_access([], upload.trial_id, "samples info")
-        elif upload.upload_type not in prism.SUPPORTED_WEIRD_MANIFESTS:
+        else:
             grant_download_access(user_email_list, upload.trial_id, upload.upload_type)
 
     @staticmethod
@@ -3448,6 +3449,8 @@ INGESTION_JOB_STATUSES = [
     "PUBLISHED",
 ]
 
+INGESTION_PHASE_STATUSES = ["NOT STARTED", "SUCCESS", "FAILED"]
+
 # Business decision to pass hex codes from the backend though that should be done by the front end...
 CLINICAL_JOB_COLORS = {
     "DRAFT": "",
@@ -3493,6 +3496,12 @@ class IngestionJobs(CommonColumns):
     uploader_email = Column(String, nullable=True)
     is_template_downloaded = Column(Boolean, nullable=True)
     master_aa_version = Column(Integer, nullable=True)
+    ingestion_phase_status = Column(
+        "ingestion_phase_status",
+        Enum(*INGESTION_PHASE_STATUSES, name="ingestion_phase_status"),
+        nullable=False,
+        default=INGESTION_PHASE_STATUSES[0],
+    )
 
     @staticmethod
     @with_default_session
@@ -3503,6 +3512,7 @@ class IngestionJobs(CommonColumns):
         error_status: str = None,
         pending: Boolean = False,
         job_type: str = "clinical",
+        ingestion_phase_status=INGESTION_PHASE_STATUSES[0],
         assay_type: str = None,
         batch_id: str = None,
         submission_id: str = None,
@@ -3524,6 +3534,7 @@ class IngestionJobs(CommonColumns):
             intake_path=intake_path,
             start_date=start_date,
             uploader_email=uploader_email,
+            ingestion_phase_status=ingestion_phase_status,
         )
         new_job.insert(session=session)
         return new_job

cidc_api/shared/gcloud_client.py

@@ -61,6 +61,7 @@ from ..config.settings import (
     GOOGLE_HL_CLINICAL_VALIDATION_TOPIC,
     GOOGLE_DL_CLINICAL_VALIDATION_TOPIC,
     GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC,
+    GOOGLE_CLINICAL_DATA_INGESTION_PROCESSING_TOPIC,
     TESTING,
     ENV,
     IS_EMAIL_ON,
@@ -486,9 +487,17 @@ async def async_gcs_files_to_pandas_dataframes(bucket_name: str, blob_names: Lis
     return dataframes
 
 
+async def async_gcs_files_to_bytes(bucket_name: str, blob_names: List[str]) -> List[DataFrame]:
+    """Async reads a XLSX or CSV files from Google Cloud Storage into a list of raw bytes"""
+
+    all_contents = await asyncio.gather(
+        *[async_get_file_bytes_from_gcs(bucket_name, blob_name) for blob_name in blob_names]
+    )
+    return all_contents
+
+
 async def async_get_file_bytes_from_gcs(bucket_name: str, blob_name: str) -> io.BytesIO:
     """Async reads a file from Google Cloud Storage and returns it as BytesIO."""
-
     async with Storage() as client:
         sheet_data = await client.download(bucket_name, blob_name)
     return io.BytesIO(sheet_data)
@@ -1059,6 +1068,12 @@ def publish_assay_metadata_validation(job_id: int) -> None:
     _report = _encode_and_publish(str(job_id), GOOGLE_ASSAY_METADATA_VALIDATION_TOPIC)
 
 
+def publish_clinical_data_ingestion(job_id: int) -> None:
+    """Start ingestion of clinical data job"""
+    # Start asynchronously
+    _report = _encode_and_publish(str(job_id), GOOGLE_CLINICAL_DATA_INGESTION_PROCESSING_TOPIC)
+
+
 def send_email(to_emails: List[str], subject: str, html_content: str, **kw) -> None:
     """
     Publish an email-to-send to the emails topic.

{nci_cidc_api_modules-1.2.49.dist-info → nci_cidc_api_modules-1.2.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.49
+Version: 1.2.51
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -43,7 +43,7 @@ Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
 Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
 Requires-Dist: opentelemetry-exporter-gcp-trace>=1.11.0
 Requires-Dist: opentelemetry-propagator-gcp>=1.11.0
-Requires-Dist: nci-cidc-schemas==0.28.11
+Requires-Dist: nci-cidc-schemas==0.28.12
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

{nci_cidc_api_modules-1.2.49.dist-info → nci_cidc_api_modules-1.2.51.dist-info}/RECORD

@@ -1,16 +1,16 @@
 boot.py,sha256=oS_16OgU34iDxyFNpJl5rb5KbfbsCXroUCNLwVPxPNg,416
-cidc_api/__init__.py,sha256=qkDVgmmc_W7WSA8UFQzKzULlTX27bnHrBTAlSeTYqYs,23
+cidc_api/__init__.py,sha256=kgsz9u_lLDc3N0akch6v9PpMXz_PW7_aEHXRb1pWgHg,23
 cidc_api/telemetry.py,sha256=LuZWkG8CKCn23O41RTNxEOQwMmfpp-fQ6zSInZhVJg8,3333
 cidc_api/config/__init__.py,sha256=5mX8GAPxUKV84iS-aGOoE-4m68LsOCGCDptXNdlgvj0,148
 cidc_api/config/db.py,sha256=mEz8ugjvRNGylCqDYHaaMqaZfDh7sbd76BowmfBvq5c,2428
 cidc_api/config/logging.py,sha256=abhVYtn8lfhIt0tyV2WHFgSmp_s2eeJh7kodB6LH4J0,1149
 cidc_api/config/secrets.py,sha256=jRFj7W43pWuPf9DZQLCKF7WPXf5cUv-BAaS3ASqhV_Q,1481
-cidc_api/config/settings.py,sha256=loern8cejUrOJJAog4z0iW9EQaa0nlN0rFrFOdZ-cLE,4483
+cidc_api/config/settings.py,sha256=IlOSJhSyYYK4XQQ4uBdD58kXO-a2AHPWTSN1AT95b9Y,4545
 cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
 cidc_api/models/data.py,sha256=TyVXk5jJO1NN05CLqejLf5BPp6yLSNNxPmFOgXUnA1M,1313
 cidc_api/models/errors.py,sha256=UncRrh-4v0CR5mpLO0erWD1y9ctus3Nk6XOSvrTjfco,338
 cidc_api/models/migrations.py,sha256=UlS5How3J4ryaRuZT6F5VQtAKikkl0LTv9MgMO_ltiQ,11161
-cidc_api/models/models.py,sha256=3O0Oc_MgovrMS1qz-r3zOnPd5_41p7sXc30Qj03QJ6s,153739
+cidc_api/models/models.py,sha256=RycbGNQHIud9mqCyIH_yrnN7PVSFwbUCjP6pe6sX5VA,154106
 cidc_api/models/schemas.py,sha256=6IE2dJoEMcMbi0Vr1V3cYKnPKU0hv9vRKBixOZHe88s,2766
 cidc_api/models/types.py,sha256=WVOTEH61FH5aGVwRbEcfNOce8NLjiEnl1ixtPWav2OA,30058
 cidc_api/models/db/stage1/__init__.py,sha256=zsxOSoogzL_xZ8B5OwcwLXvAmm6g2GYx1Zf6LLw8dq8,1917
@@ -156,12 +156,12 @@ cidc_api/shared/auth.py,sha256=EIP9AKokLNrI79Fkpv3P7WdzaddJIsuGGICc1W494X0,9110
 cidc_api/shared/email_layout.html,sha256=pBoTNw3ACHH-ncZFaNvcy5bXMqPwizR78usb0uCYtIc,7670
 cidc_api/shared/emails.py,sha256=8kNFEaSnKpY-GX_iE59QUhSp3c4_uzy3SpHYt2QjuqI,6121
 cidc_api/shared/file_handling.py,sha256=9MZXx5RfXO3A_pXf8Ulb7DQEuyp9j12eO9ad-PcTBXo,5908
-cidc_api/shared/gcloud_client.py,sha256=7q6CWpAZdqSJoAkMpm9Rw7yFwNLIk6D9WZGJeTOwo54,40113
+cidc_api/shared/gcloud_client.py,sha256=EJ2ZzZ0gwDOZx18PEzXil4F-jpruacShkVIoecw8qzY,40751
 cidc_api/shared/jose.py,sha256=-qzGzEDAlokEp9E7WtBtQkXyyfPWTYXlwYpCqVJWmqM,1830
 cidc_api/shared/rest_utils.py,sha256=RwR30WOUAYCxL7V-i2totEyeriG30GbBDvBcpLXhM9w,6594
 cidc_api/shared/utils.py,sha256=-gLnzxCR9E6h0plt2xrNisUG5_Y6GhhVwz3DgDIzpvs,367
-nci_cidc_api_modules-1.2.49.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
-nci_cidc_api_modules-1.2.49.dist-info/METADATA,sha256=vy1hCgQ0v8NmAfY8w71VOrXET5k9u95FmpAUJb-vxHQ,40301
-nci_cidc_api_modules-1.2.49.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
-nci_cidc_api_modules-1.2.49.dist-info/top_level.txt,sha256=BSJqF6ura8-bWCKZvarvQEKvidMM05lH0bLQsNOrI0o,14
-nci_cidc_api_modules-1.2.49.dist-info/RECORD,,
+nci_cidc_api_modules-1.2.51.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
+nci_cidc_api_modules-1.2.51.dist-info/METADATA,sha256=MCveB10mn5WAmlZX44mM0Z0BCCCu2wsttqjPitAF4Nc,40301
+nci_cidc_api_modules-1.2.51.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+nci_cidc_api_modules-1.2.51.dist-info/top_level.txt,sha256=BSJqF6ura8-bWCKZvarvQEKvidMM05lH0bLQsNOrI0o,14
+nci_cidc_api_modules-1.2.51.dist-info/RECORD,,

{nci_cidc_api_modules-1.2.49.dist-info → nci_cidc_api_modules-1.2.51.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.1)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any