nci-cidc-api-modules 1.2.17__tar.gz → 1.2.19__tar.gz

{nci_cidc_api_modules-1.2.17/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.17
+Version: 1.2.19
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -28,6 +28,7 @@ Requires-Dist: pyarrow>=22.0.0
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: requests>=2.32.5
 Requires-Dist: sqlalchemy>=2.0.44
+Requires-Dist: sqlalchemy-mixins~=2.0.5
 Requires-Dist: werkzeug>=3.1.3
 Requires-Dist: nci-cidc-schemas==0.28.8
 Dynamic: description
@@ -217,13 +218,14 @@ gcloud auth application-default login
 In your .env file, comment out `POSTGRES_URI` and uncommment
 `CLOUD_SQL_INSTANCE_NAME CLOUD_SQL_DB_USER CLOUD_SQL_DB_NAME` Replace `CLOUD_SQL_DB_USER` with your NIH email.
 
-### Running database migrations
+### Creating/Running database migrations
 
 This project uses [`Flask Migrate`](https://flask-migrate.readthedocs.io/en/latest/) for managing database migrations. To create a new migration and upgrade the database specified in your `.env` config:
 
 ```bash
 export FLASK_APP=cidc_api/app.py
-# Generate the migration script
+# First, make your changes to the model(s)
+# Then, let flask automatically generate the db change. Double check the migration script!
 flask db migrate -m "<a message describing the changes in this migration>"
 # Apply changes to the database
 flask db upgrade

{nci_cidc_api_modules-1.2.17 → nci_cidc_api_modules-1.2.19}/README.md

@@ -176,13 +176,14 @@ gcloud auth application-default login
 In your .env file, comment out `POSTGRES_URI` and uncommment
 `CLOUD_SQL_INSTANCE_NAME CLOUD_SQL_DB_USER CLOUD_SQL_DB_NAME` Replace `CLOUD_SQL_DB_USER` with your NIH email.
 
-### Running database migrations
+### Creating/Running database migrations
 
 This project uses [`Flask Migrate`](https://flask-migrate.readthedocs.io/en/latest/) for managing database migrations. To create a new migration and upgrade the database specified in your `.env` config:
 
 ```bash
 export FLASK_APP=cidc_api/app.py
-# Generate the migration script
+# First, make your changes to the model(s)
+# Then, let flask automatically generate the db change. Double check the migration script!
 flask db migrate -m "<a message describing the changes in this migration>"
 # Apply changes to the database
 flask db upgrade

{nci_cidc_api_modules-1.2.17 → nci_cidc_api_modules-1.2.19}/cidc_api/models/__init__.py

@@ -1,3 +1,5 @@
 from .models import *
 from .files import *
 from .schemas import *
+
+from cidc_api.models.db.base_orm import BaseORM

nci_cidc_api_modules-1.2.19/cidc_api/models/data.py

@@ -0,0 +1,15 @@
+from cidc_api.models.pydantic.stage2 import all_models
+
+standard_data_categories = [model.__data_category__ for model in all_models if hasattr(model, "__data_category__")]
+
+
+# A class to hold the representation of a trial's dataset all at once
+class Dataset(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        for data_category in standard_data_categories:
+            self[data_category] = []
+
+
+# Maps data categories like "treatment" to their associated pydantic model
+data_category_to_model = {model.__data_category__: model for model in all_models if hasattr(model, "__data_category__")}

{nci_cidc_api_modules-1.2.17 → nci_cidc_api_modules-1.2.19}/cidc_api/models/migrations.py

@@ -91,15 +91,11 @@ def migration_session():
         session.close()
 
 
-def run_metadata_migration(
-    metadata_migration: Callable[[dict], MigrationResult], use_upload_jobs_table: bool
-):
+def run_metadata_migration(metadata_migration: Callable[[dict], MigrationResult], use_upload_jobs_table: bool):
     """Migrate trial metadata, upload job patches, and downloadable files according to `metadata_migration`"""
     with migration_session() as (session, task_queue):
         try:
-            _run_metadata_migration(
-                metadata_migration, use_upload_jobs_table, task_queue, session
-            )
+            _run_metadata_migration(metadata_migration, use_upload_jobs_table, task_queue, session)
         except:
             traceback.print_exc()
             raise
@@ -122,9 +118,7 @@ class ManifestUploads(CommonColumns):
     __tablename__ = "manifest_uploads"
 
 
-def _select_successful_assay_uploads(
-    use_upload_jobs_table: bool, session: Session
-) -> List[UploadJobs]:
+def _select_successful_assay_uploads(use_upload_jobs_table: bool, session: Session) -> List[UploadJobs]:
     if use_upload_jobs_table:
         return (
             session.query(UploadJobs)
@@ -133,21 +127,12 @@ def _select_successful_assay_uploads(
             .all()
         )
 
-    return (
-        session.query(AssayUploads)
-        .filter_by(status=UploadJobStatus.MERGE_COMPLETED.value)
-        .with_for_update()
-        .all()
-    )
+    return session.query(AssayUploads).filter_by(status=UploadJobStatus.MERGE_COMPLETED.value).with_for_update().all()
 
 
-def _select_manifest_uploads(
-    use_upload_jobs_table: bool, session: Session
-) -> List[UploadJobs]:
+def _select_manifest_uploads(use_upload_jobs_table: bool, session: Session) -> List[UploadJobs]:
     if use_upload_jobs_table:
-        return (
-            session.query(UploadJobs).filter_by(multifile=False).with_for_update().all()
-        )
+        return session.query(UploadJobs).filter_by(multifile=False).with_for_update().all()
 
     return session.query(ManifestUploads).with_for_update().all()
 
@@ -188,21 +173,15 @@ def _run_metadata_migration(
 
             # Regenerate additional metadata from the migrated clinical trial
             # metadata object.
-            print(
-                f"Regenerating additional metadata for artifact with uuid {artifact['upload_placeholder']}"
-            )
+            print(f"Regenerating additional metadata for artifact with uuid {artifact['upload_placeholder']}")
             artifact_path = uuid_path_map[artifact["upload_placeholder"]]
-            df.additional_metadata = get_source(
-                migration.result, artifact_path, skip_last=True
-            )[1]
+            df.additional_metadata = get_source(migration.result, artifact_path, skip_last=True)[1]
 
             # If the GCS URI has changed, rename the blob
             # makes call to bucket.rename_blob
             new_gcs_uri = artifact["object_url"]
             if old_gcs_uri != new_gcs_uri:
-                print(
-                    f"Encountered GCS data bucket artifact URI to update: {old_gcs_uri}"
-                )
+                print(f"Encountered GCS data bucket artifact URI to update: {old_gcs_uri}")
                 renamer = PieceOfWork(
                     partial(
                         rename_gcs_blob,
@@ -220,9 +199,7 @@ def _run_metadata_migration(
                 gcs_tasks.schedule(renamer)
 
     # Migrate all assay upload successes
-    successful_assay_uploads = _select_successful_assay_uploads(
-        use_upload_jobs_table, session
-    )
+    successful_assay_uploads = _select_successful_assay_uploads(use_upload_jobs_table, session)
     for upload in successful_assay_uploads:
         print(f"Running metadata migration for assay upload: {upload.id}")
         if use_upload_jobs_table:
@@ -248,9 +225,7 @@ def _run_metadata_migration(
             if old_target_uri in migration.file_updates:
                 new_target_uri = migration.file_updates[old_target_uri]["object_url"]
                 if old_target_uri != new_target_uri:
-                    print(
-                        f"Encountered GCS upload bucket artifact URI to update: {old_upload_uri}"
-                    )
+                    print(f"Encountered GCS upload bucket artifact URI to update: {old_upload_uri}")
                     new_upload_uri = "/".join([new_target_uri, upload_timestamp])
                     renamer = PieceOfWork(
                         partial(
@@ -325,7 +300,5 @@ def republish_artifact_uploads():
     with migration_session() as (session, _):
         files = session.query(DownloadableFiles).all()
         for f in files:
-            print(
-                f"Publishing to 'artifact_upload' topic for downloadable file with in bucket url {f.object_url}"
-            )
+            print(f"Publishing to 'artifact_upload' topic for downloadable file with in bucket url {f.object_url}")
             publish_artifact_upload(f.object_url)

{nci_cidc_api_modules-1.2.17 → nci_cidc_api_modules-1.2.19}/cidc_api/models/models.py

@@ -3415,6 +3415,15 @@ class PreprocessedFiles(CommonColumns):
 
         return query.filter(cls.job_id.is_(None))
 
+    @with_default_session
+    def category_description(self, session: Session):
+        category = (
+            session.query(JobFileCategories)
+            .filter(JobFileCategories.job_id == self.job_id, JobFileCategories.category == self.file_category)
+            .first()
+        )
+        return category.description if category else None
+
 
 INGESTION_JOB_STATUSES = [
     "DRAFT",
@@ -3546,6 +3555,7 @@ class JobFileCategories(CommonColumns):
     )
 
     category = Column(String)
+    description = Column(String)
     job_id = Column(Integer, nullable=False)
     type = Column(Enum("required", "optional", name="type"), nullable=False)
     is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
@@ -3556,6 +3566,7 @@ class JobFileCategories(CommonColumns):
         category: str,
         job_id: int,
         type: str,
+        description: str = None,
         is_custom: bool = False,
         session: Session = None,
     ):
@@ -3563,6 +3574,7 @@ class JobFileCategories(CommonColumns):
             category=category,
             job_id=job_id,
             type=type,
+            description=description,
             is_custom=is_custom,
         )
         new_category.insert(session=session)
@@ -3607,6 +3619,17 @@ class CategoryDataElements(CommonColumns):
     def elements_for_category(cls, category_id: int, session: Session = None):
         return session.query(cls).filter_by(category_id=category_id).all()
 
+    @classmethod
+    @with_default_session
+    def elements_by_category_for_job(cls, job_id: int, session: Session = None):
+        """Fetch all CategoryDataElements for a job, along with the category name."""
+        return (
+            session.query(JobFileCategories.category, cls)
+            .join(cls, cls.category_id == JobFileCategories.id)
+            .filter(JobFileCategories.job_id == job_id)
+            .all()
+        )
+
 
 class FileValidationErrors(CommonColumns):
     __tablename__ = "file_validation_errors"