nci-cidc-api-modules 1.1.11__py3-none-any.whl → 1.1.13__py3-none-any.whl

cidc_api/models/models.py

@@ -28,6 +28,7 @@ import hashlib
 import os
 import re
 from collections import defaultdict
+from copy import deepcopy
 from datetime import datetime, timedelta
 from enum import Enum as EnumBaseClass
 from functools import wraps
@@ -77,6 +78,7 @@ from sqlalchemy import (
     or_,
     Table,
     MetaData,
+    true,
 )
 from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.engine import ResultProxy
@@ -92,10 +94,11 @@ from sqlalchemy.sql import (
     # instead of the sqlalchemy.sql versions we are importing here.  The solution is to
     # break up this giant file.
     and_ as sql_and,
-    # or_ as sql_or, # NOT USED
+    or_ as sql_or,
     # select, # ALREADY IMPORTED
     text,
 )
+from sqlalchemy.sql.elements import BooleanClauseList
 from sqlalchemy.sql.functions import coalesce
 from werkzeug.exceptions import BadRequest
 
@@ -184,11 +187,7 @@ class CommonColumns(BaseModel):  # type: ignore
             if hasattr(b, "__table__"):
                 columns_to_check.extend(b.__table__.columns)
 
-        ret = {
-            c.name: getattr(self, c.name)
-            for c in columns_to_check
-            if hasattr(self, c.name)
-        }
+        ret = {c.name: getattr(self, c.name) for c in columns_to_check if hasattr(self, c.name)}
         ret = {k: v for k, v in ret.items() if v is not None}
         return ret
 
@@ -269,16 +268,12 @@ class CommonColumns(BaseModel):  # type: ignore
         if sort_field:
             # Get the attribute from the class, in case this is a hybrid attribute
             sort_attribute = getattr(cls, sort_field)
-            field_with_dir = (
-                asc(sort_attribute) if sort_direction == "asc" else desc(sort_attribute)
-            )
+            field_with_dir = asc(sort_attribute) if sort_direction == "asc" else desc(sort_attribute)
             query = query.order_by(field_with_dir)
             if sort_field != "id":
                 # When sorting, need to guarantee unique order for offset/limit pagination to produce
                 # consistent results. Adding secondary "id" sort field to ensure unique order.
-                secondary_field_with_dir = (
-                    asc("id") if sort_direction == "asc" else desc("id")
-                )
+                secondary_field_with_dir = asc("id") if sort_direction == "asc" else desc("id")
                 query = query.order_by(secondary_field_with_dir)
 
         # Apply filter function
@@ -299,9 +294,7 @@ class CommonColumns(BaseModel):  # type: ignore
 
     @classmethod
     @with_default_session
-    def count_by(
-        cls, expr, session: Session, filter_: Callable[[Query], Query] = lambda q: q
-    ) -> Dict[str, int]:
+    def count_by(cls, expr, session: Session, filter_: Callable[[Query], Query] = lambda q: q) -> Dict[str, int]:
         """
         Return a dictionary mapping results of `expr` to the number of times each result
         occurs in the table related to this model. E.g., for the `UploadJobs` model,
@@ -326,9 +319,7 @@ class CommonColumns(BaseModel):  # type: ignore
         filter_: Callable[[Query], Query] = lambda q: q,
     ):
         """Get a list of distinct values for the given column."""
-        assert (
-            column_name in cls.__table__.columns.keys()
-        ), f"{cls.__tablename__} has no column {column_name}"
+        assert column_name in cls.__table__.columns.keys(), f"{cls.__tablename__} has no column {column_name}"
 
         base_query = session.query(getattr(cls, column_name))
         filtered_query = filter_(base_query)
@@ -342,9 +333,7 @@ class CommonColumns(BaseModel):  # type: ignore
     @classmethod
     def get_unique_columns(cls):
         """Get a list of all the unique columns in this table."""
-        return [
-            column for column in cls.__table__.c if column.unique or column.primary_key
-        ]
+        return [column for column in cls.__table__.c if column.unique or column.primary_key]
 
 
 class CIDCRole(EnumBaseClass):
@@ -392,6 +381,11 @@ class Users(CommonColumns):
         """Returns true if this user is an NCI Biobank user."""
         return self.role == CIDCRole.NCI_BIOBANK_USER.value
 
+    def is_admin_or_nci_user(self) -> bool:
+        """Returns true if this user is a CIDC admin or NCI Biobank user. These users
+        share full access to much of the system."""
+        return self.is_admin() or self.is_nci_user()
+
     def has_download_permissions(self) -> bool:
         """Returns false if this user is a Network Viewer or PACT User."""
         return self.role not in (
@@ -434,9 +428,7 @@ class Users(CommonColumns):
         user = Users.find_by_email(email)
         if not user:
             logger.info("Creating new user with email %s", email)
-            user = Users(
-                email=email, contact_email=email, first_n=first_n, last_n=last_n
-            )
+            user = Users(email=email, contact_email=email, first_n=first_n, last_n=last_n)
             user.insert(session=session)
         return user
 
@@ -450,9 +442,7 @@ class Users(CommonColumns):
         user_inactivity_cutoff = datetime.today() - timedelta(days=INACTIVE_USER_DAYS)
         update_query = (
             update(Users)
-            .where(
-                and_(Users._accessed < user_inactivity_cutoff, Users.disabled == False)
-            )
+            .where(and_(Users._accessed < user_inactivity_cutoff, Users.disabled == False))
             .values(disabled=True)
             .returning(Users.id)
         )
@@ -460,9 +450,7 @@ class Users(CommonColumns):
         if commit:
             session.commit()
 
-        disabled_users = [
-            Users.find_by_id(uid, session=session) for uid in disabled_user_ids
-        ]
+        disabled_users = [Users.find_by_id(uid, session=session) for uid in disabled_user_ids]
         for u in disabled_users:
             Permissions.revoke_user_permissions(u, session=session)
             revoke_bigquery_access(u.email)
@@ -501,15 +489,13 @@ class Users(CommonColumns):
             .union_all(
                 # Handle admins separately, since they can view all data for all
                 # trials even if they have no permissions assigned to them.
-                session.query(
-                    *user_columns, TrialMetadata.trial_id, literal("*,clinical_data")
-                ).filter(Users.role == CIDCRole.ADMIN.value)
+                session.query(*user_columns, TrialMetadata.trial_id, literal("*,clinical_data")).filter(
+                    Users.role == CIDCRole.ADMIN.value
+                )
             )
         )
 
-        df = pd.DataFrame(
-            query, columns=["email", "organization", "role", "trial_id", "permissions"]
-        ).fillna("*")
+        df = pd.DataFrame(query, columns=["email", "organization", "role", "trial_id", "permissions"]).fillna("*")
 
         with pd.ExcelWriter(
             io
@@ -630,19 +616,11 @@ class Permissions(CommonColumns):
 
         NOTE: values provided to the `commit` argument will be ignored. This method always commits.
         """
-        if (
-            self.upload_type == self.EVERY
-            and self.trial_id == self.EVERY
-            and not self.file_group_id
-        ):
-            raise ValueError(
-                "A permission must have a trial id, upload type, or file group."
-            )
+        if self.upload_type == self.EVERY and self.trial_id == self.EVERY and not self.file_group_id:
+            raise ValueError("A permission must have a trial id, upload type, or file group.")
 
         if self.file_group_id and self.upload_type != "file_group":
-            raise ValueError(
-                "If a permission has a file group, its upload_type must be set to file_group"
-            )
+            raise ValueError("If a permission has a file group, its upload_type must be set to file_group")
 
         grantee = Users.find_by_id(self.granted_to_user, session=session)
         if grantee is None:
@@ -724,11 +702,7 @@ class Permissions(CommonColumns):
         super().insert(session=session, commit=True)
 
         # Don't make any GCS changes if this user doesn't have download access, is disabled, or isn't approved
-        if (
-            not grantee.has_download_permissions()
-            or grantee.disabled
-            or grantee.approval_date is None
-        ):
+        if not grantee.has_download_permissions() or grantee.disabled or grantee.approval_date is None:
             # TODO: pact users do not have download permissions currently
             return
 
@@ -736,9 +710,7 @@ class Permissions(CommonColumns):
             # Grant ACL download permissions in GCS
 
             if self.upload_type == "file_group":
-                Permissions.grant_download_access_to_file_group(
-                    grantee.email, file_group
-                )
+                Permissions.grant_download_access_to_file_group(grantee.email, file_group)
             else:
 
                 # if they have any download permissions, they need the CIDC Lister role
@@ -746,9 +718,7 @@ class Permissions(CommonColumns):
                 grant_download_access(grantee.email, self.trial_id, self.upload_type)
                 # Remove permissions staged for deletion, if any
                 for perm in perms_to_delete:
-                    revoke_download_access(
-                        grantee.email, perm.trial_id, perm.upload_type
-                    )
+                    revoke_download_access(grantee.email, perm.trial_id, perm.upload_type)
         except Exception as e:
             # Add back deleted permissions, if any
             for perm in perms_to_delete:
@@ -760,9 +730,7 @@ class Permissions(CommonColumns):
             raise IAMException("IAM grant failed.") from e
 
     @with_default_session
-    def delete(
-        self, deleted_by: Union[Users, int], session: Session, commit: bool = True
-    ) -> None:
+    def delete(self, deleted_by: Union[Users, int], session: Session, commit: bool = True) -> None:
         """
         Delete this permission record from the database and revoke the corresponding IAM policy binding
         on the GCS data bucket.
@@ -795,9 +763,7 @@ class Permissions(CommonColumns):
                     revoke_lister_access(grantee.email)
 
             except Exception as e:
-                raise IAMException(
-                    "IAM revoke failed, and permission db record not removed."
-                ) from e
+                raise IAMException("IAM revoke failed, and permission db record not removed.") from e
 
         info_message = f"admin-action: {deleted_by_user.email} removed from {grantee.email} the permission {self.upload_type or 'all assays'} on {self.trial_id or 'all trials'}"
         logger.info(info_message)
@@ -841,10 +807,7 @@ class Permissions(CommonColumns):
                         # if getting EVERY, return all
                         | (upload_type == Permissions.EVERY)
                         # if permission is EVERY, don't return if looking for clinical_data
-                        | (
-                            (Permissions.upload_type == Permissions.EVERY)
-                            & (upload_type != "clinical_data")
-                        )
+                        | ((Permissions.upload_type == Permissions.EVERY) & (upload_type != "clinical_data"))
                     )
                 ),
             )
@@ -864,23 +827,16 @@ class Permissions(CommonColumns):
             permissions_list: List[Permissions] = []
             for upload in upload_type:
                 permissions_list.extend(
-                    Permissions.get_for_trial_type(
-                        trial_id=trial_id, upload_type=upload, session=session
-                    )
+                    Permissions.get_for_trial_type(trial_id=trial_id, upload_type=upload, session=session)
                 )
 
-        permissions_dict: Dict[str, Dict[str, List[Permissions]]] = defaultdict(
-            lambda: defaultdict(list)
-        )
+        permissions_dict: Dict[str, Dict[str, List[Permissions]]] = defaultdict(lambda: defaultdict(list))
         for perm in permissions_list:
             permissions_dict[perm.trial_id][perm.upload_type].append(perm)
 
         user_dict: Dict[str, Dict[str, List[Users]]] = {
             trial: {
-                upload: [
-                    Users.find_by_id(id=perm.granted_to_user, session=session)
-                    for perm in perms
-                ]
+                upload: [Users.find_by_id(id=perm.granted_to_user, session=session) for perm in perms]
                 for upload, perms in upload_dict.items()
             }
             for trial, upload_dict in permissions_dict.items()
@@ -895,11 +851,7 @@ class Permissions(CommonColumns):
             for trial, upload_dict in user_dict.items()
         }
         # remove any trial that doesn't have any uploads in it
-        user_email_dict = {
-            trial: upload_dict
-            for trial, upload_dict in user_email_dict.items()
-            if len(upload_dict)
-        }
+        user_email_dict = {trial: upload_dict for trial, upload_dict in user_email_dict.items() if len(upload_dict)}
         return user_email_dict
 
     @staticmethod
@@ -919,14 +871,8 @@ class Permissions(CommonColumns):
             session.query(Permissions)
             .filter(
                 Permissions.granted_to_user == user_id,
-                (
-                    (Permissions.trial_id == trial_id)
-                    & (Permissions.upload_type == upload_type)
-                )
-                | (
-                    (Permissions.trial_id == Permissions.EVERY)
-                    & (Permissions.upload_type == upload_type)
-                )
+                ((Permissions.trial_id == trial_id) & (Permissions.upload_type == upload_type))
+                | ((Permissions.trial_id == Permissions.EVERY) & (Permissions.upload_type == upload_type))
                 | (
                     (Permissions.trial_id == trial_id)
                     # if permission is EVERY, don't return if looking for clinical_data
@@ -949,11 +895,7 @@ class Permissions(CommonColumns):
             )
             if results:
                 file_group_ids = {file_group.id for file_group in file_groups}
-                results = [
-                    result
-                    for result in results
-                    if result.file_group_id in file_group_ids
-                ]
+                results = [result for result in results if result.file_group_id in file_group_ids]
 
         results = results and results or None
         return results
@@ -997,9 +939,7 @@ class Permissions(CommonColumns):
                 upload_type=[p.upload_type for p in trial_perms],
             )
         for perm in file_group_perms:
-            file_group: FileGroups = FileGroups.find_by_id(
-                perm.file_group_id, session=session
-            )
+            file_group: FileGroups = FileGroups.find_by_id(perm.file_group_id, session=session)
             Permissions.grant_download_access_to_file_group(user.email, file_group)
 
         # Regrant all of the user's intake bucket upload permissions, if they have any
@@ -1039,9 +979,7 @@ class Permissions(CommonColumns):
 
     @classmethod
     @with_default_session
-    def grant_download_permissions_for_upload_job(
-        cls, upload: "UploadJobs", session: Session
-    ) -> None:
+    def grant_download_permissions_for_upload_job(cls, upload: "UploadJobs", session: Session) -> None:
         """
         For a given UploadJob, issue all relevant Permissions on Google
         Loads all cross-trial permissions for the upload_type
@@ -1059,21 +997,14 @@ class Permissions(CommonColumns):
             filters.append(cls.upload_type == upload.upload_type)
         else:
             # upload.upload_type can't be None
-            filters.append(
-                or_(cls.upload_type == upload.upload_type, cls.upload_type == None)
-            )
+            filters.append(or_(cls.upload_type == upload.upload_type, cls.upload_type == None))
 
         perms = session.query(cls).filter(*filters).all()
         user_email_list: List[str] = []
 
         for perm in perms:
             user = Users.find_by_id(perm.granted_to_user, session=session)
-            if (
-                user.is_admin()
-                or user.is_nci_user()
-                or user.disabled
-                or user.email in user_email_list
-            ):
+            if user.is_admin() or user.is_nci_user() or user.disabled or user.email in user_email_list:
                 continue
 
             user_email_list.append(user.email)
@@ -1102,10 +1033,8 @@ class ValidationMultiError(Exception):
     """Holds multiple jsonschema.ValidationErrors"""
 
 
-trial_metadata_validator: json_validation._Validator = (
-    json_validation.load_and_validate_schema(
-        "clinical_trial.json", return_validator=True
-    )
+trial_metadata_validator: json_validation._Validator = json_validation.load_and_validate_schema(
+    "clinical_trial.json", return_validator=True
 )
 
 FileBundle = Dict[str, Dict[FilePurpose, List[int]]]
@@ -1121,13 +1050,13 @@ class TrialMetadata(CommonColumns):
     _metadata_idx = Index("metadata_idx", metadata_json, postgresql_using="gin")
 
     @staticmethod
-    def validate_metadata_json(metadata_json: dict) -> dict:
-        # Prior to running trial_metadata_validator.iter_error_messages on the metadata_json,
-        # strip out unnecessary manifest data for the validation so that existing manifest data
-        # that no longer conform to the post-CSMS-integration schema can be kept.
+    def validate_metadata_json(metadata_json: dict, strip_metadata=True) -> dict:
+        # Prior to running trial_metadata_validator.iter_error_messages on the metadata_json, if
+        # strip_metadata=True, will strip out unnecessary manifest data for the validation so that
+        # existing manifest data that no longer conform to the post-CSMS-integration schema can be kept.
         # See more details in the strip_metadata_for_validation function docs.
-        metadata_to_validate = json_validation.strip_metadata_for_validation(
-            metadata_json
+        metadata_to_validate = (
+            json_validation.strip_metadata_for_validation(metadata_json) if strip_metadata else metadata_json
         )
         errs = trial_metadata_validator.iter_error_messages(metadata_to_validate)
         messages = list(f"'metadata_json': {err}" for err in errs)
@@ -1163,49 +1092,34 @@ class TrialMetadata(CommonColumns):
         Find a trial by its CIMAC id.
         """
         try:
-            trial = (
-                session.query(TrialMetadata)
-                .filter_by(trial_id=trial_id)
-                .with_for_update()
-                .one()
-            )
+            trial = session.query(TrialMetadata).filter_by(trial_id=trial_id).with_for_update().one()
         except NoResultFound as e:
             raise NoResultFound(f"No trial found with id {trial_id}") from e
         return trial
 
     @staticmethod
     @with_default_session
-    def patch_assays(
-        trial_id: str, assay_patch: dict, session: Session, commit: bool = False
-    ):
+    def patch_assays(trial_id: str, assay_patch: dict, session: Session, commit: bool = False):
         """
         Applies assay updates to the metadata object from the trial with id `trial_id`.
 
         TODO: apply this update directly to the not-yet-existent TrialMetadata.manifest field
         """
-        return TrialMetadata._patch_trial_metadata(
-            trial_id, assay_patch, session=session, commit=commit
-        )
+        return TrialMetadata._patch_trial_metadata(trial_id, assay_patch, session=session, commit=commit)
 
     @staticmethod
     @with_default_session
-    def patch_manifest(
-        trial_id: str, manifest_patch: dict, session: Session, commit: bool = False
-    ):
+    def patch_manifest(trial_id: str, manifest_patch: dict, session: Session, commit: bool = False):
         """
         Applies manifest updates to the metadata object from the trial with id `trial_id`.
 
         TODO: apply this update directly to the not-yet-existent TrialMetadata.assays field
         """
-        return TrialMetadata._patch_trial_metadata(
-            trial_id, manifest_patch, session=session, commit=commit
-        )
+        return TrialMetadata._patch_trial_metadata(trial_id, manifest_patch, session=session, commit=commit)
 
     @staticmethod
     @with_default_session
-    def _patch_trial_metadata(
-        trial_id: str, json_patch: dict, session: Session, commit: bool = False
-    ):
+    def _patch_trial_metadata(trial_id: str, json_patch: dict, session: Session, commit: bool = False):
         """
         Applies updates to the metadata object from the trial with id `trial_id`
         and commits current session.
@@ -1217,9 +1131,7 @@ class TrialMetadata(CommonColumns):
         trial = TrialMetadata.select_for_update_by_trial_id(trial_id, session=session)
 
         # Merge assay metadata into the existing clinical trial metadata
-        updated_metadata, errs = prism.merge_clinical_trial_metadata(
-            json_patch, trial.metadata_json
-        )
+        updated_metadata, errs = prism.merge_clinical_trial_metadata(json_patch, trial.metadata_json)
         if errs:
             raise ValidationMultiError(errs)
         # Save updates to trial record
@@ -1234,9 +1146,7 @@ class TrialMetadata(CommonColumns):
 
     @staticmethod
     @with_default_session
-    def create(
-        trial_id: str, metadata_json: dict, session: Session, commit: bool = True
-    ):
+    def create(trial_id: str, metadata_json: dict, session: Session, commit: bool = True):
         """
         Create a new clinical trial metadata record.
         """
@@ -1248,9 +1158,7 @@ class TrialMetadata(CommonColumns):
         return trial
 
     @staticmethod
-    def merge_gcs_artifact(
-        metadata: dict, upload_type: str, uuid: str, gcs_object: Blob
-    ):
+    def merge_gcs_artifact(metadata: dict, upload_type: str, uuid: str, gcs_object: Blob):
         return prism.merge_artifact(
             ct=metadata,
             assay_type=upload_type,  # assay_type is the old name for upload_type
@@ -1263,9 +1171,7 @@ class TrialMetadata(CommonColumns):
         )
 
     @staticmethod
-    def merge_gcs_artifacts(
-        metadata: dict, upload_type: str, uuids_and_gcs_objects: List[Tuple[str, Blob]]
-    ):
+    def merge_gcs_artifacts(metadata: dict, upload_type: str, uuids_and_gcs_objects: List[Tuple[str, Blob]]):
         return prism.merge_artifacts(
             metadata,
             [
@@ -1354,32 +1260,16 @@ class TrialMetadata(CommonColumns):
         subqueries = []
 
         if include_file_bundles:
-            allowed_upload_types = []
-            if user and not user.is_admin() and not user.is_nci_user():
-                permissions = Permissions.find_for_user(user.id)
-                # An 'empty' upload_type means full trial-level access
-                allowed_upload_types = [
-                    p.upload_type for p in permissions if p.upload_type
-                ]
-                logger.info(
-                    f"Restricting file bundle for user {user.id} to {allowed_upload_types=}"
-                )
+            file_bundle_query = DownloadableFiles.build_file_bundle_query()
 
-            file_bundle_query = DownloadableFiles.build_file_bundle_query(
-                allowed_upload_types
-            )
             columns.append(file_bundle_query.c.file_bundle)
             subqueries.append(file_bundle_query)
 
         if include_counts:
             trial_summaries: List[dict] = cls.get_summaries()
 
-            participant_counts: Dict[str, int] = {
-                t["trial_id"]: t["total_participants"] for t in trial_summaries
-            }
-            sample_counts: Dict[str, int] = {
-                t["trial_id"]: t["total_samples"] for t in trial_summaries
-            }
+            participant_counts: Dict[str, int] = {t["trial_id"]: t["total_participants"] for t in trial_summaries}
+            sample_counts: Dict[str, int] = {t["trial_id"]: t["total_samples"] for t in trial_summaries}
 
         # Combine all query components
         query = session.query(*columns)
@@ -1387,6 +1277,7 @@ class TrialMetadata(CommonColumns):
             # Each subquery will have a trial_id column and one record per trial id
             query = query.outerjoin(subquery, cls.trial_id == subquery.c.trial_id)
 
+        query = query.order_by(cls.trial_id)
         query = cls._add_pagination_filters(query, **pagination_args)
 
         trials = []
@@ -1402,25 +1293,43 @@ class TrialMetadata(CommonColumns):
                     setattr(trial, column, value)
 
             if include_counts:
-                setattr(
-                    trial, "num_participants", participant_counts.get(trial.trial_id, 0)
-                )
+                setattr(trial, "num_participants", participant_counts.get(trial.trial_id, 0))
                 setattr(trial, "num_samples", sample_counts.get(trial.trial_id, 0))
 
             if include_file_bundles and hasattr(trial, "file_bundle"):
-                for assay in trial.file_bundle:
+                # File bundle has all existing object ids. Remove ones that aren't allowed by permissions.
+
+                # Gather all object ids in the file bundle
+                all_object_ids = set()
+
+                for assay, purposes in trial.file_bundle.items():
+                    for purpose, object_ids in purposes.items():
+                        all_object_ids = all_object_ids.union(object_ids)
+                # Remove any impermissible object ids
+                filtered_object_ids = DownloadableFiles.filter_object_ids_by_permissions(user, all_object_ids)
+                logger.debug(f"Filtered object ids: {len(all_object_ids)} -> {len(filtered_object_ids)}")
+
+                for assay, purposes in trial.file_bundle.items():
                     size_results = {}
-                    for files_list_key in trial.file_bundle[assay]:
-                        ids = trial.file_bundle[assay][files_list_key]
-                        size_results[f"{files_list_key}_size"] = (
-                            DownloadableFiles.get_total_bytes(
-                                filter_=lambda q: q.filter(
-                                    DownloadableFiles.id.in_(ids)
-                                )
-                            )
-                        )
+                    for purpose, object_ids in purposes.items():
+                        # Only allow object ids that are permitted
+                        permitted_object_ids = list(set(object_ids).intersection(filtered_object_ids))
+                        trial.file_bundle[assay][purpose] = permitted_object_ids
+                        if permitted_object_ids:
+                            # For any files left in the purpose, get their total size
+                            filter_ = lambda q: q.filter(DownloadableFiles.id.in_(permitted_object_ids))
+                            size_results[f"{purpose}_size"] = DownloadableFiles.get_total_bytes(filter_=filter_)
                     trial.file_bundle[assay].update(size_results)
 
+                # Trim the file bundle
+                for assay, purposes in deepcopy(trial.file_bundle).items():
+                    for purpose, object_ids in purposes.items():
+                        if not object_ids:
+                            # No file ids left in the purpose after filtering. Remove this purpose from the bundle.
+                            del trial.file_bundle[assay][purpose]
+                    if not trial.file_bundle[assay]:
+                        del trial.file_bundle[assay]
+
             trials.append(trial)
 
         return trials
@@ -2017,9 +1926,7 @@ class TrialMetadata(CommonColumns):
         summaries_query = "SELECT result FROM trial_summaries_mv"
         # Retrieve trial-level summary results from data cached in trial_summaries_mv materialized view.
         # The source of the SQL query used in trial_summaries_mv is get_summaries_query()
-        summaries = [
-            summary for (summary,) in session.execute(summaries_query) if summary
-        ]
+        summaries = [summary for (summary,) in session.execute(summaries_query) if summary]
 
         # Shortcut to impute 0 values for assays where trials don't yet have data
         summaries = pd.DataFrame(summaries).fillna(0).to_dict("records")
@@ -2037,9 +1944,7 @@ class UploadJobStatus(EnumBaseClass):
     MERGE_FAILED = "merge-failed"
 
     @classmethod
-    def is_valid_transition(
-        cls, current: str, target: str, is_manifest: bool = False
-    ) -> bool:
+    def is_valid_transition(cls, current: str, target: str, is_manifest: bool = False) -> bool:
         """
         Enforce logic about which state transitions are valid. E.g.,
         an upload whose status is "merge-completed" should never be updated
@@ -2084,9 +1989,7 @@ class UploadJobs(CommonColumns):
     )
 
     # The current status of the upload job
-    _status = Column(
-        "status", Enum(*UPLOAD_STATUSES, name="upload_job_status"), nullable=False
-    )
+    _status = Column("status", Enum(*UPLOAD_STATUSES, name="upload_job_status"), nullable=False)
     # A long, random identifier for this upload job
     token = Column(UUID, server_default=text("gen_random_uuid()"), nullable=False)
     # Text containing feedback on why the upload status is what it is
@@ -2108,9 +2011,7 @@ class UploadJobs(CommonColumns):
     trial_id = Column(String, nullable=False, index=True)
 
     # Create a GIN index on the GCS object names
-    _gcs_objects_idx = Index(
-        "upload_jobs_gcs_gcs_file_map_idx", gcs_file_map, postgresql_using="gin"
-    )
+    _gcs_objects_idx = Index("upload_jobs_gcs_gcs_file_map_idx", gcs_file_map, postgresql_using="gin")
 
     @hybrid_property
     def status(self):
@@ -2124,9 +2025,7 @@ class UploadJobs(CommonColumns):
         old_status = self.status or UploadJobStatus.STARTED.value
         is_manifest = self.upload_type in prism.SUPPORTED_MANIFESTS
         if not UploadJobStatus.is_valid_transition(old_status, status, is_manifest):
-            raise ValueError(
-                f"Upload job with status {self.status} can't transition to status {status}"
-            )
+            raise ValueError(f"Upload job with status {self.status} can't transition to status {status}")
         self._status = status
 
     def _set_status_no_validation(self, status: str):
@@ -2166,9 +2065,7 @@ class UploadJobs(CommonColumns):
         assert prism.PROTOCOL_ID_FIELD_NAME in metadata, "metadata must have a trial ID"
 
         is_manifest_upload = upload_type in prism.SUPPORTED_MANIFESTS
-        assert (
-            gcs_file_map is not None or is_manifest_upload
-        ), "assay/analysis uploads must have a gcs_file_map"
+        assert gcs_file_map is not None or is_manifest_upload, "assay/analysis uploads must have a gcs_file_map"
 
         trial_id = metadata[prism.PROTOCOL_ID_FIELD_NAME]
 
@@ -2219,9 +2116,7 @@ class UploadJobs(CommonColumns):
                 job.metadata_patch,
                 updated_artifact,
                 _,
-            ) = prism.merge_artifact_extra_metadata(
-                job.metadata_patch, uuid, job.upload_type, file
-            )
+            ) = prism.merge_artifact_extra_metadata(job.metadata_patch, uuid, job.upload_type, file)
             logger.info("Updated md for %s: %s", uuid, updated_artifact.keys())
 
         # A workaround fix for JSON field modifications not being tracked
@@ -2243,25 +2138,14 @@ class UploadJobs(CommonColumns):
     @classmethod
     @with_default_session
     def find_first_manifest_job(cls, trial_id: str, session):
-        return (
-            session.query(UploadJobs)
-            .filter_by(trial_id=trial_id, gcs_xlsx_uri="")
-            .order_by(text("id ASC"))
-            .first()
-        )
+        return session.query(UploadJobs).filter_by(trial_id=trial_id, gcs_xlsx_uri="").order_by(text("id ASC")).first()
 
     @with_default_session
-    def ingestion_success(
-        self, trial, session: Session, commit: bool = False, send_email: bool = False
-    ):
+    def ingestion_success(self, trial, session: Session, commit: bool = False, send_email: bool = False):
         """Set own status to reflect successful merge and trigger email notifying CIDC admins."""
         # Do status update if the transition is valid
-        if not UploadJobStatus.is_valid_transition(
-            self.status, UploadJobStatus.MERGE_COMPLETED.value
-        ):
-            raise Exception(
-                f"Cannot declare ingestion success given current status: {self.status}"
-            )
+        if not UploadJobStatus.is_valid_transition(self.status, UploadJobStatus.MERGE_COMPLETED.value):
+            raise Exception(f"Cannot declare ingestion success given current status: {self.status}")
         self.status = UploadJobStatus.MERGE_COMPLETED.value
 
         if commit:
@@ -2277,11 +2161,7 @@ class FilesToFileGroups(BaseModel):
     """
 
     __tablename__ = "files_to_file_groups"
-    __table_args__ = (
-        PrimaryKeyConstraint(
-            "file_group_id", "file_id", name="pk_files_to_file_groups"
-        ),
-    )
+    __table_args__ = (PrimaryKeyConstraint("file_group_id", "file_id", name="pk_files_to_file_groups"),)
     file_group_id = Column(ForeignKey("file_groups.id"), primary_key=True)
     file_id = Column(ForeignKey("downloadable_files.id"), primary_key=True)
     _created = Column(DateTime, default=func.now(), nullable=False)
@@ -2504,9 +2384,7 @@ class DownloadableFiles(CommonColumns):
         return downloadable_files_for_query, files_to_file_groups_for_query
 
     @classmethod
-    def _convert_list_results(
-        cls, downloadable_files_for_query: Table, query_files: List
-    ):
+    def _convert_list_results(cls, downloadable_files_for_query: Table, query_files: List):
         """Converts the results of a SQLalchemy expression language query into actual DownloadableFiles
         objects.  This is necessary since the UI depends on some of the derived properties in
         DownloadableFiles.
@@ -2544,9 +2422,7 @@ class DownloadableFiles(CommonColumns):
             where_clauses.append(downloadable_files_for_query.c.trial_id.in_(trial_ids))
         if facets:
             facet_groups = get_facet_groups_for_paths(facets)
-            where_clauses.append(
-                downloadable_files_for_query.c.facet_group.in_(facet_groups)
-            )
+            where_clauses.append(downloadable_files_for_query.c.facet_group.in_(facet_groups))
 
         if user and not is_admin:
             permissions = Permissions.find_for_user(user.id)
@@ -2562,27 +2438,15 @@ class DownloadableFiles(CommonColumns):
                 elif permission.file_group_id is None:
                     where_clauses.append(
                         sql_and(
-                            (
-                                downloadable_files_for_query.c.trial_id
-                                == permission.trial_id
-                            ),
-                            (
-                                downloadable_files_for_query.c.upload_type
-                                == permission.upload_type
-                            ),
+                            (downloadable_files_for_query.c.trial_id == permission.trial_id),
+                            (downloadable_files_for_query.c.upload_type == permission.upload_type),
                         )
                     )
                 else:
                     where_clauses.append(
                         sql_and(
-                            (
-                                downloadable_files_for_query.c.trial_id
-                                == permission.trial_id
-                            ),
-                            (
-                                files_to_file_groups_for_query.c.file_group_id
-                                == permission.file_group_id
-                            ),
+                            (downloadable_files_for_query.c.trial_id == permission.trial_id),
+                            (files_to_file_groups_for_query.c.file_group_id == permission.file_group_id),
                         )
                     )
 
@@ -2596,9 +2460,7 @@ class DownloadableFiles(CommonColumns):
                     )
                 )
             if full_type_perms:
-                where_clauses.append(
-                    downloadable_files_for_query.c.upload_type.in_(full_type_perms)
-                )
+                where_clauses.append(downloadable_files_for_query.c.upload_type.in_(full_type_perms))
 
         # Need to be careful about return logic.  Empty results could be because the user
         # is an admin, whereas None means the user has no permissions to view any files.
@@ -2607,6 +2469,55 @@ class DownloadableFiles(CommonColumns):
 
         return None
 
+    @classmethod
+    def _generate_where_clause_with_permissions(cls, user: Users) -> BooleanClauseList:
+        """
+        Returns a where clause for DownloadableFiles filtered down to only the files the user
+        has access to based on their permissions and role.
+
+        The generated clause will have this form
+            WHERE
+                downloadable_files.trial_id IN ('x', 'y', ...) AND upload_type != 'clinical_data' <- trial-level permissions
+            OR
+                upload_type IN ('u', 'v', ...)) <- upload-type-level permissions
+            OR
+                trial_id = '5' AND upload_type = 'mif' <- regular permissions
+            OR
+                trial_id = '6' and upload_type = 'hande'
+                ...
+        """
+        # From the perspective of viewing files, NCI Biobank users are admins.
+        if user.is_admin_or_nci_user():
+            return true()  # Admin has full permissions to all
+
+        permissions = Permissions.find_for_user(user.id)
+
+        full_access_trial_ids = [p.trial_id for p in permissions if not p.upload_type]
+        full_access_upload_types = [p.upload_type for p in permissions if not p.trial_id]
+        regular_permissions = [p for p in permissions if p.trial_id and p.upload_type]
+
+        full_access_trial_clause = sql_and(
+            DownloadableFiles.trial_id.in_(full_access_trial_ids),
+            DownloadableFiles.upload_type != "clinical_data",
+        )
+
+        full_access_upload_type_clause = sql_or(DownloadableFiles.upload_type.in_(full_access_upload_types))
+
+        regular_permission_clauses = [
+            sql_and(
+                DownloadableFiles.trial_id == p.trial_id,
+                DownloadableFiles.upload_type == p.upload_type,
+            )
+            for p in regular_permissions
+        ]
+        clause = sql_or(
+            full_access_trial_clause,
+            full_access_upload_type_clause,
+            *regular_permission_clauses,
+        )
+
+        return clause
+
     @classmethod
     @with_default_session
     def list_with_permissions(
@@ -2646,26 +2557,18 @@ class DownloadableFiles(CommonColumns):
         if where_clauses:
 
             # No where clause (the user is likely an admin).
-            statement = select([downloadable_files_for_query]).select_from(
-                downloadable_files_for_query
-            )
+            statement = select([downloadable_files_for_query]).select_from(downloadable_files_for_query)
 
         else:
             statement = (
                 select([downloadable_files_for_query])
                 .where(sql_and(*where_clauses))
-                .select_from(
-                    downloadable_files_for_query.outerjoin(
-                        files_to_file_groups_for_query
-                    )
-                )
+                .select_from(downloadable_files_for_query.outerjoin(files_to_file_groups_for_query))
             )
 
         if sort_field:
             sort_attribute = getattr(cls, sort_field)
-            field_with_dir = (
-                asc(sort_attribute) if sort_direction == "asc" else desc(sort_attribute)
-            )
+            field_with_dir = asc(sort_attribute) if sort_direction == "asc" else desc(sort_attribute)
             statement = statement.order_by(field_with_dir)
 
         # Enforce positive page numbers
@@ -2722,28 +2625,22 @@ class DownloadableFiles(CommonColumns):
         if where_clauses:
 
             # No where clause (the user is likely an admin).
-            statement = select(
-                [func.count(downloadable_files_for_query.c.id)]
-            ).select_from(downloadable_files_for_query)
+            statement = select([func.count(downloadable_files_for_query.c.id)]).select_from(
+                downloadable_files_for_query
+            )
 
         else:
             statement = (
                 select([func.count(downloadable_files_for_query.c.id)])
                 .where(sql_and(*where_clauses))
-                .select_from(
-                    downloadable_files_for_query.outerjoin(
-                        files_to_file_groups_for_query
-                    )
-                )
+                .select_from(downloadable_files_for_query.outerjoin(files_to_file_groups_for_query))
             )
 
         return session.execute(statement).fetchone()[0]
 
     @classmethod
     @with_default_session
-    def count_by_facet_with_permissions(
-        cls, session: Session, trial_ids: List[str] = None, user: Users = None
-    ):
+    def count_by_facet_with_permissions(cls, session: Session, trial_ids: List[str] = None, user: Users = None):
         """
         Returns a map of facet_group to a count of the number of files that the given user
         has permissions to view.
@@ -2787,11 +2684,7 @@ class DownloadableFiles(CommonColumns):
                     ]
                 )
                 .where(sql_and(*where_clauses))
-                .select_from(
-                    downloadable_files_for_query.outerjoin(
-                        files_to_file_groups_for_query
-                    )
-                )
+                .select_from(downloadable_files_for_query.outerjoin(files_to_file_groups_for_query))
             )
 
         statement = statement.group_by(downloadable_files_for_query.c.facet_group)
@@ -2835,35 +2728,35 @@ class DownloadableFiles(CommonColumns):
         else:
             statement = (
                 select([downloadable_files_for_query.c.object_url])
-                .where(
-                    sql_and(*where_clauses, downloadable_files_for_query.c.id.in_(ids))
-                )
-                .select_from(
-                    downloadable_files_for_query.outerjoin(
-                        files_to_file_groups_for_query
-                    )
-                )
+                .where(sql_and(*where_clauses, downloadable_files_for_query.c.id.in_(ids)))
+                .select_from(downloadable_files_for_query.outerjoin(files_to_file_groups_for_query))
             )
 
         return [row[0] for row in session.execute(statement).fetchall()]
 
     @classmethod
-    def _generate_trial_file_counts(
-        cls, downloadable_files: Iterable
-    ) -> Dict[str, int]:
+    @with_default_session
+    def filter_object_ids_by_permissions(cls, user: Users, ids: Iterable[int], session: Session) -> Iterable[int]:
+        """
+        Takes a list of object ids and filters it to return only those object ids the user has permission for.
+        """
+
+        where_clause = DownloadableFiles._generate_where_clause_with_permissions(user)
+        statement = select([DownloadableFiles.id]).where(sql_and(DownloadableFiles.id.in_(ids), where_clause))
+
+        return [row[0] for row in session.execute(statement).fetchall()]
+
+    @classmethod
+    def _generate_trial_file_counts(cls, downloadable_files: Iterable) -> Dict[str, int]:
         results = defaultdict(lambda: 0)
         for downloadable_file in downloadable_files:
             if downloadable_file.data_category:
-                results[downloadable_file.trial_id] = (
-                    results[downloadable_file.trial_id] + 1
-                )
+                results[downloadable_file.trial_id] = results[downloadable_file.trial_id] + 1
         return results
 
     @classmethod
     @with_default_session
-    def remove_participants_and_samples_info_files(
-        cls, trial_id: str, session: Session
-    ):
+    def remove_participants_and_samples_info_files(cls, trial_id: str, session: Session):
         """
         Remove participants info and samples info downloadable files
         """
@@ -2911,27 +2804,19 @@ class DownloadableFiles(CommonColumns):
         if not where_clauses:
 
             # No where clause (the user is likely an admin).
-            statement = select([downloadable_files_for_query]).select_from(
-                downloadable_files_for_query
-            )
+            statement = select([downloadable_files_for_query]).select_from(downloadable_files_for_query)
 
         else:
             statement = (
                 select([downloadable_files_for_query])
                 .where(sql_and(*where_clauses))
-                .select_from(
-                    downloadable_files_for_query.outerjoin(
-                        files_to_file_groups_for_query
-                    )
-                )
+                .select_from(downloadable_files_for_query.outerjoin(files_to_file_groups_for_query))
             )
 
         downloadable_files = DownloadableFiles._convert_list_results(
             downloadable_files_for_query, session.execute(statement).fetchall()
         )
-        trial_file_counts = DownloadableFiles._generate_trial_file_counts(
-            downloadable_files
-        )
+        trial_file_counts = DownloadableFiles._generate_trial_file_counts(downloadable_files)
         return build_trial_facets(trial_file_counts)
 
     @with_default_session
@@ -2956,13 +2841,9 @@ class DownloadableFiles(CommonColumns):
                 "trial_id": self.trial_id,
                 "id": self.id,
             }
-            related_files = result_proxy_to_models(
-                session.execute(query, params), DownloadableFiles
-            )
+            related_files = result_proxy_to_models(session.execute(query, params), DownloadableFiles)
         else:
-            not_sample_specific = not_(
-                literal_column("additional_metadata::text").like('%.cimac_id":%')
-            )
+            not_sample_specific = not_(literal_column("additional_metadata::text").like('%.cimac_id":%'))
             related_files = (
                 session.query(DownloadableFiles)
                 .filter(
@@ -3018,9 +2899,7 @@ class DownloadableFiles(CommonColumns):
                     full_type_perms.append(perm.upload_type)
                 else:
                     trial_type_perms.append((perm.trial_id, perm.upload_type))
-            df_tuples = tuple_(
-                DownloadableFiles.trial_id, DownloadableFiles.upload_type
-            )
+            df_tuples = tuple_(DownloadableFiles.trial_id, DownloadableFiles.upload_type)
             file_filters.append(
                 or_(
                     # don't include clinical_data in cross-trial permission
@@ -3069,16 +2948,9 @@ class DownloadableFiles(CommonColumns):
         etag = make_etag(filtered_metadata.values())
 
         object_url = filtered_metadata["object_url"]
-        df = (
-            session.query(DownloadableFiles)
-            .filter_by(object_url=object_url)
-            .with_for_update()
-            .first()
-        )
+        df = session.query(DownloadableFiles).filter_by(object_url=object_url).with_for_update().first()
         if df:
-            df = session.merge(
-                DownloadableFiles(id=df.id, _etag=etag, **filtered_metadata)
-            )
+            df = session.merge(DownloadableFiles(id=df.id, _etag=etag, **filtered_metadata))
         else:
             df = DownloadableFiles(_etag=etag, **filtered_metadata)
 
@@ -3107,12 +2979,7 @@ class DownloadableFiles(CommonColumns):
         """
 
         # trying to find existing one
-        df = (
-            session.query(DownloadableFiles)
-            .filter_by(object_url=blob.name)
-            .with_for_update()
-            .first()
-        )
+        df = session.query(DownloadableFiles).filter_by(object_url=blob.name).with_for_update().first()
         if not df:
             df = DownloadableFiles()
 
@@ -3145,18 +3012,14 @@ class DownloadableFiles(CommonColumns):
 
     @classmethod
     @with_default_session
-    def list_object_urls(
-        cls, ids: List[int], session: Session, filter_: Callable[[Query], Query]
-    ) -> List[str]:
+    def list_object_urls(cls, ids: List[int], session: Session, filter_: Callable[[Query], Query]) -> List[str]:
         """Get all object_urls for a batch of downloadable file record IDs"""
         query = session.query(cls.object_url).filter(cls.id.in_(ids))
         query = filter_(query)
         return [r[0] for r in query.all()]
 
     @classmethod
-    def build_file_bundle_query(
-        cls, allowed_upload_types: Optional[List[str]]
-    ) -> Query:
+    def build_file_bundle_query(cls) -> Query:
         """
         Build a query that selects nested file bundles from the downloadable files table.
         The `file_bundles` query below should produce one bundle per unique `trial_id` that
@@ -3171,8 +3034,6 @@ class DownloadableFiles(CommonColumns):
           }
         ```
         where "type" is something like `"Olink"` or `"Participants Info"` and "purpose" is a `FilePurpose` string.
-
-        If `allowed_upload_types` is provided, the query will filter by files that only have an `upload_type` that appear in the list.
         """
         tid_col, type_col, purp_col, ids_col, purps_col = (
             literal_column("trial_id"),
@@ -3182,28 +3043,24 @@ class DownloadableFiles(CommonColumns):
             literal_column("purposes"),
         )
 
-        id_bundles = select(
-            [
-                cls.trial_id,
-                cls.data_category_prefix.label(type_col.key),
-                cls.file_purpose.label(purp_col.key),
-                func.json_agg(cls.id).label(ids_col.key),
-            ]
-        ).group_by(cls.trial_id, cls.data_category_prefix, cls.file_purpose)
-
-        # Restrict files from appearing in the file bundle if the user doesn't have permissions for them
-        if allowed_upload_types:
-            id_bundles = id_bundles.filter(cls.upload_type.in_(allowed_upload_types))
-        id_bundles = id_bundles.alias("id_bundles")
-
+        id_bundles = (
+            select(
+                [
+                    cls.trial_id,
+                    cls.data_category_prefix.label(type_col.key),
+                    cls.file_purpose.label(purp_col.key),
+                    func.json_agg(cls.id).label(ids_col.key),
+                ]
+            )
+            .group_by(cls.trial_id, cls.data_category_prefix, cls.file_purpose)
+            .alias("id_bundles")
+        )
         purpose_bundles = (
             select(
                 [
                     tid_col,
                     type_col,
-                    func.json_object_agg(
-                        func.coalesce(purp_col, "miscellaneous"), ids_col
-                    ).label(purps_col.key),
+                    func.json_object_agg(func.coalesce(purp_col, "miscellaneous"), ids_col).label(purps_col.key),
                 ]
             )
             .select_from(id_bundles)
@@ -3214,9 +3071,7 @@ class DownloadableFiles(CommonColumns):
             select(
                 [
                     tid_col.label(tid_col.key),
-                    func.json_object_agg(
-                        func.coalesce(type_col, "other"), purps_col
-                    ).label("file_bundle"),
+                    func.json_object_agg(func.coalesce(type_col, "other"), purps_col).label("file_bundle"),
                 ]
             )
             .select_from(purpose_bundles)
@@ -3227,9 +3082,7 @@ class DownloadableFiles(CommonColumns):
 
     @classmethod
     @with_default_session
-    def get_total_bytes(
-        cls, session: Session, filter_: Callable[[Query], Query] = lambda q: q
-    ) -> int:
+    def get_total_bytes(cls, session: Session, filter_: Callable[[Query], Query] = lambda q: q) -> int:
         """Get the total number of bytes of data stored across all files."""
         filtered_query = filter_(session.query(func.sum(cls.file_size_bytes)))
         total_bytes = filtered_query.one()[0]
@@ -3238,9 +3091,7 @@ class DownloadableFiles(CommonColumns):
 
     @classmethod
     @with_default_session
-    def get_trial_facets(
-        cls, session: Session, filter_: Callable[[Query], Query] = lambda q: q
-    ):
+    def get_trial_facets(cls, session: Session, filter_: Callable[[Query], Query] = lambda q: q):
         trial_file_counts = cls.count_by(
             cls.trial_id,
             session=session,
@@ -3254,12 +3105,8 @@ class DownloadableFiles(CommonColumns):
     # TODO fix this
     @classmethod
     @with_default_session
-    def get_data_category_facets(
-        cls, session: Session, filter_: Callable[[Query], Query] = lambda q: q
-    ):
-        facet_group_file_counts = cls.count_by(
-            cls.facet_group, session=session, filter_=filter_
-        )
+    def get_data_category_facets(cls, session: Session, filter_: Callable[[Query], Query] = lambda q: q):
+        facet_group_file_counts = cls.count_by(cls.facet_group, session=session, filter_=filter_)
         data_category_facets = build_data_category_facets(facet_group_file_counts)
         return data_category_facets
 
@@ -3275,10 +3122,7 @@ class DownloadableFiles(CommonColumns):
 # Query clause for computing a downloadable file's data category.
 # Used above in the DownloadableFiles.data_category computed property.
 DATA_CATEGORY_CASE_CLAUSE = case(
-    [
-        (DownloadableFiles.facet_group == k, v)
-        for k, v in facet_groups_to_categories.items()
-    ]
+    [(DownloadableFiles.facet_group == k, v) for k, v in facet_groups_to_categories.items()]
 )
 
 # Query clause for computing a downloadable file's file purpose.
@@ -3291,9 +3135,7 @@ FILE_PURPOSE_CASE_CLAUSE = case(
 )
 
 
-def result_proxy_to_models(
-    result_proxy: ResultProxy, model: BaseModel
-) -> List[BaseModel]:
+def result_proxy_to_models(result_proxy: ResultProxy, model: BaseModel) -> List[BaseModel]:
     """Materialize a sqlalchemy `result_proxy` iterable as a list of `model` instances"""
     return [model(**dict(row_proxy)) for row_proxy in result_proxy.all()]