nci-cidc-api-modules 1.0.0rc0__py3-none-any.whl → 1.0.2__py3-none-any.whl

cidc_api/models/models.py

@@ -4,6 +4,8 @@ __all__ = [
     "Column",
     "CommonColumns",
     "DownloadableFiles",
+    "FileGroups",
+    "FilesToFileGroups",
     "EXTRA_DATA_TYPES",
     "IntegrityError",
     "IAMException",
@@ -26,8 +28,6 @@ from collections import defaultdict
 import re
 import hashlib
 import os
-
-os.environ["TZ"] = "UTC"
 from datetime import datetime, timedelta
 from enum import Enum as EnumBaseClass
 from functools import wraps
@@ -42,6 +42,7 @@ from typing import (
     Union,
     Callable,
     Tuple,
+    Iterable,
 )
 
 import pandas as pd
@@ -59,8 +60,9 @@ from sqlalchemy import (
     Index,
     func,
     CheckConstraint,
+    ForeignKey,
     ForeignKeyConstraint,
-    UniqueConstraint,
+    PrimaryKeyConstraint,
     tuple_,
     asc,
     desc,
@@ -71,16 +73,26 @@ from sqlalchemy import (
     not_,
     literal,
     or_,
+    Table,
+    MetaData,
 )
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy.orm import validates
+from sqlalchemy.orm import relationship, validates
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.orm.session import Session
 from sqlalchemy.orm.query import Query
-from sqlalchemy.sql import text
+from sqlalchemy.sql import (
+    # This is unfortunate but other code in this file relies on sqlalchemy.and_, or_, etc
+    # instead of the sqlalchemy.sql versions we are importing here.  The solution is to
+    # break up this giant file.
+    and_ as sql_and,
+    # or_ as sql_or, # NOT USED
+    # select, # ALREADY IMPORTED
+    text,
+)
 from sqlalchemy.sql.functions import coalesce
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.engine import ResultProxy
 
@@ -105,6 +117,7 @@ from ..config.settings import (
 )
 from ..shared import emails
 from ..shared.gcloud_client import (
+    grant_download_access_to_blob_names,
     grant_lister_access,
     grant_download_access,
     publish_artifact_upload,
@@ -116,9 +129,15 @@ from ..shared.gcloud_client import (
 )
 from ..config.logging import get_logger
 
+
+os.environ["TZ"] = "UTC"
 logger = get_logger(__name__)
 
 
+class FileGroups:
+    pass
+
+
 def with_default_session(f):
     """
     For some `f` expecting a database session instance as a keyword argument,
@@ -160,7 +179,7 @@ class CommonColumns(BaseModel):  # type: ignore
             [s for c in cls.__bases__ for s in _all_bases(c)]
         )
 
-        columns_to_check = [c for c in type(self).__table__.columns]
+        columns_to_check = list(type(self).__table__.columns)
         for b in _all_bases(type(self)):
             if hasattr(b, "__table__"):
                 columns_to_check.extend(b.__table__.columns)
@@ -312,7 +331,6 @@ class CommonColumns(BaseModel):  # type: ignore
 
     def validate(self):
         """Run custom validations on attributes set on this instance."""
-        pass
 
     @classmethod
     def get_unique_columns(cls):
@@ -408,7 +426,7 @@ class Users(CommonColumns):
 
         user = Users.find_by_email(email)
         if not user:
-            logger.info(f"Creating new user with email {email}")
+            logger.info("Creating new user with email %s", email)
             user = Users(
                 email=email, contact_email=email, first_n=first_n, last_n=last_n
             )
@@ -431,7 +449,7 @@ class Users(CommonColumns):
             .values(disabled=True)
             .returning(Users.id)
         )
-        disabled_user_ids: List[int] = [uid for uid in session.execute(update_query)]
+        disabled_user_ids: List[int] = list(session.execute(update_query))
         if commit:
             session.commit()
 
@@ -503,7 +521,11 @@ class IAMException(Exception):
     pass
 
 
-EXTRA_DATA_TYPES = ["participants info", "samples info"]
+EXTRA_DATA_TYPES = [
+    "participants info",
+    "samples info",
+    "file_group",
+]
 ALL_UPLOAD_TYPES = set(
     [
         *prism.SUPPORTED_MANIFESTS,
@@ -535,10 +557,14 @@ class Permissions(CommonColumns):
             name="ix_permissions_trial_id",
             ondelete="CASCADE",
         ),
-        UniqueConstraint(
-            "granted_to_user", "trial_id", "upload_type", name="unique_perms"
+        ForeignKeyConstraint(
+            ["file_group_id"],
+            ["file_groups.id"],
+            name="ix_permissions_file_group_id",
+            ondelete="CASCADE",
         ),
         CheckConstraint("trial_id is not null or upload_type is not null"),
+        CheckConstraint("upload_type <> 'file_group' OR file_group_id is NOT NULL"),
     )
     __mapper_args__ = {"confirm_deleted_rows": False}
 
@@ -548,6 +574,26 @@ class Permissions(CommonColumns):
     granted_to_user = Column(Integer, nullable=False, index=True)
     trial_id = Column(String, index=True)
     upload_type = Column(String)
+    file_group_id = Column(Integer, index=True)
+    file_group = relationship("FileGroups", back_populates="permissions")
+
+    __table_args__ = __table_args__ + (
+        Index(
+            "uix_upload_type_perms",
+            "granted_to_user",
+            "trial_id",
+            "upload_type",
+            unique=True,
+            postgresql_where=file_group_id.is_(None),
+        ),
+        Index(
+            "uix_file_group_perms",
+            "granted_to_user",
+            "file_group_id",
+            unique=True,
+            postgresql_where=file_group_id.isnot(None),
+        ),
+    )
 
     # Shorthand to make code related to trial- and upload-type-level permissions
     # easier to interpret.
@@ -577,8 +623,19 @@ class Permissions(CommonColumns):
 
         NOTE: values provided to the `commit` argument will be ignored. This method always commits.
         """
-        if self.upload_type == self.EVERY and self.trial_id == self.EVERY:
-            raise ValueError("A permission must have a trial id or upload type.")
+        if (
+            self.upload_type == self.EVERY
+            and self.trial_id == self.EVERY
+            and not self.file_group_id
+        ):
+            raise ValueError(
+                "A permission must have a trial id, upload type, or file group."
+            )
+
+        if self.file_group_id and self.upload_type != "file_group":
+            raise ValueError(
+                "If a permission has a file group, its upload_type must be set to file_group"
+            )
 
         grantee = Users.find_by_id(self.granted_to_user, session=session)
         if grantee is None:
@@ -595,7 +652,7 @@ class Permissions(CommonColumns):
             raise IntegrityError(
                 params=None,
                 statement=None,
-                orig=f"`granted_by_user` user must be given",
+                orig="`granted_by_user` user must be given",
             )
         if grantor is None:
             raise IntegrityError(
@@ -603,36 +660,52 @@ class Permissions(CommonColumns):
                 statement=None,
                 orig=f"`granted_by_user` user must exist, but no user found with id {self.granted_by_user}",
             )
+        file_group: FileGroups = None
+        if self.upload_type == "file_group":
+            file_group = FileGroups.find_by_id(self.file_group_id, session=session)
+            if not file_group:
+                raise IntegrityError(
+                    params=None,
+                    statement=None,
+                    orig=f"`file_group` must exist, but none found with id {self.file_group_id}",
+                )
 
-        logger.info(
-            f"admin-action: {grantor.email} gave {grantee.email} the permission {self.upload_type or 'all assays'} on {self.trial_id or 'all trials'}"
-        )
+        info_message = f"admin-action: {grantor.email} gave {grantee.email} the permission {self.upload_type or 'all assays'}{'(' + file_group.name + ')' if file_group else ''} on {self.trial_id or 'all trials'}"
+        logger.info(info_message)
 
-        # If this is a permission granting the user access to all trials for
-        # a given upload type or all upload types for a given trial, delete
-        # any related trial-upload type specific permissions to avoid
-        # redundancy in the database and in conditional IAM bindings.
-        perms_to_delete = (
-            session.query(Permissions)
-            .filter(
-                Permissions.granted_to_user == self.granted_to_user,
-                # If inserting a cross-trial perm, then select relevant
-                # trial-specific perms for deletion.
-                Permissions.trial_id != self.EVERY
-                if self.trial_id == self.EVERY
-                else Permissions.trial_id == self.trial_id,
-                # If inserting a cross-upload type perm, then select relevant
-                # upload type-specific perms for deletion. This does NOT
-                # include clinical_data, just manifests/assays/analysis.
-                and_(
-                    Permissions.upload_type != self.EVERY,
-                    Permissions.upload_type != "clinical_data",
+        if self.upload_type == "file_group":
+            # Do not delete past-assigned file_group permissions for this user.
+            perms_to_delete = []
+        else:
+            # If this is a permission granting the user access to all trials for
+            # a given upload type or all upload types for a given trial, delete
+            # any related trial-upload type specific permissions to avoid
+            # redundancy in the database and in conditional IAM bindings.
+            perms_to_delete = (
+                session.query(Permissions)
+                .filter(
+                    Permissions.granted_to_user == self.granted_to_user,
+                    # If inserting a cross-trial perm, then select relevant
+                    # trial-specific perms for deletion.
+                    (
+                        Permissions.trial_id != self.EVERY
+                        if self.trial_id == self.EVERY
+                        else Permissions.trial_id == self.trial_id
+                    ),
+                    # If inserting a cross-upload type perm, then select relevant
+                    # upload type-specific perms for deletion. This does NOT
+                    # include clinical_data, just manifests/assays/analysis.
+                    (
+                        and_(
+                            Permissions.upload_type != self.EVERY,
+                            Permissions.upload_type != "clinical_data",
+                        )
+                        if self.upload_type == self.EVERY
+                        else Permissions.upload_type == self.upload_type
+                    ),
                 )
-                if self.upload_type == self.EVERY
-                else Permissions.upload_type == self.upload_type,
+                .all()
             )
-            .all()
-        )
 
         # Add any related permission deletions to the insertion transaction.
         # If a delete operation fails, all other deletes and the insertion will
@@ -649,16 +722,26 @@ class Permissions(CommonColumns):
             or grantee.disabled
             or grantee.approval_date is None
         ):
+            # TODO: pact users do not have download permissions currently
             return
 
         try:
             # Grant ACL download permissions in GCS
-            # if they have any download permissions, they need the CIDC Lister role
-            grant_lister_access(grantee.email)
-            grant_download_access(grantee.email, self.trial_id, self.upload_type)
-            # Remove permissions staged for deletion, if any
-            for perm in perms_to_delete:
-                revoke_download_access(grantee.email, perm.trial_id, perm.upload_type)
+
+            if self.upload_type == "file_group":
+                Permissions.grant_download_access_to_file_group(
+                    grantee.email, file_group
+                )
+            else:
+
+                # if they have any download permissions, they need the CIDC Lister role
+                grant_lister_access(grantee.email)
+                grant_download_access(grantee.email, self.trial_id, self.upload_type)
+                # Remove permissions staged for deletion, if any
+                for perm in perms_to_delete:
+                    revoke_download_access(
+                        grantee.email, perm.trial_id, perm.upload_type
+                    )
         except Exception as e:
             # Add back deleted permissions, if any
             for perm in perms_to_delete:
@@ -697,7 +780,9 @@ class Permissions(CommonColumns):
                 revoke_download_access(grantee.email, self.trial_id, self.upload_type)
 
                 # If the permission to delete is the last one, also revoke Lister access
-                filter_ = lambda q: q.filter(Permissions.granted_to_user == grantee.id)
+                def filter_(q):
+                    return q.filter(Permissions.granted_to_user == grantee.id)
+
                 if Permissions.count(session=session, filter_=filter_) <= 1:
                     # this one hasn't been deleted yet, so 1 means this is the last one
                     revoke_lister_access(grantee.email)
@@ -707,9 +792,8 @@ class Permissions(CommonColumns):
                     "IAM revoke failed, and permission db record not removed."
                 ) from e
 
-        logger.info(
-            f"admin-action: {deleted_by_user.email} removed from {grantee.email} the permission {self.upload_type or 'all assays'} on {self.trial_id or 'all trials'}"
-        )
+        info_message = f"admin-action: {deleted_by_user.email} removed from {grantee.email} the permission {self.upload_type or 'all assays'} on {self.trial_id or 'all trials'}"
+        logger.info(info_message)
         super().delete(session=session, commit=True)
 
     @staticmethod
@@ -813,15 +897,18 @@ class Permissions(CommonColumns):
 
     @staticmethod
     @with_default_session
-    def find_for_user_trial_type(
-        user_id: int, trial_id: str, upload_type: str, session: Session
+    def find_for_user_trial_file(
+        user_id: int,
+        trial_id: str,
+        upload_type: str,
+        file_groups: Iterable,
+        session: Session,
     ) -> Optional["Permissions"]:
         """
-        Check if a Permissions record exists for the given user, trial, and type.
-        The result may be a trial- or assay-level permission that encompasses the
-        given trial id or upload type.
+        Check if a Permissions record exists for the given user, trial, and file.
+        The result may be a trial-, assay-, or file-group-level permission.
         """
-        return (
+        results = (
             session.query(Permissions)
             .filter(
                 Permissions.granted_to_user == user_id,
@@ -843,6 +930,27 @@ class Permissions(CommonColumns):
             .first()
         )
 
+        # If nothing found so far, check file-group-level permissions.
+        if not results and file_groups:
+            results = (
+                session.query(Permissions)
+                .join(FileGroups, Permissions.file_group_id == FileGroups.id)
+                .filter(Permissions.granted_to_user == user_id)
+                .filter(Permissions.trial_id == trial_id)
+                .filter(Permissions.file_group_id == FileGroups.id)
+                .all()
+            )
+            if results:
+                file_group_ids = {file_group.id for file_group in file_groups}
+                results = [
+                    result
+                    for result in results
+                    if result.file_group_id in file_group_ids
+                ]
+
+        results = results and results or None
+        return results
+
     @staticmethod
     @with_default_session
     def grant_user_permissions(user: Users, session: Session) -> None:
@@ -855,13 +963,23 @@ class Permissions(CommonColumns):
 
         perms = Permissions.find_for_user(user.id, session=session)
         # if they have any download permissions, they need the CIDC Lister role
-        if len(perms):
+        # If a Permission's FileGroup is None, that implies the Permission is a
+        # trial/assay type and thus lister access is required.
+        if len(perms) and any(perm.file_group_id is None for perm in perms):
             grant_lister_access(user.email)
 
         # separate permissions by trial, as they are strictly non-overlapping
         perms_by_trial: Dict[str, List[Permissions]] = defaultdict(list)
+        # Or separate permissions by filegroup
+        file_group_perms = []
+        file_group_perms_ids = []
         for perm in perms:
-            perms_by_trial[perm.trial_id].append(perm)
+            if perm.upload_type == "file_group":
+                if perm.file_group_id not in file_group_perms_ids:
+                    file_group_perms.append(perm)
+                    file_group_perms_ids.append(perm.file_group_id)
+            else:
+                perms_by_trial[perm.trial_id].append(perm)
         perms_by_trial = dict(perms_by_trial)
 
         for trial_id, trial_perms in perms_by_trial.items():
@@ -871,6 +989,11 @@ class Permissions(CommonColumns):
                 trial_id=trial_id,
                 upload_type=[p.upload_type for p in trial_perms],
             )
+        for perm in file_group_perms:
+            file_group: FileGroups = FileGroups.find_by_id(
+                perm.file_group_id, session=session
+            )
+            Permissions.grant_download_access_to_file_group(user.email, file_group)
 
         # Regrant all of the user's intake bucket upload permissions, if they have any
         refresh_intake_access(user.email)
@@ -945,9 +1068,9 @@ class Permissions(CommonColumns):
                 or user.email in user_email_list
             ):
                 continue
-            else:
-                user_email_list.append(user.email)
-                grant_lister_access(user.email)
+
+            user_email_list.append(user.email)
+            grant_lister_access(user.email)
 
         if upload.upload_type in prism.SUPPORTED_SHIPPING_MANIFESTS:
             # Passed with empty user email list because they will be queried for in CFn
@@ -957,125 +1080,20 @@ class Permissions(CommonColumns):
             grant_download_access(user_email_list, upload.trial_id, upload.upload_type)
 
     @staticmethod
-    @with_default_session
-    def grant_download_permissions(
-        trial_id: str, upload_type: str, session: Session
-    ) -> None:
-        Permissions._change_download_permissions(
-            trial_id=trial_id, upload_type=upload_type, grant=True, session=session
-        )
-
-    @staticmethod
-    @with_default_session
-    def revoke_download_permissions(
-        trial_id: str, upload_type: str, session: Session
-    ) -> None:
-        Permissions._change_download_permissions(
-            trial_id=trial_id, upload_type=upload_type, grant=False, session=session
+    def grant_download_access_to_file_group(user_email: str, file_group: FileGroups):
+        # grant to individual blobs rather than calling grant_download_access(grantee, trial, upload_type)
+        blob_names: list[str] = [df.object_url for df in file_group.downloadable_files]
+        logger.info(
+            "Granting access to %s for the following downloadables: %s",
+            user_email,
+            blob_names,
         )
-
-    @staticmethod
-    @with_default_session
-    def _change_download_permissions(
-        trial_id: str, upload_type: str, grant: bool, session: Session
-    ) -> None:
-        """
-        Allows for widespread granting/revoking of existing download permissions in GCS ACL
-        Optionally filtered for specific trials and upload types
-        If granting, also adds lister IAM permission for each user
-        If revoking, DOES NOT remove lister IAM permission from any user
-
-        Parameters
-        ----------
-        trial_id: str
-            only affect permissions for this trial
-            None for all trials
-        upload_type: str
-            only affect permissions for this upload type
-            None for all upload types except clinical_data
-        grant: bool
-            whether to grant or remove the (filtered) permissions
-            if True, adds lister IAM permission
-        session: Session
-            filled by @with_default_session if not provided
-        """
-        filters = [
-            # set the condition for the join
-            Permissions.granted_to_user == Users.id,
-            # admins have blanket access via IAM
-            Users.role != CIDCRole.ADMIN.value,
-        ]
-        if grant:
-            # NCI users and disable aren't granted download permissions
-            # but we should be able to un-grant ie revoke them
-            filters.extend(
-                [
-                    Users.role != CIDCRole.NCI_BIOBANK_USER.value,
-                    Users.disabled == False,
-                ]
-            )
-        if trial_id:
-            filters.append(
-                or_(
-                    Permissions.trial_id == trial_id, Permissions.trial_id == None
-                ),  # null for cross-trial
-            )
-        if upload_type == "clinical_data":
-            # don't get null ie cross-assay
-            filters.append(
-                Permissions.upload_type == upload_type,
-            )
-        elif upload_type:
-            filters.append(
-                or_(
-                    Permissions.upload_type == upload_type,
-                    Permissions.upload_type == None,
-                ),  # null for cross-assay
-            )
-        else:  # null for cross-assay
-            filters.append(
-                # don't affect clinical_data
-                Permissions.upload_type
-                != "clinical_data",
-            )
-
-        # List[Tuple[Permissions, Users]]
-        perms_and_users = session.query(Permissions, Users).filter(*filters).all()
-
-        # group by trial and upload type
-        # Dict[str, Dict[str, List[str]]] = {trial_id: {upload_type: [user_email, ...], ...}, ...}
-        sorted_permissions = defaultdict(lambda: defaultdict(list))
-        # also handle user lister IAM permission if granting
-        already_listed: List[str] = []
-        for perm, user in perms_and_users:
-            # make sure we put it only for the desired scope
-            sorted_permissions[trial_id if trial_id else perm.trial_id][
-                upload_type if upload_type else perm.upload_type
-            ].append(user.email)
-
-            # if granting things, grant_lister_access on every user
-            # idempotent, amounting to "add or refresh"
-            if grant and user.email not in already_listed:
-                grant_lister_access(user.email)
-                already_listed.append(user.email)
-            # if un-granting ie revoking things, don't call revoke_lister_access
-            # with the filtering, we don't know if the users have any other
-            # ACL permissions remaining that weren't affected here
-
-        # now that we've filtered and separated, just do them all
-        # new values will override passed args
-        for trial_id, trial_perms in sorted_permissions.items():
-            for upload_type, users in trial_perms.items():
-                (grant_download_access if grant else revoke_download_access)(
-                    users, trial_id, upload_type
-                )
+        grant_download_access_to_blob_names([user_email], blob_names)
 
 
 class ValidationMultiError(Exception):
     """Holds multiple jsonschema.ValidationErrors"""
 
-    pass
-
 
 trial_metadata_validator: json_validation._Validator = (
     json_validation.load_and_validate_schema(
@@ -1209,7 +1227,7 @@ class TrialMetadata(CommonColumns):
         Create a new clinical trial metadata record.
         """
 
-        logger.info(f"Creating new trial metadata with id {trial_id}")
+        logger.info("Creating new trial metadata with id %s", trial_id)
         trial = TrialMetadata(trial_id=trial_id, metadata_json=metadata_json)
         trial.insert(session=session, commit=commit)
 
@@ -1401,7 +1419,10 @@ class TrialMetadata(CommonColumns):
             session.commit()
 
     @classmethod
-    def build_trial_filter(cls, user: Users, trial_ids: List[str] = []):
+    def build_trial_filter(cls, user: Users, trial_ids: List[str] = None):
+        if trial_ids is None:
+            trial_ids = []
+
         filters = []
         if trial_ids:
             filters.append(cls.trial_id.in_(trial_ids))
@@ -1662,7 +1683,7 @@ class TrialMetadata(CommonColumns):
                     trial_metadata,
                     jsonb_array_elements(metadata_json->'participants') participant,
                     jsonb_array_elements(participant->'samples') sample
-                
+
                 where
                         sample->>'processed_sample_derivative' = 'Tumor DNA'
                     or
@@ -1778,7 +1799,7 @@ class TrialMetadata(CommonColumns):
                 trial_id,
                 jsonb_object_agg(key, value) as value
             from (
-                select 
+                select
                     trial_id,
                     key,
                     jsonb_agg(sample) as value
@@ -1896,7 +1917,7 @@ class TrialMetadata(CommonColumns):
                     select
                         trial_id,
                         key,
-                        count(cimac_id) as num_sample
+                        count(distinct cimac_id) as num_sample
                     from (
                         {generic_assay_subquery}
                         union all
@@ -1996,7 +2017,7 @@ class UploadJobs(CommonColumns):
     __tablename__ = "upload_jobs"
     # An upload job must contain a gcs_file_map is it isn't a manifest upload
     __table_args__ = (
-        CheckConstraint(f"multifile = true OR gcs_file_map != null"),
+        CheckConstraint("multifile = true OR gcs_file_map != null"),
         ForeignKeyConstraint(
             ["uploader_email"],
             ["users.email"],
@@ -2138,10 +2159,10 @@ class UploadJobs(CommonColumns):
         if job is None or job.status == UploadJobStatus.MERGE_COMPLETED.value:
             raise ValueError(f"Upload job {job_id} doesn't exist or is already merged")
 
-        logger.info(f"About to merge extra md to {job.id}/{job.status}")
+        logger.info("About to merge extra md to %s/%s", job.id, job.status)
 
         for uuid, file in files.items():
-            logger.info(f"About to parse/merge extra md on {uuid}")
+            logger.info("About to parse/merge extra md on %s", uuid)
             (
                 job.metadata_patch,
                 updated_artifact,
@@ -2149,14 +2170,14 @@ class UploadJobs(CommonColumns):
             ) = prism.merge_artifact_extra_metadata(
                 job.metadata_patch, uuid, job.upload_type, file
             )
-            logger.info(f"Updated md for {uuid}: {updated_artifact.keys()}")
+            logger.info("Updated md for %s: %s", uuid, updated_artifact.keys())
 
         # A workaround fix for JSON field modifications not being tracked
         # by SQLalchemy for some reason. Using MutableDict.as_mutable(JSON)
         # in the model doesn't seem to help.
         flag_modified(job, "metadata_patch")
 
-        logger.info(f"Updated {job.id}/{job.status} patch: {job.metadata_patch}")
+        logger.info("Updated %s/%s patch: %s", job.id, job.status, job.metadata_patch)
         session.commit()
 
     @classmethod
@@ -2188,6 +2209,62 @@ class UploadJobs(CommonColumns):
             self.alert_upload_success(trial)
 
 
+class FilesToFileGroups(BaseModel):
+    """
+    Mapping table between files (DownloadableFiles) and FileGroups.
+    """
+
+    __tablename__ = "files_to_file_groups"
+    __table_args__ = (
+        PrimaryKeyConstraint(
+            "file_group_id", "file_id", name="pk_files_to_file_groups"
+        ),
+    )
+    file_group_id = Column(ForeignKey("file_groups.id"), primary_key=True)
+    file_id = Column(ForeignKey("downloadable_files.id"), primary_key=True)
+    _created = Column(DateTime, default=func.now(), nullable=False)
+
+    # Not currently used, but may be useful in the future.
+    sort_order: int = Column(Integer)
+
+    @with_default_session
+    def insert(self, session: Session, commit: bool = True):
+        """For compatibility with CommonColumns"""
+        session.add(self)
+        if commit:
+            session.commit()
+
+
+class FileGroups(CommonColumns):
+    """
+    Entity representing an arbitrary grouping of files.  With the current permissions
+    scheme, file groups always represent groups of files within a single trial, although
+    this may not be true in the future.
+
+    File groups are currently used to restrict PACT users' access to subsets of files
+    within a trial that do not correspond to assay types or facets.  In the future, we
+    may use them to represent collections of data published in a given study.
+    """
+
+    __tablename__ = "file_groups"
+
+    name: Column = Column(String, nullable=False, unique=True)
+    display_name: Column = Column(String, nullable=True)
+    downloadable_files = relationship(
+        "DownloadableFiles",
+        secondary="files_to_file_groups",
+        back_populates="file_groups",
+    )
+
+    permissions = relationship("Permissions", back_populates="file_group")
+
+    @classmethod
+    @with_default_session
+    def find_by_name(cls, name: str, session: Session):
+        """Find the record with this name"""
+        return session.query(cls).filter_by(name=name).first()
+
+
 class DownloadableFiles(CommonColumns):
     """
     Store required fields from:
@@ -2232,6 +2309,12 @@ class DownloadableFiles(CommonColumns):
     file_name = Column(String, nullable=True)
     data_format = Column(String, nullable=True)
 
+    file_groups = relationship(
+        "FileGroups",
+        secondary="files_to_file_groups",
+        back_populates="downloadable_files",
+    )
+
     FILE_EXT_REGEX = r"\.([^./]*(\.gz)?)$"
 
     @hybrid_property
@@ -2240,15 +2323,15 @@ class DownloadableFiles(CommonColumns):
         return match.group(1) if match else None
 
     @file_ext.expression
-    def file_ext(cls):
-        return func.substring(cls.object_url, cls.FILE_EXT_REGEX)
+    def file_ext(self):
+        return func.substring(self.object_url, self.FILE_EXT_REGEX)
 
     @hybrid_property
     def data_category(self):
         return facet_groups_to_categories.get(self.facet_group)
 
     @data_category.expression
-    def data_category(cls):
+    def data_category(self):
         return DATA_CATEGORY_CASE_CLAUSE
 
     @hybrid_property
@@ -2262,7 +2345,7 @@ class DownloadableFiles(CommonColumns):
         return self.data_category.split(FACET_NAME_DELIM, 1)[0]
 
     @data_category_prefix.expression
-    def data_category_prefix(cls):
+    def data_category_prefix(self):
         return func.split_part(DATA_CATEGORY_CASE_CLAUSE, FACET_NAME_DELIM, 1)
 
     @hybrid_property
@@ -2270,7 +2353,7 @@ class DownloadableFiles(CommonColumns):
         return details_dict.get(self.facet_group).file_purpose
 
     @file_purpose.expression
-    def file_purpose(cls):
+    def file_purpose(self):
         return FILE_PURPOSE_CASE_CLAUSE
 
     @property
@@ -2298,6 +2381,473 @@ class DownloadableFiles(CommonColumns):
     def check_additional_metadata_default(self, key, value):
         return {} if value in ["null", None, {}] else value
 
+    @classmethod
+    def _generate_query_objects(cls):
+
+        # This is really unfortunate and a huge duplication of code from the column
+        # definitions in this class, but it seemed to be the only way to get the
+        # SQLAlchemy Expression Language API to work.
+        metadata = MetaData()
+        downloadable_files_for_query = Table(
+            "downloadable_files",
+            metadata,
+            Column("_created", BigInteger, nullable=False),
+            Column("_updated", DateTime, default=func.now(), nullable=False),
+            Column("_etag", String(40), nullable=False),
+            Column("id", Integer, primary_key=True, autoincrement=True, nullable=False),
+            Column("file_size_bytes", BigInteger, nullable=False),
+            Column("uploaded_timestamp", DateTime, nullable=False),
+            Column("facet_group", String, nullable=False),
+            Column("additional_metadata", JSONB, nullable=False),
+            Column("upload_type", String, nullable=False),
+            Column("md5_hash", String, nullable=True),
+            Column("crc32c_hash", String, nullable=True),
+            Column("trial_id", String, nullable=False),
+            Column("object_url", String, nullable=False, index=True, unique=True),
+            Column("visible", Boolean, default=True),
+            Column("analysis_friendly", Boolean, default=False),
+            Column("clustergrammer", JSONB, nullable=True),
+            Column("ihc_combined_plot", JSONB, nullable=True),
+            Column("file_name", String, nullable=True),
+            Column("data_format", String, nullable=True),
+        )
+
+        # TODO(jcallaway): consider this reflection-based approach instead.  It doesn't
+        # currently work because of the relationship()s and foreign keys, among other
+        # problems.
+        # columns = []
+        # attributes = inspect.getmembers(CommonColumns, lambda x: not (inspect.isroutine(x)))
+        # for k, v in attributes:
+        #     if not(k.startswith('__') and k.endswith('__')) and isinstance(v, Column):
+        #         v.name = k
+        #         columns.append(v)
+        # attributes = inspect.getmembers(DownloadableFiles, lambda x: not (inspect.isroutine(x)))
+        # for k, v in attributes:
+        #     if not (k.startswith('__') and k.endswith('__')) and isinstance(v, Column):
+        #         v.name = k
+        #         columns.append(v)
+        # downloadable_files_for_query = Table("downloadable_files", metadata, *columns)
+
+        files_to_file_groups_for_query = Table(
+            "files_to_file_groups",
+            metadata,
+            Column("_created", BigInteger, nullable=False),
+            Column("_updated", DateTime, default=func.now(), nullable=False),
+            Column("_etag", String(40), nullable=False),
+            Column("id", Integer, primary_key=True, autoincrement=True, nullable=False),
+            Column("file_group_id", Integer, nullable=False),
+            Column("file_id", ForeignKey("downloadable_files.id")),
+        )
+
+        return downloadable_files_for_query, files_to_file_groups_for_query
+
+    @classmethod
+    def _convert_list_results(
+        cls, downloadable_files_for_query: Table, query_files: List
+    ):
+        """Converts the results of a SQLalchemy expression language query into actual DownloadableFiles
+        objects.  This is necessary since the UI depends on some of the derived properties in
+        DownloadableFiles.
+        """
+        results = []
+        for query_file in query_files:
+            args = {}
+            for column in downloadable_files_for_query.c:
+                args[column.name] = getattr(query_file, column.name)
+            results.append(DownloadableFiles(**args))
+
+        return results
+
+    @classmethod
+    def _generate_where_clauses(
+        cls,
+        downloadable_files_for_query: Table,
+        files_to_file_groups_for_query: Table,
+        trial_ids: List[str],
+        facets: List[List[str]],
+        user: Users,
+    ):
+        """
+        Returns a list of where clauses for use by list_with_permissions() and count_with_permissions().
+        If the list is empty, this means to execute the query without a where clause (likely because
+        the user is an admin).  If None is returned instead of a list, this means that the user has no
+        permissions to view any files specified.
+        """
+
+        # From the perspective of viewing files, NCI Biobank users are admins.
+        is_admin = user and (user.is_admin() or user.is_nci_user())
+
+        where_clauses = []
+        if trial_ids:
+            where_clauses.append(downloadable_files_for_query.c.trial_id.in_(trial_ids))
+        if facets:
+            facet_groups = get_facet_groups_for_paths(facets)
+            where_clauses.append(
+                downloadable_files_for_query.c.facet_group.in_(facet_groups)
+            )
+
+        if user and not is_admin:
+            permissions = Permissions.find_for_user(user.id)
+            full_trial_perms, full_type_perms = [], []
+            for permission in permissions:
+
+                # If upload_type or trial_id is null, that means to grant permissions to
+                # *all* files for that upload type or trial.
+                if permission.upload_type is None:
+                    full_trial_perms.append(permission.trial_id)
+                elif permission.trial_id is None:
+                    full_type_perms.append(permission.upload_type)
+                elif permission.file_group_id is None:
+                    where_clauses.append(
+                        sql_and(
+                            (
+                                downloadable_files_for_query.c.trial_id
+                                == permission.trial_id
+                            ),
+                            (
+                                downloadable_files_for_query.c.upload_type
+                                == permission.upload_type
+                            ),
+                        )
+                    )
+                else:
+                    where_clauses.append(
+                        sql_and(
+                            (
+                                downloadable_files_for_query.c.trial_id
+                                == permission.trial_id
+                            ),
+                            (
+                                files_to_file_groups_for_query.c.file_group_id
+                                == permission.file_group_id
+                            ),
+                        )
+                    )
+
+            if full_trial_perms:
+
+                # don't include clinical_data in cross-trial permission
+                where_clauses.append(
+                    sql_and(
+                        downloadable_files_for_query.c.trial_id.in_(full_trial_perms),
+                        (downloadable_files_for_query.c.upload_type != "clinical_data"),
+                    )
+                )
+            if full_type_perms:
+                where_clauses.append(
+                    downloadable_files_for_query.c.upload_type.in_(full_type_perms)
+                )
+
+        # Need to be careful about return logic.  Empty results could be because the user
+        # is an admin, whereas None means the user has no permissions to view any files.
+        if is_admin or where_clauses:
+            return where_clauses
+
+        return None
+
+    @classmethod
+    @with_default_session
+    def list_with_permissions(
+        cls,
+        session: Session,
+        trial_ids: List[str] = None,
+        facets: List[List[str]] = None,
+        page_num: int = 0,
+        page_size: int = PAGINATION_PAGE_SIZE,
+        sort_field: Optional[str] = None,
+        sort_direction: Optional[str] = None,
+        user: Users = None,
+    ):
+        """List records in this table, with pagination support."""
+        if trial_ids is None:
+            trial_ids = []
+
+        if facets is None:
+            facets = []
+
+        (
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+        ) = DownloadableFiles._generate_query_objects()
+        where_clauses = DownloadableFiles._generate_where_clauses(
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+            trial_ids,
+            facets,
+            user,
+        )
+        if where_clauses is None:
+
+            # User doesn't have permissions to view any files; no need to issue a query.
+            return {}
+
+        if where_clauses:
+
+            # No where clause (the user is likely an admin).
+            statement = select([downloadable_files_for_query]).select_from(
+                downloadable_files_for_query
+            )
+
+        else:
+            statement = (
+                select([downloadable_files_for_query])
+                .where(sql_and(*where_clauses))
+                .select_from(
+                    downloadable_files_for_query.outerjoin(
+                        files_to_file_groups_for_query
+                    )
+                )
+            )
+
+        if sort_field:
+            sort_attribute = getattr(cls, sort_field)
+            field_with_dir = (
+                asc(sort_attribute) if sort_direction == "asc" else desc(sort_attribute)
+            )
+            statement = statement.order_by(field_with_dir)
+
+        # Enforce positive page numbers
+        page_num = 0 if page_num < 0 else page_num
+
+        # Enforce maximum page size
+        page_size = min(page_size, MAX_PAGINATION_PAGE_SIZE)
+        statement = statement.limit(page_size).offset(page_num * page_size)
+
+        return DownloadableFiles._convert_list_results(
+            downloadable_files_for_query, session.execute(statement).fetchall()
+        )
+
+    @classmethod
+    @with_default_session
+    def count_with_permissions(
+        cls,
+        session: Session,
+        trial_ids: List[str] = None,
+        facets: List[List[str]] = None,
+        # page_num: int = 0,
+        # page_size: int = PAGINATION_PAGE_SIZE,
+        # sort_field: Optional[str] = None,
+        # sort_direction: Optional[str] = None,
+        user: Users = None,
+    ):
+        """
+        Return the total number of records that would be returned by equivalent calls to
+        list_with_permissions() (disregarding results paging).
+        """
+
+        if trial_ids is None:
+            trial_ids = []
+
+        if facets is None:
+            facets = []
+
+        (
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+        ) = DownloadableFiles._generate_query_objects()
+        where_clauses = DownloadableFiles._generate_where_clauses(
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+            trial_ids,
+            facets,
+            user,
+        )
+        if where_clauses is None:
+
+            # User doesn't have permissions to view any files; no need to issue a query.
+            return 0
+
+        if where_clauses:
+
+            # No where clause (the user is likely an admin).
+            statement = select(
+                [func.count(downloadable_files_for_query.c.id)]
+            ).select_from(downloadable_files_for_query)
+
+        else:
+            statement = (
+                select([func.count(downloadable_files_for_query.c.id)])
+                .where(sql_and(*where_clauses))
+                .select_from(
+                    downloadable_files_for_query.outerjoin(
+                        files_to_file_groups_for_query
+                    )
+                )
+            )
+
+        return session.execute(statement).fetchone()[0]
+
+    @classmethod
+    @with_default_session
+    def count_by_facet_with_permissions(
+        cls, session: Session, trial_ids: List[str] = None, user: Users = None
+    ):
+        """
+        Returns a map of facet_group to a count of the number of files that the given user
+        has permissions to view.
+        """
+
+        if trial_ids is None:
+            trial_ids = []
+
+        (
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+        ) = DownloadableFiles._generate_query_objects()
+        where_clauses = DownloadableFiles._generate_where_clauses(
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+            trial_ids,
+            None,
+            user,
+        )
+        if where_clauses is None:
+
+            # User doesn't have permissions to view any files; no need to issue a query.
+            return {}
+
+        if not where_clauses:
+
+            # No where clause (the user is likely an admin).
+            statement = select(
+                [
+                    downloadable_files_for_query.c.facet_group,
+                    func.count(downloadable_files_for_query.c.id),
+                ]
+            ).select_from(downloadable_files_for_query)
+
+        else:
+            statement = (
+                select(
+                    [
+                        downloadable_files_for_query.c.facet_group,
+                        func.count(downloadable_files_for_query.c.id),
+                    ]
+                )
+                .where(sql_and(*where_clauses))
+                .select_from(
+                    downloadable_files_for_query.outerjoin(
+                        files_to_file_groups_for_query
+                    )
+                )
+            )
+
+        statement = statement.group_by(downloadable_files_for_query.c.facet_group)
+        results = session.execute(statement).fetchall()
+        return dict(results)
+
+    @classmethod
+    @with_default_session
+    def list_object_urls_with_permissions(
+        cls, session: Session, user: Users = None, ids: Iterable[int] = None
+    ) -> Iterable[str]:
+        """
+        Checks that the given user has access to all the files identified by ids, and returns
+        the object_urls for those the user has permission for.
+        """
+        (
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+        ) = DownloadableFiles._generate_query_objects()
+        where_clauses = DownloadableFiles._generate_where_clauses(
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+            None,
+            None,
+            user,
+        )
+        if where_clauses is None:
+
+            # User doesn't have permissions to view any files; no need to issue a query.
+            return {}
+
+        if not where_clauses:
+
+            # No where clause (the user is likely an admin).
+            statement = (
+                select([downloadable_files_for_query.c.object_url])
+                .where(downloadable_files_for_query.c.id.in_(ids))
+                .select_from(downloadable_files_for_query)
+            )
+
+        else:
+            statement = (
+                select([downloadable_files_for_query.c.object_url])
+                .where(
+                    sql_and(*where_clauses, downloadable_files_for_query.c.id.in_(ids))
+                )
+                .select_from(
+                    downloadable_files_for_query.outerjoin(
+                        files_to_file_groups_for_query
+                    )
+                )
+            )
+
+        return [row[0] for row in session.execute(statement).fetchall()]
+
+    @classmethod
+    def _generate_trial_file_counts(
+        cls, downloadable_files: Iterable
+    ) -> Dict[str, int]:
+        results = defaultdict(lambda: 0)
+        for downloadable_file in downloadable_files:
+            if downloadable_file.data_category:
+                results[downloadable_file.trial_id] = (
+                    results[downloadable_file.trial_id] + 1
+                )
+        return results
+
+    @classmethod
+    @with_default_session
+    def get_trial_facets_with_permissions(
+        cls, session: Session, facets: Iterable[Iterable], user: Users = None
+    ) -> Dict[str, int]:
+        """
+        Returns a map of trial ID to count of files that the user has permissions to view.
+        Files with a non-true data_category property are not included in these counts.
+        """
+        (
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+        ) = DownloadableFiles._generate_query_objects()
+        where_clauses = DownloadableFiles._generate_where_clauses(
+            downloadable_files_for_query,
+            files_to_file_groups_for_query,
+            None,
+            facets,
+            user,
+        )
+        if where_clauses is None:
+
+            # User doesn't have permissions to view any files; no need to issue a query.
+            return {}
+
+        if not where_clauses:
+
+            # No where clause (the user is likely an admin).
+            statement = select([downloadable_files_for_query]).select_from(
+                downloadable_files_for_query
+            )
+
+        else:
+            statement = (
+                select([downloadable_files_for_query])
+                .where(sql_and(*where_clauses))
+                .select_from(
+                    downloadable_files_for_query.outerjoin(
+                        files_to_file_groups_for_query
+                    )
+                )
+            )
+
+        downloadable_files = DownloadableFiles._convert_list_results(
+            downloadable_files_for_query, session.execute(statement).fetchall()
+        )
+        trial_file_counts = DownloadableFiles._generate_trial_file_counts(
+            downloadable_files
+        )
+        return build_trial_facets(trial_file_counts)
+
     @with_default_session
     def get_related_files(self, session: Session) -> list:
         """
@@ -2340,15 +2890,16 @@ class DownloadableFiles(CommonColumns):
 
         return related_files
 
+    # this is the old file filter code which we are temporarily putting back in to make facet search work again
+    # TODO use old serach code again
     @staticmethod
     def build_file_filter(
-        trial_ids: List[str] = [], facets: List[List[str]] = [], user: Users = None
+        trial_ids: List[str] = None, facets: List[List[str]] = None, user: Users = None
     ) -> Callable[[Query], Query]:
         """
         Build a file filter function based on the provided parameters. The resultant
         filter can then be passed as the `filter_` argument of `DownloadableFiles.list`
         or `DownloadableFiles.count`.
-
         Args:
             trial_ids: if provided, the filter will include only files with these trial IDs.
             upload_types: if provided, the filter will include only files with these upload types.
@@ -2358,6 +2909,12 @@ class DownloadableFiles(CommonColumns):
         Returns:
             A function that adds filters to a query against the DownloadableFiles table.
         """
+        if trial_ids is None:
+            trial_ids = []
+
+        if facets is None:
+            facets = []
+
         file_filters = []
         if trial_ids:
             file_filters.append(DownloadableFiles.trial_id.in_(trial_ids))
@@ -2418,10 +2975,10 @@ class DownloadableFiles(CommonColumns):
             "additional_metadata": additional_metadata,
         }
 
+        # TODO maybe put non supported stuff from file_metadata to some misc jsonb column?
         for key, value in file_metadata.items():
             if key in supported_columns:
                 filtered_metadata[key] = value
-        # TODO maybe put non supported stuff from file_metadata to some misc jsonb column?
 
         etag = make_etag(filtered_metadata.values())
 
@@ -2584,6 +3141,7 @@ class DownloadableFiles(CommonColumns):
         """Get the total number of bytes of data stored across all files."""
         filtered_query = filter_(session.query(func.sum(cls.file_size_bytes)))
         total_bytes = filtered_query.one()[0]
+        # return int(total_bytes)
         return int(total_bytes or 0)
 
     @classmethod
@@ -2600,6 +3158,8 @@ class DownloadableFiles(CommonColumns):
         trial_facets = build_trial_facets(trial_file_counts)
         return trial_facets
 
+    # old code to make filter search work
+    # TODO fix this
     @classmethod
     @with_default_session
     def get_data_category_facets(
@@ -2611,6 +3171,14 @@ class DownloadableFiles(CommonColumns):
         data_category_facets = build_data_category_facets(facet_group_file_counts)
         return data_category_facets
 
+    # new code that has been commented out to make filter search work
+    # def get_data_category_facets(cls, trial_ids, user):
+    #    facet_group_file_counts = DownloadableFiles.count_by_facet_with_permissions(
+    #        trial_ids=trial_ids, user=user
+    #    )
+    #    data_category_facets = build_data_category_facets(facet_group_file_counts)
+    #    return data_category_facets
+
 
 # Query clause for computing a downloadable file's data category.
 # Used above in the DownloadableFiles.data_category computed property.