nci-cidc-api-modules 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cidc_api/config/db.py +1 -1
- cidc_api/config/secrets.py +2 -2
- cidc_api/config/settings.py +1 -2
- cidc_api/csms/auth.py +14 -7
- cidc_api/models/csms_api.py +101 -83
- cidc_api/models/files/details.py +18 -38
- cidc_api/models/files/facets.py +6 -6
- cidc_api/models/migrations.py +16 -9
- cidc_api/models/models.py +186 -158
- cidc_api/shared/auth.py +18 -13
- cidc_api/shared/gcloud_client.py +75 -73
- cidc_api/shared/rest_utils.py +6 -5
- {nci_cidc_api_modules-1.0.0.dist-info → nci_cidc_api_modules-1.0.1.dist-info}/METADATA +1 -1
- nci_cidc_api_modules-1.0.1.dist-info/RECORD +25 -0
- {nci_cidc_api_modules-1.0.0.dist-info → nci_cidc_api_modules-1.0.1.dist-info}/WHEEL +1 -1
- nci_cidc_api_modules-1.0.0.dist-info/RECORD +0 -25
- {nci_cidc_api_modules-1.0.0.dist-info → nci_cidc_api_modules-1.0.1.dist-info}/LICENSE +0 -0
- {nci_cidc_api_modules-1.0.0.dist-info → nci_cidc_api_modules-1.0.1.dist-info}/top_level.txt +0 -0
cidc_api/models/models.py
CHANGED
@@ -28,9 +28,6 @@ from collections import defaultdict
|
|
28
28
|
import re
|
29
29
|
import hashlib
|
30
30
|
import os
|
31
|
-
import inspect
|
32
|
-
|
33
|
-
os.environ["TZ"] = "UTC"
|
34
31
|
from datetime import datetime, timedelta
|
35
32
|
from enum import Enum as EnumBaseClass
|
36
33
|
from functools import wraps
|
@@ -65,7 +62,6 @@ from sqlalchemy import (
|
|
65
62
|
CheckConstraint,
|
66
63
|
ForeignKey,
|
67
64
|
ForeignKeyConstraint,
|
68
|
-
UniqueConstraint,
|
69
65
|
PrimaryKeyConstraint,
|
70
66
|
tuple_,
|
71
67
|
asc,
|
@@ -80,27 +76,23 @@ from sqlalchemy import (
|
|
80
76
|
Table,
|
81
77
|
MetaData,
|
82
78
|
)
|
83
|
-
from sqlalchemy.orm import
|
84
|
-
|
85
|
-
|
86
|
-
|
79
|
+
from sqlalchemy.orm import relationship, validates
|
80
|
+
from sqlalchemy.orm.attributes import flag_modified
|
81
|
+
from sqlalchemy.orm.exc import NoResultFound
|
82
|
+
from sqlalchemy.orm.session import Session
|
83
|
+
from sqlalchemy.orm.query import Query
|
87
84
|
from sqlalchemy.sql import (
|
88
85
|
# This is unfortunate but other code in this file relies on sqlalchemy.and_, or_, etc
|
89
86
|
# instead of the sqlalchemy.sql versions we are importing here. The solution is to
|
90
87
|
# break up this giant file.
|
91
88
|
and_ as sql_and,
|
92
|
-
or_ as sql_or,
|
93
|
-
select,
|
89
|
+
# or_ as sql_or, # NOT USED
|
90
|
+
# select, # ALREADY IMPORTED
|
94
91
|
text,
|
95
92
|
)
|
93
|
+
from sqlalchemy.sql.functions import coalesce
|
96
94
|
from sqlalchemy.exc import IntegrityError
|
97
95
|
from sqlalchemy.ext.hybrid import hybrid_property
|
98
|
-
from sqlalchemy.orm import validates
|
99
|
-
from sqlalchemy.orm.attributes import flag_modified
|
100
|
-
from sqlalchemy.orm.exc import NoResultFound
|
101
|
-
from sqlalchemy.orm.session import Session
|
102
|
-
from sqlalchemy.orm.query import Query
|
103
|
-
from sqlalchemy.sql.functions import coalesce
|
104
96
|
from sqlalchemy.dialects.postgresql import JSONB, UUID
|
105
97
|
from sqlalchemy.engine import ResultProxy
|
106
98
|
|
@@ -137,6 +129,8 @@ from ..shared.gcloud_client import (
|
|
137
129
|
)
|
138
130
|
from ..config.logging import get_logger
|
139
131
|
|
132
|
+
|
133
|
+
os.environ["TZ"] = "UTC"
|
140
134
|
logger = get_logger(__name__)
|
141
135
|
|
142
136
|
|
@@ -185,7 +179,7 @@ class CommonColumns(BaseModel): # type: ignore
|
|
185
179
|
[s for c in cls.__bases__ for s in _all_bases(c)]
|
186
180
|
)
|
187
181
|
|
188
|
-
columns_to_check =
|
182
|
+
columns_to_check = list(type(self).__table__.columns)
|
189
183
|
for b in _all_bases(type(self)):
|
190
184
|
if hasattr(b, "__table__"):
|
191
185
|
columns_to_check.extend(b.__table__.columns)
|
@@ -337,7 +331,6 @@ class CommonColumns(BaseModel): # type: ignore
|
|
337
331
|
|
338
332
|
def validate(self):
|
339
333
|
"""Run custom validations on attributes set on this instance."""
|
340
|
-
pass
|
341
334
|
|
342
335
|
@classmethod
|
343
336
|
def get_unique_columns(cls):
|
@@ -433,7 +426,7 @@ class Users(CommonColumns):
|
|
433
426
|
|
434
427
|
user = Users.find_by_email(email)
|
435
428
|
if not user:
|
436
|
-
logger.info(
|
429
|
+
logger.info("Creating new user with email %s", email)
|
437
430
|
user = Users(
|
438
431
|
email=email, contact_email=email, first_n=first_n, last_n=last_n
|
439
432
|
)
|
@@ -456,7 +449,7 @@ class Users(CommonColumns):
|
|
456
449
|
.values(disabled=True)
|
457
450
|
.returning(Users.id)
|
458
451
|
)
|
459
|
-
disabled_user_ids: List[int] =
|
452
|
+
disabled_user_ids: List[int] = list(session.execute(update_query))
|
460
453
|
if commit:
|
461
454
|
session.commit()
|
462
455
|
|
@@ -659,7 +652,7 @@ class Permissions(CommonColumns):
|
|
659
652
|
raise IntegrityError(
|
660
653
|
params=None,
|
661
654
|
statement=None,
|
662
|
-
orig=
|
655
|
+
orig="`granted_by_user` user must be given",
|
663
656
|
)
|
664
657
|
if grantor is None:
|
665
658
|
raise IntegrityError(
|
@@ -677,9 +670,8 @@ class Permissions(CommonColumns):
|
|
677
670
|
orig=f"`file_group` must exist, but none found with id {self.file_group_id}",
|
678
671
|
)
|
679
672
|
|
680
|
-
|
681
|
-
|
682
|
-
)
|
673
|
+
info_message = f"admin-action: {grantor.email} gave {grantee.email} the permission {self.upload_type or 'all assays'}{'(' + file_group.name + ')' if file_group else ''} on {self.trial_id or 'all trials'}"
|
674
|
+
logger.info(info_message)
|
683
675
|
|
684
676
|
if self.upload_type == "file_group":
|
685
677
|
# Do not delete past-assigned file_group permissions for this user.
|
@@ -695,18 +687,22 @@ class Permissions(CommonColumns):
|
|
695
687
|
Permissions.granted_to_user == self.granted_to_user,
|
696
688
|
# If inserting a cross-trial perm, then select relevant
|
697
689
|
# trial-specific perms for deletion.
|
698
|
-
|
699
|
-
|
700
|
-
|
690
|
+
(
|
691
|
+
Permissions.trial_id != self.EVERY
|
692
|
+
if self.trial_id == self.EVERY
|
693
|
+
else Permissions.trial_id == self.trial_id
|
694
|
+
),
|
701
695
|
# If inserting a cross-upload type perm, then select relevant
|
702
696
|
# upload type-specific perms for deletion. This does NOT
|
703
697
|
# include clinical_data, just manifests/assays/analysis.
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
698
|
+
(
|
699
|
+
and_(
|
700
|
+
Permissions.upload_type != self.EVERY,
|
701
|
+
Permissions.upload_type != "clinical_data",
|
702
|
+
)
|
703
|
+
if self.upload_type == self.EVERY
|
704
|
+
else Permissions.upload_type == self.upload_type
|
705
|
+
),
|
710
706
|
)
|
711
707
|
.all()
|
712
708
|
)
|
@@ -784,7 +780,9 @@ class Permissions(CommonColumns):
|
|
784
780
|
revoke_download_access(grantee.email, self.trial_id, self.upload_type)
|
785
781
|
|
786
782
|
# If the permission to delete is the last one, also revoke Lister access
|
787
|
-
filter_
|
783
|
+
def filter_(q):
|
784
|
+
return q.filter(Permissions.granted_to_user == grantee.id)
|
785
|
+
|
788
786
|
if Permissions.count(session=session, filter_=filter_) <= 1:
|
789
787
|
# this one hasn't been deleted yet, so 1 means this is the last one
|
790
788
|
revoke_lister_access(grantee.email)
|
@@ -794,9 +792,8 @@ class Permissions(CommonColumns):
|
|
794
792
|
"IAM revoke failed, and permission db record not removed."
|
795
793
|
) from e
|
796
794
|
|
797
|
-
|
798
|
-
|
799
|
-
)
|
795
|
+
info_message = f"admin-action: {deleted_by_user.email} removed from {grantee.email} the permission {self.upload_type or 'all assays'} on {self.trial_id or 'all trials'}"
|
796
|
+
logger.info(info_message)
|
800
797
|
super().delete(session=session, commit=True)
|
801
798
|
|
802
799
|
@staticmethod
|
@@ -968,7 +965,7 @@ class Permissions(CommonColumns):
|
|
968
965
|
# if they have any download permissions, they need the CIDC Lister role
|
969
966
|
# If a Permission's FileGroup is None, that implies the Permission is a
|
970
967
|
# trial/assay type and thus lister access is required.
|
971
|
-
if len(perms) and any(
|
968
|
+
if len(perms) and any(perm.file_group_id is None for perm in perms):
|
972
969
|
grant_lister_access(user.email)
|
973
970
|
|
974
971
|
# separate permissions by trial, as they are strictly non-overlapping
|
@@ -1071,9 +1068,9 @@ class Permissions(CommonColumns):
|
|
1071
1068
|
or user.email in user_email_list
|
1072
1069
|
):
|
1073
1070
|
continue
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1071
|
+
|
1072
|
+
user_email_list.append(user.email)
|
1073
|
+
grant_lister_access(user.email)
|
1077
1074
|
|
1078
1075
|
if upload.upload_type in prism.SUPPORTED_SHIPPING_MANIFESTS:
|
1079
1076
|
# Passed with empty user email list because they will be queried for in CFn
|
@@ -1087,7 +1084,9 @@ class Permissions(CommonColumns):
|
|
1087
1084
|
# grant to individual blobs rather than calling grant_download_access(grantee, trial, upload_type)
|
1088
1085
|
blob_names: list[str] = [df.object_url for df in file_group.downloadable_files]
|
1089
1086
|
logger.info(
|
1090
|
-
|
1087
|
+
"Granting access to %s for the following downloadables: %s",
|
1088
|
+
user_email,
|
1089
|
+
blob_names,
|
1091
1090
|
)
|
1092
1091
|
grant_download_access_to_blob_names([user_email], blob_names)
|
1093
1092
|
|
@@ -1095,8 +1094,6 @@ class Permissions(CommonColumns):
|
|
1095
1094
|
class ValidationMultiError(Exception):
|
1096
1095
|
"""Holds multiple jsonschema.ValidationErrors"""
|
1097
1096
|
|
1098
|
-
pass
|
1099
|
-
|
1100
1097
|
|
1101
1098
|
trial_metadata_validator: json_validation._Validator = (
|
1102
1099
|
json_validation.load_and_validate_schema(
|
@@ -1230,7 +1227,7 @@ class TrialMetadata(CommonColumns):
|
|
1230
1227
|
Create a new clinical trial metadata record.
|
1231
1228
|
"""
|
1232
1229
|
|
1233
|
-
logger.info(
|
1230
|
+
logger.info("Creating new trial metadata with id %s", trial_id)
|
1234
1231
|
trial = TrialMetadata(trial_id=trial_id, metadata_json=metadata_json)
|
1235
1232
|
trial.insert(session=session, commit=commit)
|
1236
1233
|
|
@@ -1422,7 +1419,10 @@ class TrialMetadata(CommonColumns):
|
|
1422
1419
|
session.commit()
|
1423
1420
|
|
1424
1421
|
@classmethod
|
1425
|
-
def build_trial_filter(cls, user: Users, trial_ids: List[str] =
|
1422
|
+
def build_trial_filter(cls, user: Users, trial_ids: List[str] = None):
|
1423
|
+
if trial_ids is None:
|
1424
|
+
trial_ids = []
|
1425
|
+
|
1426
1426
|
filters = []
|
1427
1427
|
if trial_ids:
|
1428
1428
|
filters.append(cls.trial_id.in_(trial_ids))
|
@@ -1683,7 +1683,7 @@ class TrialMetadata(CommonColumns):
|
|
1683
1683
|
trial_metadata,
|
1684
1684
|
jsonb_array_elements(metadata_json->'participants') participant,
|
1685
1685
|
jsonb_array_elements(participant->'samples') sample
|
1686
|
-
|
1686
|
+
|
1687
1687
|
where
|
1688
1688
|
sample->>'processed_sample_derivative' = 'Tumor DNA'
|
1689
1689
|
or
|
@@ -1799,7 +1799,7 @@ class TrialMetadata(CommonColumns):
|
|
1799
1799
|
trial_id,
|
1800
1800
|
jsonb_object_agg(key, value) as value
|
1801
1801
|
from (
|
1802
|
-
select
|
1802
|
+
select
|
1803
1803
|
trial_id,
|
1804
1804
|
key,
|
1805
1805
|
jsonb_agg(sample) as value
|
@@ -2017,7 +2017,7 @@ class UploadJobs(CommonColumns):
|
|
2017
2017
|
__tablename__ = "upload_jobs"
|
2018
2018
|
# An upload job must contain a gcs_file_map is it isn't a manifest upload
|
2019
2019
|
__table_args__ = (
|
2020
|
-
CheckConstraint(
|
2020
|
+
CheckConstraint("multifile = true OR gcs_file_map != null"),
|
2021
2021
|
ForeignKeyConstraint(
|
2022
2022
|
["uploader_email"],
|
2023
2023
|
["users.email"],
|
@@ -2159,10 +2159,10 @@ class UploadJobs(CommonColumns):
|
|
2159
2159
|
if job is None or job.status == UploadJobStatus.MERGE_COMPLETED.value:
|
2160
2160
|
raise ValueError(f"Upload job {job_id} doesn't exist or is already merged")
|
2161
2161
|
|
2162
|
-
logger.info(
|
2162
|
+
logger.info("About to merge extra md to %s/%s", job.id, job.status)
|
2163
2163
|
|
2164
2164
|
for uuid, file in files.items():
|
2165
|
-
logger.info(
|
2165
|
+
logger.info("About to parse/merge extra md on %s", uuid)
|
2166
2166
|
(
|
2167
2167
|
job.metadata_patch,
|
2168
2168
|
updated_artifact,
|
@@ -2170,14 +2170,14 @@ class UploadJobs(CommonColumns):
|
|
2170
2170
|
) = prism.merge_artifact_extra_metadata(
|
2171
2171
|
job.metadata_patch, uuid, job.upload_type, file
|
2172
2172
|
)
|
2173
|
-
logger.info(
|
2173
|
+
logger.info("Updated md for %s: %s", uuid, updated_artifact.keys())
|
2174
2174
|
|
2175
2175
|
# A workaround fix for JSON field modifications not being tracked
|
2176
2176
|
# by SQLalchemy for some reason. Using MutableDict.as_mutable(JSON)
|
2177
2177
|
# in the model doesn't seem to help.
|
2178
2178
|
flag_modified(job, "metadata_patch")
|
2179
2179
|
|
2180
|
-
logger.info(
|
2180
|
+
logger.info("Updated %s/%s patch: %s", job.id, job.status, job.metadata_patch)
|
2181
2181
|
session.commit()
|
2182
2182
|
|
2183
2183
|
@classmethod
|
@@ -2323,15 +2323,15 @@ class DownloadableFiles(CommonColumns):
|
|
2323
2323
|
return match.group(1) if match else None
|
2324
2324
|
|
2325
2325
|
@file_ext.expression
|
2326
|
-
def file_ext(
|
2327
|
-
return func.substring(
|
2326
|
+
def file_ext(self):
|
2327
|
+
return func.substring(self.object_url, self.FILE_EXT_REGEX)
|
2328
2328
|
|
2329
2329
|
@hybrid_property
|
2330
2330
|
def data_category(self):
|
2331
2331
|
return facet_groups_to_categories.get(self.facet_group)
|
2332
2332
|
|
2333
2333
|
@data_category.expression
|
2334
|
-
def data_category(
|
2334
|
+
def data_category(self):
|
2335
2335
|
return DATA_CATEGORY_CASE_CLAUSE
|
2336
2336
|
|
2337
2337
|
@hybrid_property
|
@@ -2345,7 +2345,7 @@ class DownloadableFiles(CommonColumns):
|
|
2345
2345
|
return self.data_category.split(FACET_NAME_DELIM, 1)[0]
|
2346
2346
|
|
2347
2347
|
@data_category_prefix.expression
|
2348
|
-
def data_category_prefix(
|
2348
|
+
def data_category_prefix(self):
|
2349
2349
|
return func.split_part(DATA_CATEGORY_CASE_CLAUSE, FACET_NAME_DELIM, 1)
|
2350
2350
|
|
2351
2351
|
@hybrid_property
|
@@ -2353,7 +2353,7 @@ class DownloadableFiles(CommonColumns):
|
|
2353
2353
|
return details_dict.get(self.facet_group).file_purpose
|
2354
2354
|
|
2355
2355
|
@file_purpose.expression
|
2356
|
-
def file_purpose(
|
2356
|
+
def file_purpose(self):
|
2357
2357
|
return FILE_PURPOSE_CASE_CLAUSE
|
2358
2358
|
|
2359
2359
|
@property
|
@@ -2388,7 +2388,7 @@ class DownloadableFiles(CommonColumns):
|
|
2388
2388
|
# definitions in this class, but it seemed to be the only way to get the
|
2389
2389
|
# SQLAlchemy Expression Language API to work.
|
2390
2390
|
metadata = MetaData()
|
2391
|
-
|
2391
|
+
downloadable_files_for_query = Table(
|
2392
2392
|
"downloadable_files",
|
2393
2393
|
metadata,
|
2394
2394
|
Column("_created", BigInteger, nullable=False),
|
@@ -2415,24 +2415,20 @@ class DownloadableFiles(CommonColumns):
|
|
2415
2415
|
# TODO(jcallaway): consider this reflection-based approach instead. It doesn't
|
2416
2416
|
# currently work because of the relationship()s and foreign keys, among other
|
2417
2417
|
# problems.
|
2418
|
-
|
2419
|
-
|
2420
|
-
|
2421
|
-
|
2422
|
-
|
2423
|
-
|
2424
|
-
|
2425
|
-
|
2426
|
-
|
2427
|
-
|
2428
|
-
|
2429
|
-
|
2430
|
-
|
2431
|
-
|
2432
|
-
downloadableFiles_forQuery = Table("downloadable_files", metadata, *columns)
|
2433
|
-
"""
|
2434
|
-
|
2435
|
-
filesToFileGroups_forQuery = Table(
|
2418
|
+
# columns = []
|
2419
|
+
# attributes = inspect.getmembers(CommonColumns, lambda x: not (inspect.isroutine(x)))
|
2420
|
+
# for k, v in attributes:
|
2421
|
+
# if not(k.startswith('__') and k.endswith('__')) and isinstance(v, Column):
|
2422
|
+
# v.name = k
|
2423
|
+
# columns.append(v)
|
2424
|
+
# attributes = inspect.getmembers(DownloadableFiles, lambda x: not (inspect.isroutine(x)))
|
2425
|
+
# for k, v in attributes:
|
2426
|
+
# if not (k.startswith('__') and k.endswith('__')) and isinstance(v, Column):
|
2427
|
+
# v.name = k
|
2428
|
+
# columns.append(v)
|
2429
|
+
# downloadable_files_for_query = Table("downloadable_files", metadata, *columns)
|
2430
|
+
|
2431
|
+
files_to_file_groups_for_query = Table(
|
2436
2432
|
"files_to_file_groups",
|
2437
2433
|
metadata,
|
2438
2434
|
Column("_created", BigInteger, nullable=False),
|
@@ -2443,11 +2439,11 @@ class DownloadableFiles(CommonColumns):
|
|
2443
2439
|
Column("file_id", ForeignKey("downloadable_files.id")),
|
2444
2440
|
)
|
2445
2441
|
|
2446
|
-
return
|
2442
|
+
return downloadable_files_for_query, files_to_file_groups_for_query
|
2447
2443
|
|
2448
2444
|
@classmethod
|
2449
2445
|
def _convert_list_results(
|
2450
|
-
cls,
|
2446
|
+
cls, downloadable_files_for_query: Table, query_files: List
|
2451
2447
|
):
|
2452
2448
|
"""Converts the results of a SQLalchemy expression language query into actual DownloadableFiles
|
2453
2449
|
objects. This is necessary since the UI depends on some of the derived properties in
|
@@ -2456,7 +2452,7 @@ class DownloadableFiles(CommonColumns):
|
|
2456
2452
|
results = []
|
2457
2453
|
for query_file in query_files:
|
2458
2454
|
args = {}
|
2459
|
-
for column in
|
2455
|
+
for column in downloadable_files_for_query.c:
|
2460
2456
|
args[column.name] = getattr(query_file, column.name)
|
2461
2457
|
results.append(DownloadableFiles(**args))
|
2462
2458
|
|
@@ -2465,8 +2461,8 @@ class DownloadableFiles(CommonColumns):
|
|
2465
2461
|
@classmethod
|
2466
2462
|
def _generate_where_clauses(
|
2467
2463
|
cls,
|
2468
|
-
|
2469
|
-
|
2464
|
+
downloadable_files_for_query: Table,
|
2465
|
+
files_to_file_groups_for_query: Table,
|
2470
2466
|
trial_ids: List[str],
|
2471
2467
|
facets: List[List[str]],
|
2472
2468
|
user: Users,
|
@@ -2483,11 +2479,11 @@ class DownloadableFiles(CommonColumns):
|
|
2483
2479
|
|
2484
2480
|
where_clauses = []
|
2485
2481
|
if trial_ids:
|
2486
|
-
where_clauses.append(
|
2482
|
+
where_clauses.append(downloadable_files_for_query.c.trial_id.in_(trial_ids))
|
2487
2483
|
if facets:
|
2488
2484
|
facet_groups = get_facet_groups_for_paths(facets)
|
2489
2485
|
where_clauses.append(
|
2490
|
-
|
2486
|
+
downloadable_files_for_query.c.facet_group.in_(facet_groups)
|
2491
2487
|
)
|
2492
2488
|
|
2493
2489
|
if user and not is_admin:
|
@@ -2505,11 +2501,11 @@ class DownloadableFiles(CommonColumns):
|
|
2505
2501
|
where_clauses.append(
|
2506
2502
|
sql_and(
|
2507
2503
|
(
|
2508
|
-
|
2504
|
+
downloadable_files_for_query.c.trial_id
|
2509
2505
|
== permission.trial_id
|
2510
2506
|
),
|
2511
2507
|
(
|
2512
|
-
|
2508
|
+
downloadable_files_for_query.c.upload_type
|
2513
2509
|
== permission.upload_type
|
2514
2510
|
),
|
2515
2511
|
)
|
@@ -2518,11 +2514,11 @@ class DownloadableFiles(CommonColumns):
|
|
2518
2514
|
where_clauses.append(
|
2519
2515
|
sql_and(
|
2520
2516
|
(
|
2521
|
-
|
2517
|
+
downloadable_files_for_query.c.trial_id
|
2522
2518
|
== permission.trial_id
|
2523
2519
|
),
|
2524
2520
|
(
|
2525
|
-
|
2521
|
+
files_to_file_groups_for_query.c.file_group_id
|
2526
2522
|
== permission.file_group_id
|
2527
2523
|
),
|
2528
2524
|
)
|
@@ -2533,29 +2529,29 @@ class DownloadableFiles(CommonColumns):
|
|
2533
2529
|
# don't include clinical_data in cross-trial permission
|
2534
2530
|
where_clauses.append(
|
2535
2531
|
sql_and(
|
2536
|
-
|
2537
|
-
(
|
2532
|
+
downloadable_files_for_query.c.trial_id.in_(full_trial_perms),
|
2533
|
+
(downloadable_files_for_query.c.upload_type != "clinical_data"),
|
2538
2534
|
)
|
2539
2535
|
)
|
2540
2536
|
if full_type_perms:
|
2541
2537
|
where_clauses.append(
|
2542
|
-
|
2538
|
+
downloadable_files_for_query.c.upload_type.in_(full_type_perms)
|
2543
2539
|
)
|
2544
2540
|
|
2545
2541
|
# Need to be careful about return logic. Empty results could be because the user
|
2546
2542
|
# is an admin, whereas None means the user has no permissions to view any files.
|
2547
2543
|
if is_admin or where_clauses:
|
2548
2544
|
return where_clauses
|
2549
|
-
|
2550
|
-
|
2545
|
+
|
2546
|
+
return None
|
2551
2547
|
|
2552
2548
|
@classmethod
|
2553
2549
|
@with_default_session
|
2554
2550
|
def list_with_permissions(
|
2555
2551
|
cls,
|
2556
2552
|
session: Session,
|
2557
|
-
trial_ids: List[str] =
|
2558
|
-
facets: List[List[str]] =
|
2553
|
+
trial_ids: List[str] = None,
|
2554
|
+
facets: List[List[str]] = None,
|
2559
2555
|
page_num: int = 0,
|
2560
2556
|
page_size: int = PAGINATION_PAGE_SIZE,
|
2561
2557
|
sort_field: Optional[str] = None,
|
@@ -2563,13 +2559,19 @@ class DownloadableFiles(CommonColumns):
|
|
2563
2559
|
user: Users = None,
|
2564
2560
|
):
|
2565
2561
|
"""List records in this table, with pagination support."""
|
2562
|
+
if trial_ids is None:
|
2563
|
+
trial_ids = []
|
2564
|
+
|
2565
|
+
if facets is None:
|
2566
|
+
facets = []
|
2567
|
+
|
2566
2568
|
(
|
2567
|
-
|
2568
|
-
|
2569
|
+
downloadable_files_for_query,
|
2570
|
+
files_to_file_groups_for_query,
|
2569
2571
|
) = DownloadableFiles._generate_query_objects()
|
2570
2572
|
where_clauses = DownloadableFiles._generate_where_clauses(
|
2571
|
-
|
2572
|
-
|
2573
|
+
downloadable_files_for_query,
|
2574
|
+
files_to_file_groups_for_query,
|
2573
2575
|
trial_ids,
|
2574
2576
|
facets,
|
2575
2577
|
user,
|
@@ -2579,19 +2581,21 @@ class DownloadableFiles(CommonColumns):
|
|
2579
2581
|
# User doesn't have permissions to view any files; no need to issue a query.
|
2580
2582
|
return {}
|
2581
2583
|
|
2582
|
-
|
2584
|
+
if where_clauses:
|
2583
2585
|
|
2584
2586
|
# No where clause (the user is likely an admin).
|
2585
|
-
statement = select([
|
2586
|
-
|
2587
|
+
statement = select([downloadable_files_for_query]).select_from(
|
2588
|
+
downloadable_files_for_query
|
2587
2589
|
)
|
2588
2590
|
|
2589
2591
|
else:
|
2590
2592
|
statement = (
|
2591
|
-
select([
|
2593
|
+
select([downloadable_files_for_query])
|
2592
2594
|
.where(sql_and(*where_clauses))
|
2593
2595
|
.select_from(
|
2594
|
-
|
2596
|
+
downloadable_files_for_query.outerjoin(
|
2597
|
+
files_to_file_groups_for_query
|
2598
|
+
)
|
2595
2599
|
)
|
2596
2600
|
)
|
2597
2601
|
|
@@ -2610,7 +2614,7 @@ class DownloadableFiles(CommonColumns):
|
|
2610
2614
|
statement = statement.limit(page_size).offset(page_num * page_size)
|
2611
2615
|
|
2612
2616
|
return DownloadableFiles._convert_list_results(
|
2613
|
-
|
2617
|
+
downloadable_files_for_query, session.execute(statement).fetchall()
|
2614
2618
|
)
|
2615
2619
|
|
2616
2620
|
@classmethod
|
@@ -2618,25 +2622,32 @@ class DownloadableFiles(CommonColumns):
|
|
2618
2622
|
def count_with_permissions(
|
2619
2623
|
cls,
|
2620
2624
|
session: Session,
|
2621
|
-
trial_ids: List[str] =
|
2622
|
-
facets: List[List[str]] =
|
2623
|
-
page_num: int = 0,
|
2624
|
-
page_size: int = PAGINATION_PAGE_SIZE,
|
2625
|
-
sort_field: Optional[str] = None,
|
2626
|
-
sort_direction: Optional[str] = None,
|
2625
|
+
trial_ids: List[str] = None,
|
2626
|
+
facets: List[List[str]] = None,
|
2627
|
+
# page_num: int = 0,
|
2628
|
+
# page_size: int = PAGINATION_PAGE_SIZE,
|
2629
|
+
# sort_field: Optional[str] = None,
|
2630
|
+
# sort_direction: Optional[str] = None,
|
2627
2631
|
user: Users = None,
|
2628
2632
|
):
|
2629
2633
|
"""
|
2630
2634
|
Return the total number of records that would be returned by equivalent calls to
|
2631
2635
|
list_with_permissions() (disregarding results paging).
|
2632
2636
|
"""
|
2637
|
+
|
2638
|
+
if trial_ids is None:
|
2639
|
+
trial_ids = []
|
2640
|
+
|
2641
|
+
if facets is None:
|
2642
|
+
facets = []
|
2643
|
+
|
2633
2644
|
(
|
2634
|
-
|
2635
|
-
|
2645
|
+
downloadable_files_for_query,
|
2646
|
+
files_to_file_groups_for_query,
|
2636
2647
|
) = DownloadableFiles._generate_query_objects()
|
2637
2648
|
where_clauses = DownloadableFiles._generate_where_clauses(
|
2638
|
-
|
2639
|
-
|
2649
|
+
downloadable_files_for_query,
|
2650
|
+
files_to_file_groups_for_query,
|
2640
2651
|
trial_ids,
|
2641
2652
|
facets,
|
2642
2653
|
user,
|
@@ -2646,19 +2657,21 @@ class DownloadableFiles(CommonColumns):
|
|
2646
2657
|
# User doesn't have permissions to view any files; no need to issue a query.
|
2647
2658
|
return 0
|
2648
2659
|
|
2649
|
-
|
2660
|
+
if where_clauses:
|
2650
2661
|
|
2651
2662
|
# No where clause (the user is likely an admin).
|
2652
2663
|
statement = select(
|
2653
|
-
[func.count(
|
2654
|
-
).select_from(
|
2664
|
+
[func.count(downloadable_files_for_query.c.id)]
|
2665
|
+
).select_from(downloadable_files_for_query)
|
2655
2666
|
|
2656
2667
|
else:
|
2657
2668
|
statement = (
|
2658
|
-
select([func.count(
|
2669
|
+
select([func.count(downloadable_files_for_query.c.id)])
|
2659
2670
|
.where(sql_and(*where_clauses))
|
2660
2671
|
.select_from(
|
2661
|
-
|
2672
|
+
downloadable_files_for_query.outerjoin(
|
2673
|
+
files_to_file_groups_for_query
|
2674
|
+
)
|
2662
2675
|
)
|
2663
2676
|
)
|
2664
2677
|
|
@@ -2667,19 +2680,23 @@ class DownloadableFiles(CommonColumns):
|
|
2667
2680
|
@classmethod
|
2668
2681
|
@with_default_session
|
2669
2682
|
def count_by_facet_with_permissions(
|
2670
|
-
cls, session: Session, trial_ids: List[str] =
|
2683
|
+
cls, session: Session, trial_ids: List[str] = None, user: Users = None
|
2671
2684
|
):
|
2672
2685
|
"""
|
2673
2686
|
Returns a map of facet_group to a count of the number of files that the given user
|
2674
2687
|
has permissions to view.
|
2675
2688
|
"""
|
2689
|
+
|
2690
|
+
if trial_ids is None:
|
2691
|
+
trial_ids = []
|
2692
|
+
|
2676
2693
|
(
|
2677
|
-
|
2678
|
-
|
2694
|
+
downloadable_files_for_query,
|
2695
|
+
files_to_file_groups_for_query,
|
2679
2696
|
) = DownloadableFiles._generate_query_objects()
|
2680
2697
|
where_clauses = DownloadableFiles._generate_where_clauses(
|
2681
|
-
|
2682
|
-
|
2698
|
+
downloadable_files_for_query,
|
2699
|
+
files_to_file_groups_for_query,
|
2683
2700
|
trial_ids,
|
2684
2701
|
None,
|
2685
2702
|
user,
|
@@ -2689,33 +2706,35 @@ class DownloadableFiles(CommonColumns):
|
|
2689
2706
|
# User doesn't have permissions to view any files; no need to issue a query.
|
2690
2707
|
return {}
|
2691
2708
|
|
2692
|
-
|
2709
|
+
if not where_clauses:
|
2693
2710
|
|
2694
2711
|
# No where clause (the user is likely an admin).
|
2695
2712
|
statement = select(
|
2696
2713
|
[
|
2697
|
-
|
2698
|
-
func.count(
|
2714
|
+
downloadable_files_for_query.c.facet_group,
|
2715
|
+
func.count(downloadable_files_for_query.c.id),
|
2699
2716
|
]
|
2700
|
-
).select_from(
|
2717
|
+
).select_from(downloadable_files_for_query)
|
2701
2718
|
|
2702
2719
|
else:
|
2703
2720
|
statement = (
|
2704
2721
|
select(
|
2705
2722
|
[
|
2706
|
-
|
2707
|
-
func.count(
|
2723
|
+
downloadable_files_for_query.c.facet_group,
|
2724
|
+
func.count(downloadable_files_for_query.c.id),
|
2708
2725
|
]
|
2709
2726
|
)
|
2710
2727
|
.where(sql_and(*where_clauses))
|
2711
2728
|
.select_from(
|
2712
|
-
|
2729
|
+
downloadable_files_for_query.outerjoin(
|
2730
|
+
files_to_file_groups_for_query
|
2731
|
+
)
|
2713
2732
|
)
|
2714
2733
|
)
|
2715
2734
|
|
2716
|
-
statement = statement.group_by(
|
2735
|
+
statement = statement.group_by(downloadable_files_for_query.c.facet_group)
|
2717
2736
|
results = session.execute(statement).fetchall()
|
2718
|
-
return
|
2737
|
+
return dict(results)
|
2719
2738
|
|
2720
2739
|
@classmethod
|
2721
2740
|
@with_default_session
|
@@ -2727,12 +2746,12 @@ class DownloadableFiles(CommonColumns):
|
|
2727
2746
|
the object_urls for those the user has permission for.
|
2728
2747
|
"""
|
2729
2748
|
(
|
2730
|
-
|
2731
|
-
|
2749
|
+
downloadable_files_for_query,
|
2750
|
+
files_to_file_groups_for_query,
|
2732
2751
|
) = DownloadableFiles._generate_query_objects()
|
2733
2752
|
where_clauses = DownloadableFiles._generate_where_clauses(
|
2734
|
-
|
2735
|
-
|
2753
|
+
downloadable_files_for_query,
|
2754
|
+
files_to_file_groups_for_query,
|
2736
2755
|
None,
|
2737
2756
|
None,
|
2738
2757
|
user,
|
@@ -2742,23 +2761,25 @@ class DownloadableFiles(CommonColumns):
|
|
2742
2761
|
# User doesn't have permissions to view any files; no need to issue a query.
|
2743
2762
|
return {}
|
2744
2763
|
|
2745
|
-
|
2764
|
+
if not where_clauses:
|
2746
2765
|
|
2747
2766
|
# No where clause (the user is likely an admin).
|
2748
2767
|
statement = (
|
2749
|
-
select([
|
2750
|
-
.where(
|
2751
|
-
.select_from(
|
2768
|
+
select([downloadable_files_for_query.c.object_url])
|
2769
|
+
.where(downloadable_files_for_query.c.id.in_(ids))
|
2770
|
+
.select_from(downloadable_files_for_query)
|
2752
2771
|
)
|
2753
2772
|
|
2754
2773
|
else:
|
2755
2774
|
statement = (
|
2756
|
-
select([
|
2775
|
+
select([downloadable_files_for_query.c.object_url])
|
2757
2776
|
.where(
|
2758
|
-
sql_and(*where_clauses,
|
2777
|
+
sql_and(*where_clauses, downloadable_files_for_query.c.id.in_(ids))
|
2759
2778
|
)
|
2760
2779
|
.select_from(
|
2761
|
-
|
2780
|
+
downloadable_files_for_query.outerjoin(
|
2781
|
+
files_to_file_groups_for_query
|
2782
|
+
)
|
2762
2783
|
)
|
2763
2784
|
)
|
2764
2785
|
|
@@ -2786,12 +2807,12 @@ class DownloadableFiles(CommonColumns):
|
|
2786
2807
|
Files with a non-true data_category property are not included in these counts.
|
2787
2808
|
"""
|
2788
2809
|
(
|
2789
|
-
|
2790
|
-
|
2810
|
+
downloadable_files_for_query,
|
2811
|
+
files_to_file_groups_for_query,
|
2791
2812
|
) = DownloadableFiles._generate_query_objects()
|
2792
2813
|
where_clauses = DownloadableFiles._generate_where_clauses(
|
2793
|
-
|
2794
|
-
|
2814
|
+
downloadable_files_for_query,
|
2815
|
+
files_to_file_groups_for_query,
|
2795
2816
|
None,
|
2796
2817
|
facets,
|
2797
2818
|
user,
|
@@ -2801,25 +2822,26 @@ class DownloadableFiles(CommonColumns):
|
|
2801
2822
|
# User doesn't have permissions to view any files; no need to issue a query.
|
2802
2823
|
return {}
|
2803
2824
|
|
2804
|
-
|
2825
|
+
if not where_clauses:
|
2805
2826
|
|
2806
2827
|
# No where clause (the user is likely an admin).
|
2807
|
-
statement = select([
|
2808
|
-
|
2828
|
+
statement = select([downloadable_files_for_query]).select_from(
|
2829
|
+
downloadable_files_for_query
|
2809
2830
|
)
|
2810
2831
|
|
2811
2832
|
else:
|
2812
2833
|
statement = (
|
2813
|
-
select([
|
2834
|
+
select([downloadable_files_for_query])
|
2814
2835
|
.where(sql_and(*where_clauses))
|
2815
2836
|
.select_from(
|
2816
|
-
|
2837
|
+
downloadable_files_for_query.outerjoin(
|
2838
|
+
files_to_file_groups_for_query
|
2839
|
+
)
|
2817
2840
|
)
|
2818
2841
|
)
|
2819
2842
|
|
2820
|
-
results = session.execute(statement).fetchall()
|
2821
2843
|
downloadable_files = DownloadableFiles._convert_list_results(
|
2822
|
-
|
2844
|
+
downloadable_files_for_query, session.execute(statement).fetchall()
|
2823
2845
|
)
|
2824
2846
|
trial_file_counts = DownloadableFiles._generate_trial_file_counts(
|
2825
2847
|
downloadable_files
|
@@ -2872,7 +2894,7 @@ class DownloadableFiles(CommonColumns):
|
|
2872
2894
|
# TODO use old serach code again
|
2873
2895
|
@staticmethod
|
2874
2896
|
def build_file_filter(
|
2875
|
-
trial_ids: List[str] =
|
2897
|
+
trial_ids: List[str] = None, facets: List[List[str]] = None, user: Users = None
|
2876
2898
|
) -> Callable[[Query], Query]:
|
2877
2899
|
"""
|
2878
2900
|
Build a file filter function based on the provided parameters. The resultant
|
@@ -2887,6 +2909,12 @@ class DownloadableFiles(CommonColumns):
|
|
2887
2909
|
Returns:
|
2888
2910
|
A function that adds filters to a query against the DownloadableFiles table.
|
2889
2911
|
"""
|
2912
|
+
if trial_ids is None:
|
2913
|
+
trial_ids = []
|
2914
|
+
|
2915
|
+
if facets is None:
|
2916
|
+
facets = []
|
2917
|
+
|
2890
2918
|
file_filters = []
|
2891
2919
|
if trial_ids:
|
2892
2920
|
file_filters.append(DownloadableFiles.trial_id.in_(trial_ids))
|
@@ -2947,10 +2975,10 @@ class DownloadableFiles(CommonColumns):
|
|
2947
2975
|
"additional_metadata": additional_metadata,
|
2948
2976
|
}
|
2949
2977
|
|
2978
|
+
# TODO maybe put non supported stuff from file_metadata to some misc jsonb column?
|
2950
2979
|
for key, value in file_metadata.items():
|
2951
2980
|
if key in supported_columns:
|
2952
2981
|
filtered_metadata[key] = value
|
2953
|
-
# TODO maybe put non supported stuff from file_metadata to some misc jsonb column?
|
2954
2982
|
|
2955
2983
|
etag = make_etag(filtered_metadata.values())
|
2956
2984
|
|