nci-cidc-api-modules 1.2.9__py3-none-any.whl → 1.2.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cidc_api/config/settings.py +5 -0
- cidc_api/models/models.py +27 -23
- cidc_api/shared/email_layout.html +258 -0
- cidc_api/shared/emails.py +32 -2
- cidc_api/shared/file_handling.py +15 -5
- cidc_api/shared/gcloud_client.py +13 -4
- cidc_api/shared/utils.py +8 -0
- {nci_cidc_api_modules-1.2.9.dist-info → nci_cidc_api_modules-1.2.16.dist-info}/METADATA +14 -14
- {nci_cidc_api_modules-1.2.9.dist-info → nci_cidc_api_modules-1.2.16.dist-info}/RECORD +12 -10
- {nci_cidc_api_modules-1.2.9.dist-info → nci_cidc_api_modules-1.2.16.dist-info}/WHEEL +0 -0
- {nci_cidc_api_modules-1.2.9.dist-info → nci_cidc_api_modules-1.2.16.dist-info}/licenses/LICENSE +0 -0
- {nci_cidc_api_modules-1.2.9.dist-info → nci_cidc_api_modules-1.2.16.dist-info}/top_level.txt +0 -0
cidc_api/config/settings.py
CHANGED
|
@@ -83,6 +83,7 @@ GOOGLE_EMAILS_TOPIC = environ["GOOGLE_EMAILS_TOPIC"]
|
|
|
83
83
|
GOOGLE_ARTIFACT_UPLOAD_TOPIC = environ["GOOGLE_ARTIFACT_UPLOAD_TOPIC"]
|
|
84
84
|
GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC = environ["GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC"]
|
|
85
85
|
GOOGLE_HL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_HL_CLINICAL_VALIDATION_TOPIC"]
|
|
86
|
+
GOOGLE_DL_CLINICAL_VALIDATION_TOPIC = environ["GOOGLE_DL_CLINICAL_VALIDATION_TOPIC"]
|
|
86
87
|
GOOGLE_AND_OPERATOR = " && "
|
|
87
88
|
GOOGLE_OR_OPERATOR = " || "
|
|
88
89
|
|
|
@@ -107,5 +108,9 @@ else:
|
|
|
107
108
|
IS_EMAIL_ON = environ.get("IS_EMAIL_ON")
|
|
108
109
|
|
|
109
110
|
|
|
111
|
+
# notification emails
|
|
112
|
+
CIDC_CLINICAL_DATA_EMAIL = environ.get("CIDC_CLINICAL_DATA_EMAIL")
|
|
113
|
+
CIDC_ADMIN_EMAIL = environ.get("CIDC_ADMIN_EMAIL")
|
|
114
|
+
|
|
110
115
|
# Accumulate all constants defined in this file in a single dictionary
|
|
111
116
|
SETTINGS = {k: v for k, v in globals().items() if k.isupper()}
|
cidc_api/models/models.py
CHANGED
|
@@ -26,10 +26,12 @@ __all__ = [
|
|
|
26
26
|
"FileValidationErrors",
|
|
27
27
|
"IngestionJobs",
|
|
28
28
|
"JobFileCategories",
|
|
29
|
+
"CategoryDataElements",
|
|
29
30
|
"ValidationConfigs",
|
|
30
31
|
"TRIAL_APPENDIX_A",
|
|
31
32
|
"TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER",
|
|
32
33
|
"REQUEST_LETTER",
|
|
34
|
+
"DETAILED_VALIDATION",
|
|
33
35
|
"ADMIN_FILE_CATEGORIES",
|
|
34
36
|
"FINAL_JOB_STATUS",
|
|
35
37
|
"INGESTION_JOB_STATUSES",
|
|
@@ -130,7 +132,6 @@ from ..config.settings import (
|
|
|
130
132
|
MAX_PAGINATION_PAGE_SIZE,
|
|
131
133
|
TESTING,
|
|
132
134
|
INACTIVE_USER_DAYS,
|
|
133
|
-
GOOGLE_CLINICAL_DATA_BUCKET,
|
|
134
135
|
)
|
|
135
136
|
from ..shared import emails
|
|
136
137
|
from ..shared.gcloud_client import (
|
|
@@ -144,7 +145,6 @@ from ..shared.gcloud_client import (
|
|
|
144
145
|
revoke_intake_access,
|
|
145
146
|
revoke_lister_access,
|
|
146
147
|
revoke_bigquery_access,
|
|
147
|
-
gcs_xlsx_or_csv_file_to_pandas_dataframe,
|
|
148
148
|
)
|
|
149
149
|
|
|
150
150
|
os.environ["TZ"] = "UTC"
|
|
@@ -3449,30 +3449,9 @@ class IngestionJobs(CommonColumns):
|
|
|
3449
3449
|
|
|
3450
3450
|
@with_default_session
|
|
3451
3451
|
def transition_status(self, status: str, session: Session):
|
|
3452
|
-
# create required categories after opening job for submission
|
|
3453
|
-
if self.status == "DRAFT" and status == "INITIAL SUBMISSION":
|
|
3454
|
-
for category in self.derive_required_categories_from_appendix_a():
|
|
3455
|
-
JobFileCategories.create(category=category, job_id=self.id, type="required")
|
|
3456
3452
|
self.status = status
|
|
3457
3453
|
self.update(session=session)
|
|
3458
3454
|
|
|
3459
|
-
def derive_required_categories_from_appendix_a(self) -> List:
|
|
3460
|
-
appendix_a = PreprocessedFiles.get_files_by_category_and_status(TRIAL_APPENDIX_A, "current", job_id=self.id)[0]
|
|
3461
|
-
df = gcs_xlsx_or_csv_file_to_pandas_dataframe(GOOGLE_CLINICAL_DATA_BUCKET, appendix_a.object_url)
|
|
3462
|
-
categories = []
|
|
3463
|
-
headers_ended = False
|
|
3464
|
-
for _index, row in df.iterrows():
|
|
3465
|
-
cell = str(row.iloc[0])
|
|
3466
|
-
if headers_ended:
|
|
3467
|
-
if cell != "nan" and cell not in categories:
|
|
3468
|
-
categories.append(cell)
|
|
3469
|
-
elif cell.lower() == TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER.lower():
|
|
3470
|
-
headers_ended = True
|
|
3471
|
-
if "data_dictionary" not in categories:
|
|
3472
|
-
# Ensure Data_Dictionary is always a required file category
|
|
3473
|
-
categories.append("data_dictionary")
|
|
3474
|
-
return categories
|
|
3475
|
-
|
|
3476
3455
|
@classmethod
|
|
3477
3456
|
@with_default_session
|
|
3478
3457
|
def atomic_set_job_as_pending(cls, job_id: int, session: Session) -> Boolean:
|
|
@@ -3552,6 +3531,7 @@ class JobFileCategories(CommonColumns):
|
|
|
3552
3531
|
category = Column(String)
|
|
3553
3532
|
job_id = Column(Integer)
|
|
3554
3533
|
type = Column(Enum("required", "optional", name="type"))
|
|
3534
|
+
is_custom = Column(Boolean, default=False, server_default="false")
|
|
3555
3535
|
|
|
3556
3536
|
@staticmethod
|
|
3557
3537
|
@with_default_session
|
|
@@ -3559,12 +3539,14 @@ class JobFileCategories(CommonColumns):
|
|
|
3559
3539
|
category: str,
|
|
3560
3540
|
job_id: int,
|
|
3561
3541
|
type: str,
|
|
3542
|
+
is_custom: bool = False,
|
|
3562
3543
|
session: Session = None,
|
|
3563
3544
|
):
|
|
3564
3545
|
new_category = JobFileCategories(
|
|
3565
3546
|
category=category,
|
|
3566
3547
|
job_id=job_id,
|
|
3567
3548
|
type=type,
|
|
3549
|
+
is_custom=is_custom,
|
|
3568
3550
|
)
|
|
3569
3551
|
new_category.insert(session=session)
|
|
3570
3552
|
return new_category
|
|
@@ -3576,6 +3558,28 @@ class JobFileCategories(CommonColumns):
|
|
|
3576
3558
|
return [c.category for c in categories]
|
|
3577
3559
|
|
|
3578
3560
|
|
|
3561
|
+
class CategoryDataElements(CommonColumns):
|
|
3562
|
+
__tablename__ = "category_data_elements"
|
|
3563
|
+
__table_args__ = (
|
|
3564
|
+
ForeignKeyConstraint(
|
|
3565
|
+
["category_id"],
|
|
3566
|
+
["job_file_categories.id"],
|
|
3567
|
+
ondelete="CASCADE",
|
|
3568
|
+
),
|
|
3569
|
+
Index(
|
|
3570
|
+
"idx_elements_category_id" "category_id",
|
|
3571
|
+
"name",
|
|
3572
|
+
unique=True,
|
|
3573
|
+
),
|
|
3574
|
+
)
|
|
3575
|
+
|
|
3576
|
+
category_id = Column(Integer, nullable=False)
|
|
3577
|
+
name = Column(String, nullable=False)
|
|
3578
|
+
is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
|
|
3579
|
+
element_type = Column(String, nullable=False)
|
|
3580
|
+
cardinality = Column(String, nullable=True)
|
|
3581
|
+
|
|
3582
|
+
|
|
3579
3583
|
class FileValidationErrors(CommonColumns):
|
|
3580
3584
|
__tablename__ = "file_validation_errors"
|
|
3581
3585
|
__table_args__ = (
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
|
|
4
|
+
<head>
|
|
5
|
+
<title>Manifest Status Notification</title>
|
|
6
|
+
<meta charset="utf-8">
|
|
7
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
8
|
+
<meta http-equiv="Content-Type" content="text/html charset=UTF-8" />
|
|
9
|
+
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
|
10
|
+
<style type="text/css">
|
|
11
|
+
/* CLIENT-SPECIFIC STYLES */
|
|
12
|
+
body,
|
|
13
|
+
table,
|
|
14
|
+
td,
|
|
15
|
+
a {
|
|
16
|
+
-webkit-text-size-adjust: 100%;
|
|
17
|
+
-ms-text-size-adjust: 100%;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/* Prevent WebKit and Windows mobile changing default text sizes */
|
|
21
|
+
table,
|
|
22
|
+
td {
|
|
23
|
+
mso-table-lspace: 0pt;
|
|
24
|
+
mso-table-rspace: 0pt;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/* Remove spacing between tables in Outlook 2007 and up */
|
|
28
|
+
img {
|
|
29
|
+
-ms-interpolation-mode: bicubic;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/* Allow smoother rendering of resized image in Internet Explorer */
|
|
33
|
+
|
|
34
|
+
/* RESET STYLES */
|
|
35
|
+
img {
|
|
36
|
+
border: 0;
|
|
37
|
+
height: auto;
|
|
38
|
+
line-height: 100%;
|
|
39
|
+
outline: none;
|
|
40
|
+
text-decoration: none;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
table {
|
|
44
|
+
border-collapse: collapse !important;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
body {
|
|
48
|
+
height: 100% !important;
|
|
49
|
+
margin: 0 !important;
|
|
50
|
+
padding: 0 !important;
|
|
51
|
+
width: 100% !important;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/* iOS BLUE LINKS */
|
|
55
|
+
a[x-apple-data-detectors] {
|
|
56
|
+
color: inherit !important;
|
|
57
|
+
text-decoration: none !important;
|
|
58
|
+
font-size: inherit !important;
|
|
59
|
+
font-family: inherit !important;
|
|
60
|
+
font-weight: inherit !important;
|
|
61
|
+
line-height: inherit !important;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/* MOBILE STYLES */
|
|
65
|
+
@media screen and (max-width: 525px) {
|
|
66
|
+
|
|
67
|
+
/* ALLOWS FOR FLUID TABLES */
|
|
68
|
+
.wrapper {
|
|
69
|
+
width: 100% !important;
|
|
70
|
+
max-width: 100% !important;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/* ADJUSTS LAYOUT OF LOGO IMAGE */
|
|
74
|
+
.logo img {
|
|
75
|
+
margin: 0 auto !important;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/* USE THESE CLASSES TO HIDE CONTENT ON MOBILE */
|
|
79
|
+
.mobile-hide {
|
|
80
|
+
display: none !important;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
.img-max {
|
|
84
|
+
max-width: 100% !important;
|
|
85
|
+
width: 100% !important;
|
|
86
|
+
height: auto !important;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/* FULL-WIDTH TABLES */
|
|
90
|
+
.responsive-table {
|
|
91
|
+
width: 100% !important;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/* UTILITY CLASSES FOR ADJUSTING PADDING ON MOBILE */
|
|
95
|
+
.padding {
|
|
96
|
+
padding: 10px 5% 15px 5% !important;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
.padding-meta {
|
|
100
|
+
padding: 30px 5% 0px 5% !important;
|
|
101
|
+
text-align: center;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
.padding-copy {
|
|
105
|
+
padding: 10px 5% 10px 5% !important;
|
|
106
|
+
text-align: center;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
.no-padding {
|
|
110
|
+
padding: 0 !important;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
.section-padding {
|
|
114
|
+
padding: 50px 15px 50px 15px !important;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/* ADJUST BUTTONS ON MOBILE */
|
|
118
|
+
.mobile-button-container {
|
|
119
|
+
margin: 0 auto;
|
|
120
|
+
width: 100% !important;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
.mobile-button {
|
|
124
|
+
padding: 15px !important;
|
|
125
|
+
border: 0 !important;
|
|
126
|
+
font-size: 16px !important;
|
|
127
|
+
display: block !important;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/* ANDROID CENTER FIX */
|
|
133
|
+
div[style*="margin: 16px 0;"] {
|
|
134
|
+
margin: 0 !important;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
a {
|
|
138
|
+
color: white;
|
|
139
|
+
text-decoration: underline;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
.blue-link {
|
|
143
|
+
color: #1B6B82;
|
|
144
|
+
}
|
|
145
|
+
</style>
|
|
146
|
+
</head>
|
|
147
|
+
|
|
148
|
+
<body style="margin: 0 !important; padding: 0 !important; background-color:#F5F5F5;">
|
|
149
|
+
|
|
150
|
+
<!-- ONE COLUMN SECTION -->
|
|
151
|
+
<table bgcolor="#F5F5F5" border="0" cellpadding="0" cellspacing="0" width="100%">
|
|
152
|
+
|
|
153
|
+
<tr>
|
|
154
|
+
<td align="center" style="padding: 60px 16px 16px; margin-top: -32px;" class="section-padding">
|
|
155
|
+
<!--[if (gte mso 9)|(IE)]>
|
|
156
|
+
<table align="center" border="0" cellspacing="0" cellpadding="0" width="500">
|
|
157
|
+
<tr>
|
|
158
|
+
<td align="center" valign="top" width="500">
|
|
159
|
+
<![endif]-->
|
|
160
|
+
<table bgcolor="#ffffff" border="0" cellpadding="0" cellspacing="0" width="100%" style="max-width: 520px;"
|
|
161
|
+
class="responsive-table">
|
|
162
|
+
<tr>
|
|
163
|
+
<td>
|
|
164
|
+
<table width="100%" border="0" cellspacing="0" cellpadding="0">
|
|
165
|
+
<tr>
|
|
166
|
+
<td>
|
|
167
|
+
<table width="100%" border="0" cellspacing="0" cellpadding="0">
|
|
168
|
+
<tr>
|
|
169
|
+
<td align="center"
|
|
170
|
+
style="padding: 32px 0 0 0; font-size: 18px; line-height: 25px; font-family: Helvetica, Arial, sans-serif; color: #3F414F;"
|
|
171
|
+
class="padding-copy">
|
|
172
|
+
Cancer Immunologic Data Center (CIDC)</td>
|
|
173
|
+
</tr>
|
|
174
|
+
<tr>
|
|
175
|
+
<td align="center"
|
|
176
|
+
style="padding: 50px 32px 0 32px; line-height: 42px; font-family: Helvetica, Arial, sans-serif; color: #666666;">
|
|
177
|
+
<h1 style="margin:0; mso-line-height-rule:exactly; padding: 0 32px 0 32px">
|
|
178
|
+
$HEADING
|
|
179
|
+
</h1>
|
|
180
|
+
</td>
|
|
181
|
+
</tr>
|
|
182
|
+
<tr>
|
|
183
|
+
<td align="center"
|
|
184
|
+
style="padding: 40px 40px 40px 40px; font-size: 16px; line-height: 25px; font-family: Helvetica, Arial, sans-serif; color: #666666;"
|
|
185
|
+
class="padding-copy">
|
|
186
|
+
$CONTENT
|
|
187
|
+
</td>
|
|
188
|
+
</tr>
|
|
189
|
+
</table>
|
|
190
|
+
</td>
|
|
191
|
+
</tr>
|
|
192
|
+
</table>
|
|
193
|
+
</td>
|
|
194
|
+
</tr>
|
|
195
|
+
</table>
|
|
196
|
+
<!--[if (gte mso 9)|(IE)]>
|
|
197
|
+
</td>
|
|
198
|
+
</tr>
|
|
199
|
+
</table>
|
|
200
|
+
<![endif]-->
|
|
201
|
+
</td>
|
|
202
|
+
</tr>
|
|
203
|
+
<tr>
|
|
204
|
+
<td align="center" style="padding: 20px 0px;">
|
|
205
|
+
<!--[if (gte mso 9)|(IE)]>
|
|
206
|
+
<table align="center" border="0" cellspacing="0" cellpadding="0" width="500">
|
|
207
|
+
<tr>
|
|
208
|
+
<td align="center" valign="top" width="500">
|
|
209
|
+
<![endif]-->
|
|
210
|
+
<table bgcolor="#1B6B82" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"
|
|
211
|
+
style="max-width: 520px; " class="responsive-table">
|
|
212
|
+
<tr>
|
|
213
|
+
<td align="center"
|
|
214
|
+
style="padding: 20px 0 0 0; font-size: 18px; line-height: 18px; font-family: Helvetica, Arial, sans-serif; color:#fff; text-align: center">
|
|
215
|
+
Need help?
|
|
216
|
+
</td>
|
|
217
|
+
</tr>
|
|
218
|
+
<tr>
|
|
219
|
+
<td align="center"
|
|
220
|
+
style="padding: 10px 0 20px 0; font-size: 14px; line-height: 18px; font-family: Helvetica, Arial, sans-serif; color:#fff; text-align: center">
|
|
221
|
+
Send us an email at <a href="mailto:ncicidcadmin@mail.nih.gov">ncicidcadmin@mail.nih.gov</a>
|
|
222
|
+
</td>
|
|
223
|
+
</tr>
|
|
224
|
+
</table>
|
|
225
|
+
<!--[if (gte mso 9)|(IE)]>
|
|
226
|
+
</td>
|
|
227
|
+
</tr>
|
|
228
|
+
</table>
|
|
229
|
+
<![endif]-->
|
|
230
|
+
</td>
|
|
231
|
+
</tr>
|
|
232
|
+
<tr>
|
|
233
|
+
<td align="center" style="padding: 20px 0px;">
|
|
234
|
+
<!--[if (gte mso 9)|(IE)]>
|
|
235
|
+
<table align="center" border="0" cellspacing="0" cellpadding="0" width="500">
|
|
236
|
+
<tr>
|
|
237
|
+
<td align="center" valign="top" width="500">
|
|
238
|
+
<![endif]-->
|
|
239
|
+
<table width="100%" border="0" cellspacing="0" cellpadding="0" align="center" style="max-width: 520px; "
|
|
240
|
+
class="responsive-table">
|
|
241
|
+
<tr>
|
|
242
|
+
<td align="center"
|
|
243
|
+
style="padding: 10px 0 20px 0; font-size: 14px; line-height: 18px; font-family: Helvetica, Arial, sans-serif; color:#666666;">
|
|
244
|
+
National Cancer Institute at the National institutes of Health
|
|
245
|
+
</td>
|
|
246
|
+
</tr>
|
|
247
|
+
</table>
|
|
248
|
+
<!--[if (gte mso 9)|(IE)]>
|
|
249
|
+
</td>
|
|
250
|
+
</tr>
|
|
251
|
+
</table>
|
|
252
|
+
<![endif]-->
|
|
253
|
+
</td>
|
|
254
|
+
</tr>
|
|
255
|
+
</table>
|
|
256
|
+
</body>
|
|
257
|
+
|
|
258
|
+
</html>
|
cidc_api/shared/emails.py
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
"""Template functions for CIDC email bodies."""
|
|
2
2
|
|
|
3
3
|
import html
|
|
4
|
+
import pathlib
|
|
4
5
|
from functools import wraps
|
|
5
6
|
from typing import List
|
|
6
7
|
|
|
7
8
|
from . import gcloud_client
|
|
8
|
-
from ..config.settings import ENV
|
|
9
|
+
from ..config.settings import ENV, ALLOWED_CLIENT_URL, CIDC_CLINICAL_DATA_EMAIL, CIDC_ADMIN_EMAIL
|
|
9
10
|
|
|
10
11
|
# emails this list for
|
|
11
12
|
# - new user registration,
|
|
@@ -44,7 +45,7 @@ def confirm_account_approval(user) -> dict:
|
|
|
44
45
|
html_content = f"""
|
|
45
46
|
<p>Hello {user.first_n},</p>
|
|
46
47
|
<p>
|
|
47
|
-
Your CIMAC-CIDC Portal account has been approved!
|
|
48
|
+
Your CIMAC-CIDC Portal account has been approved!
|
|
48
49
|
To begin browsing and downloading data, visit https://cidc.nci.nih.gov.
|
|
49
50
|
</p>
|
|
50
51
|
<p>
|
|
@@ -150,3 +151,32 @@ def notify_mailing_list(subject: str, html_content: str) -> dict:
|
|
|
150
151
|
"html_content": html_content,
|
|
151
152
|
}
|
|
152
153
|
return email
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
with open(pathlib.Path(__file__).parent.joinpath("email_layout.html"), encoding="utf-8") as file:
|
|
157
|
+
EMAIL_LAYOUT = file.read()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@sendable
|
|
161
|
+
def new_validation_review(job: dict) -> dict:
|
|
162
|
+
# these two emails are set only in prod env file
|
|
163
|
+
if CIDC_CLINICAL_DATA_EMAIL and CIDC_ADMIN_EMAIL:
|
|
164
|
+
to_emails = [CIDC_CLINICAL_DATA_EMAIL, CIDC_ADMIN_EMAIL]
|
|
165
|
+
else:
|
|
166
|
+
to_emails = CIDC_MAILING_LIST
|
|
167
|
+
|
|
168
|
+
subject = f"Trial {job.trial_id} ({job.version}) Ready for Validation Review"
|
|
169
|
+
html_heading = "Clinical Data Upload Notification"
|
|
170
|
+
job_url = f"{ALLOWED_CLIENT_URL}/upload-clinical-data/{job.id}"
|
|
171
|
+
|
|
172
|
+
html_content = f"""
|
|
173
|
+
<b>Ready for Validation Review:</b><br>
|
|
174
|
+
<b><a class="blue-link" href={job_url}>Trial {job.trial_id} version {job.version} (Job ID {job.id})</a></b>
|
|
175
|
+
Please follow the link to access the trial's clinical data submission page.
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
return {
|
|
179
|
+
"to_emails": to_emails,
|
|
180
|
+
"subject": subject,
|
|
181
|
+
"html_content": EMAIL_LAYOUT.replace("$HEADING", html_heading).replace("$CONTENT", html_content),
|
|
182
|
+
}
|
cidc_api/shared/file_handling.py
CHANGED
|
@@ -8,22 +8,31 @@ from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
|
|
|
8
8
|
from ..models import PreprocessedFiles
|
|
9
9
|
from ..shared.auth import get_current_user
|
|
10
10
|
from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
|
|
11
|
+
from sqlalchemy.orm.session import Session
|
|
11
12
|
|
|
12
13
|
logger = get_logger(__name__)
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
def set_current_file(
|
|
16
|
+
def set_current_file(
|
|
17
|
+
file: FileStorage, file_category: str, gcs_folder: str, session: Session, uploader_email: str, job_id: int = None
|
|
18
|
+
) -> PreprocessedFiles:
|
|
16
19
|
"""
|
|
17
20
|
Archives any existing 'current' files for the given category and job,
|
|
18
21
|
then uploads the new file as the latest 'current' version.
|
|
19
22
|
"""
|
|
20
|
-
latest_version = PreprocessedFiles.archive_current_files(file_category, job_id=job_id)
|
|
21
|
-
latest_file = create_file(file, gcs_folder, file_category, job_id, latest_version + 1)
|
|
23
|
+
latest_version = PreprocessedFiles.archive_current_files(file_category, job_id=job_id, session=session)
|
|
24
|
+
latest_file = create_file(file, gcs_folder, file_category, session, uploader_email, job_id, latest_version + 1)
|
|
22
25
|
return latest_file
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
def create_file(
|
|
26
|
-
file: FileStorage,
|
|
29
|
+
file: FileStorage,
|
|
30
|
+
gcs_folder: str,
|
|
31
|
+
file_category: str,
|
|
32
|
+
session: Session,
|
|
33
|
+
uploader_email: str,
|
|
34
|
+
job_id: int = None,
|
|
35
|
+
version: int = None,
|
|
27
36
|
) -> PreprocessedFiles:
|
|
28
37
|
"""Upload file to GCS and create corresponding metadata record in the database."""
|
|
29
38
|
status = "pending" if gcs_folder.endswith("pending/") else "current"
|
|
@@ -36,10 +45,11 @@ def create_file(
|
|
|
36
45
|
file_name=file.filename,
|
|
37
46
|
object_url=gcs_file_path,
|
|
38
47
|
file_category=file_category,
|
|
39
|
-
uploader_email=
|
|
48
|
+
uploader_email=uploader_email,
|
|
40
49
|
status=status,
|
|
41
50
|
job_id=job_id,
|
|
42
51
|
version=version,
|
|
52
|
+
session=session,
|
|
43
53
|
)
|
|
44
54
|
return file
|
|
45
55
|
|
cidc_api/shared/gcloud_client.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Utilities for interacting with the Google Cloud Platform APIs."""
|
|
2
2
|
|
|
3
|
-
# pylint: disable=logging-fstring-interpolation
|
|
3
|
+
# pylint: disable=logging-fstring-interpolation,too-many-lines
|
|
4
4
|
|
|
5
5
|
import base64
|
|
6
6
|
import datetime
|
|
@@ -37,8 +37,8 @@ from sqlalchemy.orm.session import Session
|
|
|
37
37
|
from werkzeug.datastructures import FileStorage
|
|
38
38
|
from werkzeug.utils import secure_filename
|
|
39
39
|
|
|
40
|
-
from cidc_api.config.secrets import get_secrets_manager
|
|
41
40
|
from ..config.logging import get_logger
|
|
41
|
+
from ..config.secrets import get_secrets_manager
|
|
42
42
|
from ..config.settings import (
|
|
43
43
|
DEV_USE_GCS,
|
|
44
44
|
GOOGLE_INTAKE_ROLE,
|
|
@@ -55,12 +55,15 @@ from ..config.settings import (
|
|
|
55
55
|
GOOGLE_ARTIFACT_UPLOAD_TOPIC,
|
|
56
56
|
GOOGLE_GRANT_DOWNLOAD_PERMISSIONS_TOPIC,
|
|
57
57
|
GOOGLE_HL_CLINICAL_VALIDATION_TOPIC,
|
|
58
|
+
GOOGLE_DL_CLINICAL_VALIDATION_TOPIC,
|
|
58
59
|
TESTING,
|
|
59
60
|
ENV,
|
|
60
61
|
IS_EMAIL_ON,
|
|
61
62
|
DEV_CFUNCTIONS_SERVER,
|
|
62
63
|
INACTIVE_USER_DAYS,
|
|
63
64
|
)
|
|
65
|
+
from ..shared.utils import strip_whitespaces
|
|
66
|
+
|
|
64
67
|
|
|
65
68
|
os.environ["TZ"] = "UTC"
|
|
66
69
|
logger = get_logger(__name__)
|
|
@@ -426,9 +429,9 @@ def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
|
|
|
426
429
|
|
|
427
430
|
# TODO: specify sheet in xlsx file and/or accept tsv and xls files
|
|
428
431
|
if blob_name[-3:] == "csv":
|
|
429
|
-
return pd.read_csv(temp_file)
|
|
432
|
+
return strip_whitespaces(pd.read_csv(temp_file))
|
|
430
433
|
elif blob_name[-4:] == "xlsx":
|
|
431
|
-
return pd.read_excel(temp_file)
|
|
434
|
+
return strip_whitespaces(pd.read_excel(temp_file))
|
|
432
435
|
else:
|
|
433
436
|
raise Exception("Can only read csv or xlsx files")
|
|
434
437
|
|
|
@@ -985,6 +988,12 @@ def publish_hl_clinical_validation(job_id: int) -> None:
|
|
|
985
988
|
_report = _encode_and_publish(str(job_id), GOOGLE_HL_CLINICAL_VALIDATION_TOPIC)
|
|
986
989
|
|
|
987
990
|
|
|
991
|
+
def publish_detailed_validation(job_id: int) -> None:
|
|
992
|
+
"""Start detailed validation and create the detailed validation preprocessed file"""
|
|
993
|
+
# Start validation asynchronously
|
|
994
|
+
_report = _encode_and_publish(str(job_id), GOOGLE_DL_CLINICAL_VALIDATION_TOPIC)
|
|
995
|
+
|
|
996
|
+
|
|
988
997
|
def send_email(to_emails: List[str], subject: str, html_content: str, **kw) -> None:
|
|
989
998
|
"""
|
|
990
999
|
Publish an email-to-send to the emails topic.
|
cidc_api/shared/utils.py
ADDED
|
@@ -1,35 +1,35 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nci_cidc_api_modules
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.16
|
|
4
4
|
Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
|
|
5
5
|
Home-page: https://github.com/NCI-CIDC/cidc-api-gae
|
|
6
6
|
License: MIT license
|
|
7
7
|
Requires-Python: >=3.13
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
License-File: LICENSE
|
|
10
|
-
Requires-Dist: certifi>=2025.
|
|
11
|
-
Requires-Dist: cloud-sql-python-connector[pg8000]>=1.18.
|
|
10
|
+
Requires-Dist: certifi>=2025.10.5
|
|
11
|
+
Requires-Dist: cloud-sql-python-connector[pg8000]>=1.18.5
|
|
12
12
|
Requires-Dist: flask>=3.1.2
|
|
13
13
|
Requires-Dist: flask-migrate>=4.1.0
|
|
14
14
|
Requires-Dist: flask-sqlalchemy>=3.1.1
|
|
15
|
-
Requires-Dist: google-auth
|
|
16
|
-
Requires-Dist: google-api-python-client>=2.
|
|
15
|
+
Requires-Dist: google-auth==2.41.1
|
|
16
|
+
Requires-Dist: google-api-python-client>=2.185.0
|
|
17
17
|
Requires-Dist: google-cloud-bigquery>=3.38.0
|
|
18
|
-
Requires-Dist: google-cloud-pubsub>=2.
|
|
19
|
-
Requires-Dist: google-cloud-secret-manager>=2.
|
|
20
|
-
Requires-Dist: google-cloud-storage>=3.4.
|
|
18
|
+
Requires-Dist: google-cloud-pubsub>=2.32.0
|
|
19
|
+
Requires-Dist: google-cloud-secret-manager>=2.25.0
|
|
20
|
+
Requires-Dist: google-cloud-storage>=3.4.1
|
|
21
21
|
Requires-Dist: jinja2>=3.1.6
|
|
22
|
-
Requires-Dist: marshmallow>=4.0.
|
|
22
|
+
Requires-Dist: marshmallow>=4.0.1
|
|
23
23
|
Requires-Dist: marshmallow-sqlalchemy>=1.4.2
|
|
24
|
-
Requires-Dist: numpy>=2.3.
|
|
24
|
+
Requires-Dist: numpy>=2.3.4
|
|
25
25
|
Requires-Dist: packaging>=25.0
|
|
26
26
|
Requires-Dist: pandas>=2.3.3
|
|
27
|
-
Requires-Dist: pyarrow>=
|
|
28
|
-
Requires-Dist: python-dotenv>=1.
|
|
27
|
+
Requires-Dist: pyarrow>=22.0.0
|
|
28
|
+
Requires-Dist: python-dotenv>=1.2.1
|
|
29
29
|
Requires-Dist: requests>=2.32.5
|
|
30
|
-
Requires-Dist: sqlalchemy>=2.0.
|
|
30
|
+
Requires-Dist: sqlalchemy>=2.0.44
|
|
31
31
|
Requires-Dist: werkzeug>=3.1.3
|
|
32
|
-
Requires-Dist: nci-cidc-schemas==0.28.
|
|
32
|
+
Requires-Dist: nci-cidc-schemas==0.28.8
|
|
33
33
|
Dynamic: description
|
|
34
34
|
Dynamic: description-content-type
|
|
35
35
|
Dynamic: home-page
|
|
@@ -2,23 +2,25 @@ cidc_api/config/__init__.py,sha256=5mX8GAPxUKV84iS-aGOoE-4m68LsOCGCDptXNdlgvj0,1
|
|
|
2
2
|
cidc_api/config/db.py,sha256=eFCkJDeykRIJZ25kePVjRNrZcrcKTc1s2K0yErr46LI,1683
|
|
3
3
|
cidc_api/config/logging.py,sha256=abhVYtn8lfhIt0tyV2WHFgSmp_s2eeJh7kodB6LH4J0,1149
|
|
4
4
|
cidc_api/config/secrets.py,sha256=jRFj7W43pWuPf9DZQLCKF7WPXf5cUv-BAaS3ASqhV_Q,1481
|
|
5
|
-
cidc_api/config/settings.py,sha256=
|
|
5
|
+
cidc_api/config/settings.py,sha256=ttOGvk_6zVMn4dtxIZ2-0w3wF2fpAUVfGpVZbKJ2b6s,4653
|
|
6
6
|
cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
|
|
7
7
|
cidc_api/models/migrations.py,sha256=gp9vtkYbA9FFy2s-7woelAmsvQbJ41LO2_DY-YkFIrQ,11464
|
|
8
|
-
cidc_api/models/models.py,sha256=
|
|
8
|
+
cidc_api/models/models.py,sha256=Id0BjccYd0aj6C-WCGXtYDxdpzK5KLjzub0HLVsnrxk,146517
|
|
9
9
|
cidc_api/models/schemas.py,sha256=6IE2dJoEMcMbi0Vr1V3cYKnPKU0hv9vRKBixOZHe88s,2766
|
|
10
10
|
cidc_api/models/files/__init__.py,sha256=8BMTnUSHzUbz0lBeEQY6NvApxDD3GMWMduoVMos2g4Y,213
|
|
11
11
|
cidc_api/models/files/details.py,sha256=sZkGM7iEV4-J6IDQCdiMV6KBDLbPxCOqUMaU3aY9rX8,65153
|
|
12
12
|
cidc_api/models/files/facets.py,sha256=RZoe0FadRGJw8_-jzJQkc4dIVSuaYlunNrsopb2dYho,33325
|
|
13
13
|
cidc_api/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
14
|
cidc_api/shared/auth.py,sha256=PHqmVGkqDjbmUofytVFwD_9ssgUomESl3fFtFHPwZYQ,9062
|
|
15
|
-
cidc_api/shared/
|
|
16
|
-
cidc_api/shared/
|
|
17
|
-
cidc_api/shared/
|
|
15
|
+
cidc_api/shared/email_layout.html,sha256=pBoTNw3ACHH-ncZFaNvcy5bXMqPwizR78usb0uCYtIc,7670
|
|
16
|
+
cidc_api/shared/emails.py,sha256=8kNFEaSnKpY-GX_iE59QUhSp3c4_uzy3SpHYt2QjuqI,6121
|
|
17
|
+
cidc_api/shared/file_handling.py,sha256=4xkekdrzxWsfmfBtWpylyDMT98ICjQwx1meqqh1FAis,4103
|
|
18
|
+
cidc_api/shared/gcloud_client.py,sha256=ovXGS2ynaBgB_23prj23H10GNN4fectiVF7Hj4LJXQk,37302
|
|
18
19
|
cidc_api/shared/jose.py,sha256=-qzGzEDAlokEp9E7WtBtQkXyyfPWTYXlwYpCqVJWmqM,1830
|
|
19
20
|
cidc_api/shared/rest_utils.py,sha256=RwR30WOUAYCxL7V-i2totEyeriG30GbBDvBcpLXhM9w,6594
|
|
20
|
-
|
|
21
|
-
nci_cidc_api_modules-1.2.
|
|
22
|
-
nci_cidc_api_modules-1.2.
|
|
23
|
-
nci_cidc_api_modules-1.2.
|
|
24
|
-
nci_cidc_api_modules-1.2.
|
|
21
|
+
cidc_api/shared/utils.py,sha256=oDGC8MHxEf7MDuzWynZuE66OfNUnRZE8z7Yn2Q9kYO8,178
|
|
22
|
+
nci_cidc_api_modules-1.2.16.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
|
|
23
|
+
nci_cidc_api_modules-1.2.16.dist-info/METADATA,sha256=0AG4rjFYqfR1q7wd44iGY7RCqqLyCylA4aHGv5zmkOI,39540
|
|
24
|
+
nci_cidc_api_modules-1.2.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
25
|
+
nci_cidc_api_modules-1.2.16.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
|
|
26
|
+
nci_cidc_api_modules-1.2.16.dist-info/RECORD,,
|
|
File without changes
|
{nci_cidc_api_modules-1.2.9.dist-info → nci_cidc_api_modules-1.2.16.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{nci_cidc_api_modules-1.2.9.dist-info → nci_cidc_api_modules-1.2.16.dist-info}/top_level.txt
RENAMED
|
File without changes
|