nci-cidc-api-modules 1.2.13__py3-none-any.whl → 1.2.16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cidc_api/models/models.py CHANGED
@@ -26,6 +26,7 @@ __all__ = [
26
26
  "FileValidationErrors",
27
27
  "IngestionJobs",
28
28
  "JobFileCategories",
29
+ "CategoryDataElements",
29
30
  "ValidationConfigs",
30
31
  "TRIAL_APPENDIX_A",
31
32
  "TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER",
@@ -131,7 +132,6 @@ from ..config.settings import (
131
132
  MAX_PAGINATION_PAGE_SIZE,
132
133
  TESTING,
133
134
  INACTIVE_USER_DAYS,
134
- GOOGLE_CLINICAL_DATA_BUCKET,
135
135
  )
136
136
  from ..shared import emails
137
137
  from ..shared.gcloud_client import (
@@ -145,7 +145,6 @@ from ..shared.gcloud_client import (
145
145
  revoke_intake_access,
146
146
  revoke_lister_access,
147
147
  revoke_bigquery_access,
148
- gcs_xlsx_or_csv_file_to_pandas_dataframe,
149
148
  )
150
149
 
151
150
  os.environ["TZ"] = "UTC"
@@ -3450,30 +3449,9 @@ class IngestionJobs(CommonColumns):
3450
3449
 
3451
3450
  @with_default_session
3452
3451
  def transition_status(self, status: str, session: Session):
3453
- # create required categories after opening job for submission
3454
- if self.status == "DRAFT" and status == "INITIAL SUBMISSION":
3455
- for category in self.derive_required_categories_from_appendix_a():
3456
- JobFileCategories.create(category=category, job_id=self.id, type="required")
3457
3452
  self.status = status
3458
3453
  self.update(session=session)
3459
3454
 
3460
- def derive_required_categories_from_appendix_a(self) -> List:
3461
- appendix_a = PreprocessedFiles.get_files_by_category_and_status(TRIAL_APPENDIX_A, "current", job_id=self.id)[0]
3462
- df = gcs_xlsx_or_csv_file_to_pandas_dataframe(GOOGLE_CLINICAL_DATA_BUCKET, appendix_a.object_url)
3463
- categories = []
3464
- headers_ended = False
3465
- for _index, row in df.iterrows():
3466
- cell = str(row.iloc[0])
3467
- if headers_ended:
3468
- if cell != "nan" and cell not in categories:
3469
- categories.append(cell)
3470
- elif cell.lower() == TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER.lower():
3471
- headers_ended = True
3472
- if "data_dictionary" not in categories:
3473
- # Ensure Data_Dictionary is always a required file category
3474
- categories.append("data_dictionary")
3475
- return categories
3476
-
3477
3455
  @classmethod
3478
3456
  @with_default_session
3479
3457
  def atomic_set_job_as_pending(cls, job_id: int, session: Session) -> Boolean:
@@ -3553,6 +3531,7 @@ class JobFileCategories(CommonColumns):
3553
3531
  category = Column(String)
3554
3532
  job_id = Column(Integer)
3555
3533
  type = Column(Enum("required", "optional", name="type"))
3534
+ is_custom = Column(Boolean, default=False, server_default="false")
3556
3535
 
3557
3536
  @staticmethod
3558
3537
  @with_default_session
@@ -3560,12 +3539,14 @@ class JobFileCategories(CommonColumns):
3560
3539
  category: str,
3561
3540
  job_id: int,
3562
3541
  type: str,
3542
+ is_custom: bool = False,
3563
3543
  session: Session = None,
3564
3544
  ):
3565
3545
  new_category = JobFileCategories(
3566
3546
  category=category,
3567
3547
  job_id=job_id,
3568
3548
  type=type,
3549
+ is_custom=is_custom,
3569
3550
  )
3570
3551
  new_category.insert(session=session)
3571
3552
  return new_category
@@ -3577,6 +3558,28 @@ class JobFileCategories(CommonColumns):
3577
3558
  return [c.category for c in categories]
3578
3559
 
3579
3560
 
3561
+ class CategoryDataElements(CommonColumns):
3562
+ __tablename__ = "category_data_elements"
3563
+ __table_args__ = (
3564
+ ForeignKeyConstraint(
3565
+ ["category_id"],
3566
+ ["job_file_categories.id"],
3567
+ ondelete="CASCADE",
3568
+ ),
3569
+ Index(
3570
+ "idx_elements_category_id" "category_id",
3571
+ "name",
3572
+ unique=True,
3573
+ ),
3574
+ )
3575
+
3576
+ category_id = Column(Integer, nullable=False)
3577
+ name = Column(String, nullable=False)
3578
+ is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
3579
+ element_type = Column(String, nullable=False)
3580
+ cardinality = Column(String, nullable=True)
3581
+
3582
+
3580
3583
  class FileValidationErrors(CommonColumns):
3581
3584
  __tablename__ = "file_validation_errors"
3582
3585
  __table_args__ = (
@@ -0,0 +1,258 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+
4
+ <head>
5
+ <title>Manifest Status Notification</title>
6
+ <meta charset="utf-8">
7
+ <meta name="viewport" content="width=device-width, initial-scale=1">
8
+ <meta http-equiv="Content-Type" content="text/html charset=UTF-8" />
9
+ <meta http-equiv="X-UA-Compatible" content="IE=edge" />
10
+ <style type="text/css">
11
+ /* CLIENT-SPECIFIC STYLES */
12
+ body,
13
+ table,
14
+ td,
15
+ a {
16
+ -webkit-text-size-adjust: 100%;
17
+ -ms-text-size-adjust: 100%;
18
+ }
19
+
20
+ /* Prevent WebKit and Windows mobile changing default text sizes */
21
+ table,
22
+ td {
23
+ mso-table-lspace: 0pt;
24
+ mso-table-rspace: 0pt;
25
+ }
26
+
27
+ /* Remove spacing between tables in Outlook 2007 and up */
28
+ img {
29
+ -ms-interpolation-mode: bicubic;
30
+ }
31
+
32
+ /* Allow smoother rendering of resized image in Internet Explorer */
33
+
34
+ /* RESET STYLES */
35
+ img {
36
+ border: 0;
37
+ height: auto;
38
+ line-height: 100%;
39
+ outline: none;
40
+ text-decoration: none;
41
+ }
42
+
43
+ table {
44
+ border-collapse: collapse !important;
45
+ }
46
+
47
+ body {
48
+ height: 100% !important;
49
+ margin: 0 !important;
50
+ padding: 0 !important;
51
+ width: 100% !important;
52
+ }
53
+
54
+ /* iOS BLUE LINKS */
55
+ a[x-apple-data-detectors] {
56
+ color: inherit !important;
57
+ text-decoration: none !important;
58
+ font-size: inherit !important;
59
+ font-family: inherit !important;
60
+ font-weight: inherit !important;
61
+ line-height: inherit !important;
62
+ }
63
+
64
+ /* MOBILE STYLES */
65
+ @media screen and (max-width: 525px) {
66
+
67
+ /* ALLOWS FOR FLUID TABLES */
68
+ .wrapper {
69
+ width: 100% !important;
70
+ max-width: 100% !important;
71
+ }
72
+
73
+ /* ADJUSTS LAYOUT OF LOGO IMAGE */
74
+ .logo img {
75
+ margin: 0 auto !important;
76
+ }
77
+
78
+ /* USE THESE CLASSES TO HIDE CONTENT ON MOBILE */
79
+ .mobile-hide {
80
+ display: none !important;
81
+ }
82
+
83
+ .img-max {
84
+ max-width: 100% !important;
85
+ width: 100% !important;
86
+ height: auto !important;
87
+ }
88
+
89
+ /* FULL-WIDTH TABLES */
90
+ .responsive-table {
91
+ width: 100% !important;
92
+ }
93
+
94
+ /* UTILITY CLASSES FOR ADJUSTING PADDING ON MOBILE */
95
+ .padding {
96
+ padding: 10px 5% 15px 5% !important;
97
+ }
98
+
99
+ .padding-meta {
100
+ padding: 30px 5% 0px 5% !important;
101
+ text-align: center;
102
+ }
103
+
104
+ .padding-copy {
105
+ padding: 10px 5% 10px 5% !important;
106
+ text-align: center;
107
+ }
108
+
109
+ .no-padding {
110
+ padding: 0 !important;
111
+ }
112
+
113
+ .section-padding {
114
+ padding: 50px 15px 50px 15px !important;
115
+ }
116
+
117
+ /* ADJUST BUTTONS ON MOBILE */
118
+ .mobile-button-container {
119
+ margin: 0 auto;
120
+ width: 100% !important;
121
+ }
122
+
123
+ .mobile-button {
124
+ padding: 15px !important;
125
+ border: 0 !important;
126
+ font-size: 16px !important;
127
+ display: block !important;
128
+ }
129
+
130
+ }
131
+
132
+ /* ANDROID CENTER FIX */
133
+ div[style*="margin: 16px 0;"] {
134
+ margin: 0 !important;
135
+ }
136
+
137
+ a {
138
+ color: white;
139
+ text-decoration: underline;
140
+ }
141
+
142
+ .blue-link {
143
+ color: #1B6B82;
144
+ }
145
+ </style>
146
+ </head>
147
+
148
+ <body style="margin: 0 !important; padding: 0 !important; background-color:#F5F5F5;">
149
+
150
+ <!-- ONE COLUMN SECTION -->
151
+ <table bgcolor="#F5F5F5" border="0" cellpadding="0" cellspacing="0" width="100%">
152
+
153
+ <tr>
154
+ <td align="center" style="padding: 60px 16px 16px; margin-top: -32px;" class="section-padding">
155
+ <!--[if (gte mso 9)|(IE)]>
156
+ <table align="center" border="0" cellspacing="0" cellpadding="0" width="500">
157
+ <tr>
158
+ <td align="center" valign="top" width="500">
159
+ <![endif]-->
160
+ <table bgcolor="#ffffff" border="0" cellpadding="0" cellspacing="0" width="100%" style="max-width: 520px;"
161
+ class="responsive-table">
162
+ <tr>
163
+ <td>
164
+ <table width="100%" border="0" cellspacing="0" cellpadding="0">
165
+ <tr>
166
+ <td>
167
+ <table width="100%" border="0" cellspacing="0" cellpadding="0">
168
+ <tr>
169
+ <td align="center"
170
+ style="padding: 32px 0 0 0; font-size: 18px; line-height: 25px; font-family: Helvetica, Arial, sans-serif; color: #3F414F;"
171
+ class="padding-copy">
172
+ Cancer Immunologic Data Center (CIDC)</td>
173
+ </tr>
174
+ <tr>
175
+ <td align="center"
176
+ style="padding: 50px 32px 0 32px; line-height: 42px; font-family: Helvetica, Arial, sans-serif; color: #666666;">
177
+ <h1 style="margin:0; mso-line-height-rule:exactly; padding: 0 32px 0 32px">
178
+ $HEADING
179
+ </h1>
180
+ </td>
181
+ </tr>
182
+ <tr>
183
+ <td align="center"
184
+ style="padding: 40px 40px 40px 40px; font-size: 16px; line-height: 25px; font-family: Helvetica, Arial, sans-serif; color: #666666;"
185
+ class="padding-copy">
186
+ $CONTENT
187
+ </td>
188
+ </tr>
189
+ </table>
190
+ </td>
191
+ </tr>
192
+ </table>
193
+ </td>
194
+ </tr>
195
+ </table>
196
+ <!--[if (gte mso 9)|(IE)]>
197
+ </td>
198
+ </tr>
199
+ </table>
200
+ <![endif]-->
201
+ </td>
202
+ </tr>
203
+ <tr>
204
+ <td align="center" style="padding: 20px 0px;">
205
+ <!--[if (gte mso 9)|(IE)]>
206
+ <table align="center" border="0" cellspacing="0" cellpadding="0" width="500">
207
+ <tr>
208
+ <td align="center" valign="top" width="500">
209
+ <![endif]-->
210
+ <table bgcolor="#1B6B82" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"
211
+ style="max-width: 520px; " class="responsive-table">
212
+ <tr>
213
+ <td align="center"
214
+ style="padding: 20px 0 0 0; font-size: 18px; line-height: 18px; font-family: Helvetica, Arial, sans-serif; color:#fff; text-align: center">
215
+ Need help?
216
+ </td>
217
+ </tr>
218
+ <tr>
219
+ <td align="center"
220
+ style="padding: 10px 0 20px 0; font-size: 14px; line-height: 18px; font-family: Helvetica, Arial, sans-serif; color:#fff; text-align: center">
221
+ Send us an email at <a href="mailto:ncicidcadmin@mail.nih.gov">ncicidcadmin@mail.nih.gov</a>
222
+ </td>
223
+ </tr>
224
+ </table>
225
+ <!--[if (gte mso 9)|(IE)]>
226
+ </td>
227
+ </tr>
228
+ </table>
229
+ <![endif]-->
230
+ </td>
231
+ </tr>
232
+ <tr>
233
+ <td align="center" style="padding: 20px 0px;">
234
+ <!--[if (gte mso 9)|(IE)]>
235
+ <table align="center" border="0" cellspacing="0" cellpadding="0" width="500">
236
+ <tr>
237
+ <td align="center" valign="top" width="500">
238
+ <![endif]-->
239
+ <table width="100%" border="0" cellspacing="0" cellpadding="0" align="center" style="max-width: 520px; "
240
+ class="responsive-table">
241
+ <tr>
242
+ <td align="center"
243
+ style="padding: 10px 0 20px 0; font-size: 14px; line-height: 18px; font-family: Helvetica, Arial, sans-serif; color:#666666;">
244
+ National Cancer Institute at the National institutes of Health
245
+ </td>
246
+ </tr>
247
+ </table>
248
+ <!--[if (gte mso 9)|(IE)]>
249
+ </td>
250
+ </tr>
251
+ </table>
252
+ <![endif]-->
253
+ </td>
254
+ </tr>
255
+ </table>
256
+ </body>
257
+
258
+ </html>
@@ -1,6 +1,6 @@
1
1
  """Utilities for interacting with the Google Cloud Platform APIs."""
2
2
 
3
- # pylint: disable=logging-fstring-interpolation
3
+ # pylint: disable=logging-fstring-interpolation,too-many-lines
4
4
 
5
5
  import base64
6
6
  import datetime
@@ -37,8 +37,8 @@ from sqlalchemy.orm.session import Session
37
37
  from werkzeug.datastructures import FileStorage
38
38
  from werkzeug.utils import secure_filename
39
39
 
40
- from cidc_api.config.secrets import get_secrets_manager
41
40
  from ..config.logging import get_logger
41
+ from ..config.secrets import get_secrets_manager
42
42
  from ..config.settings import (
43
43
  DEV_USE_GCS,
44
44
  GOOGLE_INTAKE_ROLE,
@@ -62,6 +62,8 @@ from ..config.settings import (
62
62
  DEV_CFUNCTIONS_SERVER,
63
63
  INACTIVE_USER_DAYS,
64
64
  )
65
+ from ..shared.utils import strip_whitespaces
66
+
65
67
 
66
68
  os.environ["TZ"] = "UTC"
67
69
  logger = get_logger(__name__)
@@ -427,9 +429,9 @@ def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
427
429
 
428
430
  # TODO: specify sheet in xlsx file and/or accept tsv and xls files
429
431
  if blob_name[-3:] == "csv":
430
- return pd.read_csv(temp_file)
432
+ return strip_whitespaces(pd.read_csv(temp_file))
431
433
  elif blob_name[-4:] == "xlsx":
432
- return pd.read_excel(temp_file)
434
+ return strip_whitespaces(pd.read_excel(temp_file))
433
435
  else:
434
436
  raise Exception("Can only read csv or xlsx files")
435
437
 
@@ -0,0 +1,8 @@
1
+ def strip_whitespaces(df):
2
+ def stripper(x):
3
+ if x and isinstance(x, str):
4
+ return x.strip()
5
+ else:
6
+ return x
7
+
8
+ return df.map(stripper)
@@ -1,35 +1,35 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nci_cidc_api_modules
3
- Version: 1.2.13
3
+ Version: 1.2.16
4
4
  Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
5
5
  Home-page: https://github.com/NCI-CIDC/cidc-api-gae
6
6
  License: MIT license
7
7
  Requires-Python: >=3.13
8
8
  Description-Content-Type: text/markdown
9
9
  License-File: LICENSE
10
- Requires-Dist: certifi>=2025.8.3
11
- Requires-Dist: cloud-sql-python-connector[pg8000]>=1.18.4
10
+ Requires-Dist: certifi>=2025.10.5
11
+ Requires-Dist: cloud-sql-python-connector[pg8000]>=1.18.5
12
12
  Requires-Dist: flask>=3.1.2
13
13
  Requires-Dist: flask-migrate>=4.1.0
14
14
  Requires-Dist: flask-sqlalchemy>=3.1.1
15
- Requires-Dist: google-auth>=2.41.0
16
- Requires-Dist: google-api-python-client>=2.183.0
15
+ Requires-Dist: google-auth==2.41.1
16
+ Requires-Dist: google-api-python-client>=2.185.0
17
17
  Requires-Dist: google-cloud-bigquery>=3.38.0
18
- Requires-Dist: google-cloud-pubsub>=2.31.1
19
- Requires-Dist: google-cloud-secret-manager>=2.24.0
20
- Requires-Dist: google-cloud-storage>=3.4.0
18
+ Requires-Dist: google-cloud-pubsub>=2.32.0
19
+ Requires-Dist: google-cloud-secret-manager>=2.25.0
20
+ Requires-Dist: google-cloud-storage>=3.4.1
21
21
  Requires-Dist: jinja2>=3.1.6
22
- Requires-Dist: marshmallow>=4.0.0
22
+ Requires-Dist: marshmallow>=4.0.1
23
23
  Requires-Dist: marshmallow-sqlalchemy>=1.4.2
24
- Requires-Dist: numpy>=2.3.2
24
+ Requires-Dist: numpy>=2.3.4
25
25
  Requires-Dist: packaging>=25.0
26
26
  Requires-Dist: pandas>=2.3.3
27
- Requires-Dist: pyarrow>=21.0.0
28
- Requires-Dist: python-dotenv>=1.1.1
27
+ Requires-Dist: pyarrow>=22.0.0
28
+ Requires-Dist: python-dotenv>=1.2.1
29
29
  Requires-Dist: requests>=2.32.5
30
- Requires-Dist: sqlalchemy>=2.0.43
30
+ Requires-Dist: sqlalchemy>=2.0.44
31
31
  Requires-Dist: werkzeug>=3.1.3
32
- Requires-Dist: nci-cidc-schemas==0.28.7
32
+ Requires-Dist: nci-cidc-schemas==0.28.8
33
33
  Dynamic: description
34
34
  Dynamic: description-content-type
35
35
  Dynamic: home-page
@@ -5,20 +5,22 @@ cidc_api/config/secrets.py,sha256=jRFj7W43pWuPf9DZQLCKF7WPXf5cUv-BAaS3ASqhV_Q,14
5
5
  cidc_api/config/settings.py,sha256=ttOGvk_6zVMn4dtxIZ2-0w3wF2fpAUVfGpVZbKJ2b6s,4653
6
6
  cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
7
7
  cidc_api/models/migrations.py,sha256=gp9vtkYbA9FFy2s-7woelAmsvQbJ41LO2_DY-YkFIrQ,11464
8
- cidc_api/models/models.py,sha256=b7HWj8uOD_xdv7p3WkrhRODauLCVGvSLQsSbJGUmr9Q,146956
8
+ cidc_api/models/models.py,sha256=Id0BjccYd0aj6C-WCGXtYDxdpzK5KLjzub0HLVsnrxk,146517
9
9
  cidc_api/models/schemas.py,sha256=6IE2dJoEMcMbi0Vr1V3cYKnPKU0hv9vRKBixOZHe88s,2766
10
10
  cidc_api/models/files/__init__.py,sha256=8BMTnUSHzUbz0lBeEQY6NvApxDD3GMWMduoVMos2g4Y,213
11
11
  cidc_api/models/files/details.py,sha256=sZkGM7iEV4-J6IDQCdiMV6KBDLbPxCOqUMaU3aY9rX8,65153
12
12
  cidc_api/models/files/facets.py,sha256=RZoe0FadRGJw8_-jzJQkc4dIVSuaYlunNrsopb2dYho,33325
13
13
  cidc_api/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
14
  cidc_api/shared/auth.py,sha256=PHqmVGkqDjbmUofytVFwD_9ssgUomESl3fFtFHPwZYQ,9062
15
+ cidc_api/shared/email_layout.html,sha256=pBoTNw3ACHH-ncZFaNvcy5bXMqPwizR78usb0uCYtIc,7670
15
16
  cidc_api/shared/emails.py,sha256=8kNFEaSnKpY-GX_iE59QUhSp3c4_uzy3SpHYt2QjuqI,6121
16
17
  cidc_api/shared/file_handling.py,sha256=4xkekdrzxWsfmfBtWpylyDMT98ICjQwx1meqqh1FAis,4103
17
- cidc_api/shared/gcloud_client.py,sha256=BaoYv-lWWLwdZvyRjCWCyAyZMXtqlT8IPJpw9XDODCY,37210
18
+ cidc_api/shared/gcloud_client.py,sha256=ovXGS2ynaBgB_23prj23H10GNN4fectiVF7Hj4LJXQk,37302
18
19
  cidc_api/shared/jose.py,sha256=-qzGzEDAlokEp9E7WtBtQkXyyfPWTYXlwYpCqVJWmqM,1830
19
20
  cidc_api/shared/rest_utils.py,sha256=RwR30WOUAYCxL7V-i2totEyeriG30GbBDvBcpLXhM9w,6594
20
- nci_cidc_api_modules-1.2.13.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
21
- nci_cidc_api_modules-1.2.13.dist-info/METADATA,sha256=gRTTJ2KEYN1B9Ac0rim5nAPVRsQtw5NWr0E9KJhSIc0,39539
22
- nci_cidc_api_modules-1.2.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
23
- nci_cidc_api_modules-1.2.13.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
24
- nci_cidc_api_modules-1.2.13.dist-info/RECORD,,
21
+ cidc_api/shared/utils.py,sha256=oDGC8MHxEf7MDuzWynZuE66OfNUnRZE8z7Yn2Q9kYO8,178
22
+ nci_cidc_api_modules-1.2.16.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
23
+ nci_cidc_api_modules-1.2.16.dist-info/METADATA,sha256=0AG4rjFYqfR1q7wd44iGY7RCqqLyCylA4aHGv5zmkOI,39540
24
+ nci_cidc_api_modules-1.2.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
25
+ nci_cidc_api_modules-1.2.16.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
26
+ nci_cidc_api_modules-1.2.16.dist-info/RECORD,,