cdapython 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cdapython/__init__.py +7 -0
- cdapython/application_utilities.py +54 -0
- cdapython/config/__init__.py +0 -0
- cdapython/config/logger_default_config.yml +19 -0
- cdapython/discover.py +1412 -0
- cdapython/get_data.py +1174 -0
- cdapython/logging_wrappers.py +186 -0
- cdapython/summarize.py +1801 -0
- cdapython/validation.py +622 -0
- cdapython-2.0.0.dist-info/LICENSE +201 -0
- cdapython-2.0.0.dist-info/METADATA +21 -0
- cdapython-2.0.0.dist-info/RECORD +13 -0
- cdapython-2.0.0.dist-info/WHEEL +4 -0
cdapython/discover.py
ADDED
|
@@ -0,0 +1,1412 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
import cda_client
|
|
7
|
+
import cda_client.api.columns.columns_endpoint_columns_get
|
|
8
|
+
import cda_client.api.release_metadata.release_metadata_endpoint_release_metadata_get
|
|
9
|
+
import cda_client.api.column_values.column_values_endpoint_column_values_column_post
|
|
10
|
+
|
|
11
|
+
from cdapython.application_utilities import get_api_url
|
|
12
|
+
from cda_client.errors import UnexpectedStatus
|
|
13
|
+
from cdapython.logging_wrappers import get_logger
|
|
14
|
+
|
|
15
|
+
#############################################################################################################################
|
|
16
|
+
#############################################################################################################################
|
|
17
|
+
# Nomenclature notes:
|
|
18
|
+
#
|
|
19
|
+
# * try to standardize all potential user-facing synonyms for basic database data structures
|
|
20
|
+
# (field, entity, endpoint, cell, value, term, etc.) to 'table', 'column', 'row' and 'value'.
|
|
21
|
+
#############################################################################################################################
|
|
22
|
+
#############################################################################################################################
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
#############################################################################################################################
|
|
26
|
+
#
|
|
27
|
+
# cda_functions(): Return a list of user-facing cdapython functions useful for both scripting and interactive data sessions.
|
|
28
|
+
#
|
|
29
|
+
#############################################################################################################################
|
|
30
|
+
|
|
31
|
+
def cda_functions():
|
|
32
|
+
"""
|
|
33
|
+
Returns a list of cdapython functions useful for both scripting and interactive data sessions.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
# There doesn't seem to be a need to check invocation context before just printing to standard output,
|
|
37
|
+
# here, as this function is primarily designed to be consulted in interactive sessions. If anyone ever
|
|
38
|
+
# finds themselves checking this function's output programmatically, for some reason, and is annoyed
|
|
39
|
+
# by the print statement, we will entertain pull requests to be more sensitive about our assumptions.
|
|
40
|
+
|
|
41
|
+
print( '\nYou can get complete usage details for each function by calling "help( function_name )".', end='\n\n' )
|
|
42
|
+
|
|
43
|
+
return [
|
|
44
|
+
'tables',
|
|
45
|
+
'columns',
|
|
46
|
+
'column_values',
|
|
47
|
+
'summarize_subjects',
|
|
48
|
+
'summarize_files',
|
|
49
|
+
'get_subject_data',
|
|
50
|
+
'get_file_data',
|
|
51
|
+
'intersect_subject_results',
|
|
52
|
+
'intersect_file_results',
|
|
53
|
+
'expand_subject_results',
|
|
54
|
+
'expand_file_results',
|
|
55
|
+
'get_valid_log_levels',
|
|
56
|
+
'get_log_level',
|
|
57
|
+
'set_log_level',
|
|
58
|
+
'enable_console_logging',
|
|
59
|
+
'disable_console_logging',
|
|
60
|
+
'enable_file_logging',
|
|
61
|
+
'disable_file_logging',
|
|
62
|
+
'get_api_url',
|
|
63
|
+
'set_api_url',
|
|
64
|
+
'cda_functions'
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
#############################################################################################################################
|
|
68
|
+
#
|
|
69
|
+
# END cda_functions()
|
|
70
|
+
#
|
|
71
|
+
#############################################################################################################################
|
|
72
|
+
|
|
73
|
+
#############################################################################################################################
|
|
74
|
+
#
|
|
75
|
+
# tables(): Return a list of all searchable CDA data tables.
|
|
76
|
+
#
|
|
77
|
+
#############################################################################################################################
|
|
78
|
+
|
|
79
|
+
def tables():
|
|
80
|
+
"""
|
|
81
|
+
Get a list of all searchable CDA data tables.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
list of strings: names of searchable CDA tables.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
log = get_logger()
|
|
88
|
+
|
|
89
|
+
# Call columns(), extract unique values from the `table` column of the
|
|
90
|
+
# resulting DataFrame, and return those values to the user as a list.
|
|
91
|
+
|
|
92
|
+
log.debug( 'Calling columns()' )
|
|
93
|
+
|
|
94
|
+
columns_result_df = columns( return_data_as='dataframe' )
|
|
95
|
+
|
|
96
|
+
if columns_result_df is None:
|
|
97
|
+
log.error( 'Something went fatally wrong with columns( return_data_as="dataframe" ): got a null DataFrame back.' )
|
|
98
|
+
return
|
|
99
|
+
else:
|
|
100
|
+
return sorted( columns_result_df['table'].unique() )
|
|
101
|
+
|
|
102
|
+
#############################################################################################################################
|
|
103
|
+
#
|
|
104
|
+
# END tables()
|
|
105
|
+
#
|
|
106
|
+
#############################################################################################################################
|
|
107
|
+
|
|
108
|
+
#############################################################################################################################
|
|
109
|
+
#
|
|
110
|
+
# columns(): Provide user with structured metadata describing searchable CDA columns:
|
|
111
|
+
#
|
|
112
|
+
# (containing) table
|
|
113
|
+
# column (name)
|
|
114
|
+
# data_type (stored in column)
|
|
115
|
+
# (values are) nullable(?)
|
|
116
|
+
# (prose) description
|
|
117
|
+
#
|
|
118
|
+
#############################################################################################################################
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def columns(
|
|
122
|
+
*,
|
|
123
|
+
return_data_as='',
|
|
124
|
+
output_file='',
|
|
125
|
+
sort_by='',
|
|
126
|
+
**filter_arguments
|
|
127
|
+
):
|
|
128
|
+
"""
|
|
129
|
+
Get structured metadata describing searchable CDA columns.
|
|
130
|
+
|
|
131
|
+
Arguments:
|
|
132
|
+
return_data_as ( string; optional: 'dataframe' or 'list' or 'tsv' ):
|
|
133
|
+
Specify how columns() should return results: as a pandas DataFrame,
|
|
134
|
+
a Python list, or as output written to a TSV file named by the user.
|
|
135
|
+
If this argument is omitted, columns() will default to returning
|
|
136
|
+
results as a DataFrame.
|
|
137
|
+
|
|
138
|
+
output_file( string; optional ):
|
|
139
|
+
If return_data_as='tsv' is specified, output_file should contain a
|
|
140
|
+
resolvable path to a file into which columns() will write
|
|
141
|
+
tab-delimited results.
|
|
142
|
+
|
|
143
|
+
sort_by( string or list of strings; optional:
|
|
144
|
+
any combination of 'table', 'column', 'data_type',
|
|
145
|
+
and/or 'nullable'):
|
|
146
|
+
Specify the column metadata field(s) on which to sort result data.
|
|
147
|
+
Results will be sorted first by the first named field; groups of
|
|
148
|
+
records sharing the same value in the first field will then be
|
|
149
|
+
sub-sorted by the second field, and so on.
|
|
150
|
+
|
|
151
|
+
Any field with a suffix of ':desc' appended to it will be sorted
|
|
152
|
+
in reverse order; adding ':asc' will ensure ascending sort order.
|
|
153
|
+
Example: sort_by=[ 'table', 'nullable:desc', 'column:asc' ]
|
|
154
|
+
|
|
155
|
+
Filter arguments:
|
|
156
|
+
table ( string or list of strings; optional ):
|
|
157
|
+
Restrict returned data to columns from tables whose names match any
|
|
158
|
+
of the given strings. A wildcard (asterisk) at either end (or both
|
|
159
|
+
ends) of each string will allow partial matches. Case will be
|
|
160
|
+
ignored.
|
|
161
|
+
|
|
162
|
+
column ( string or list of strings; optional ):
|
|
163
|
+
Restrict returned data to columns whose name matches any of the
|
|
164
|
+
given strings. A wildcard (asterisk) at either end (or both ends)
|
|
165
|
+
of each string will allow partial matches. Case will be ignored.
|
|
166
|
+
|
|
167
|
+
data_type ( string or list of strings; optional ):
|
|
168
|
+
Restrict returned data to columns whose data type matches any of
|
|
169
|
+
the given strings. A wildcard (asterisk) at either end (or both
|
|
170
|
+
ends) of each string will allow partial matches. Case will be
|
|
171
|
+
ignored.
|
|
172
|
+
|
|
173
|
+
nullable ( boolean; optional ):
|
|
174
|
+
If set to True, restrict returned data to columns whose values are
|
|
175
|
+
allowed to be empty; if False, return data only for columns
|
|
176
|
+
requiring nonempty values.
|
|
177
|
+
|
|
178
|
+
description ( string or list of strings; optional ):
|
|
179
|
+
Restrict returned data to columns whose `description` field matches
|
|
180
|
+
any of the given strings. Wildcards will be automatically applied
|
|
181
|
+
if not provided, to support straightforward keyword searching of this
|
|
182
|
+
field without requiring too much extra punctuation. Case will be
|
|
183
|
+
ignored.
|
|
184
|
+
|
|
185
|
+
exclude_table ( string or list of strings; optional ):
|
|
186
|
+
Restrict returned data to columns from tables whose names do _not_
|
|
187
|
+
match any of the given strings. A wildcard (asterisk) at either end
|
|
188
|
+
(or both ends) of each string will allow partial matches. Case will
|
|
189
|
+
be ignored.
|
|
190
|
+
|
|
191
|
+
Returns:
|
|
192
|
+
pandas.DataFrame where each row is a metadata record describing one
|
|
193
|
+
searchable CDA column and is comprised of the following fields:
|
|
194
|
+
|
|
195
|
+
`table` (string: name of the CDA table containing this column)
|
|
196
|
+
`column` (string: name of this column)
|
|
197
|
+
`data_type` (string: data type of this column)
|
|
198
|
+
`nullable` (boolean: if True, this column can contain null values)`
|
|
199
|
+
`description` (string: prose description of this column)
|
|
200
|
+
|
|
201
|
+
OR list of column names
|
|
202
|
+
|
|
203
|
+
OR returns nothing, but writes results to a user-specified TSV file
|
|
204
|
+
"""
|
|
205
|
+
|
|
206
|
+
log = get_logger()
|
|
207
|
+
|
|
208
|
+
#############################################################################################################################
|
|
209
|
+
# Process return-type directives `return_data_as` and `output_file`.
|
|
210
|
+
|
|
211
|
+
if not isinstance(return_data_as, str):
|
|
212
|
+
|
|
213
|
+
log.error( f"Unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
|
|
214
|
+
return
|
|
215
|
+
|
|
216
|
+
# Let's not be picky if someone wants to give us return_data_as='DataFrame' or return_data_as='TSV'
|
|
217
|
+
|
|
218
|
+
return_data_as = return_data_as.lower()
|
|
219
|
+
|
|
220
|
+
# We can't do much validation on filenames. If `output_file` isn't
|
|
221
|
+
# a locally writeable path, it'll fail when we try to open it for
|
|
222
|
+
# writing. Strip trailing whitespace from both ends and wrap the
|
|
223
|
+
# file-access operation (later, below) in a try{} block.
|
|
224
|
+
|
|
225
|
+
if not isinstance( output_file, str ):
|
|
226
|
+
|
|
227
|
+
log.error( f"The `output_file` parameter, if not omitted, should be a string containing a path to the desired output file. You supplied '{output_file}', which is not a string, let alone a valid path." )
|
|
228
|
+
return
|
|
229
|
+
|
|
230
|
+
output_file = output_file.strip()
|
|
231
|
+
|
|
232
|
+
allowed_return_types = { '', 'dataframe', 'tsv', 'list' }
|
|
233
|
+
|
|
234
|
+
if return_data_as not in allowed_return_types:
|
|
235
|
+
|
|
236
|
+
# Complain if we receive an unexpected `return_data_as` value.
|
|
237
|
+
log.error( f"Unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
|
|
238
|
+
return
|
|
239
|
+
|
|
240
|
+
elif return_data_as == 'tsv' and output_file == '':
|
|
241
|
+
|
|
242
|
+
# If the user asks for a TSV, they also have to give us a path for that TSV. If they didn't, complain.
|
|
243
|
+
log.error( "Return type 'tsv' was requested, but 'output_file' was not specified. Please specify output_file='some/path/string/to/write/your/tsv/to/your_tsv_output_file.tsv'." )
|
|
244
|
+
return
|
|
245
|
+
|
|
246
|
+
elif return_data_as != 'tsv' and output_file != '':
|
|
247
|
+
|
|
248
|
+
# If the user put something in the `output_file` parameter but didn't specify `result_data_as='tsv'`,
|
|
249
|
+
# they most likely want their data saved to a file (so ignoring the parameter misconfiguration
|
|
250
|
+
# isn't safe), but ultimately we can't be sure what they meant (so taking an action isn't safe),
|
|
251
|
+
# so we complain and ask them to clarify.
|
|
252
|
+
|
|
253
|
+
log.error( f"'output_file' was specified, but this is only meaningful if 'return_data_as' is set to 'tsv'. You requested return_data_as='{return_data_as}'." )
|
|
254
|
+
log.error( '(Note that if you don\'t specify any value for \'return_data_as\', it defaults to \'dataframe\'.).' )
|
|
255
|
+
return
|
|
256
|
+
|
|
257
|
+
#############################################################################################################################
|
|
258
|
+
# Process `sort_by` directives.
|
|
259
|
+
|
|
260
|
+
if isinstance(sort_by, str):
|
|
261
|
+
|
|
262
|
+
# Make `sort_by` a list, if it's not, so we don't have to split the way we
|
|
263
|
+
# process this information into parallel distinct branches.
|
|
264
|
+
|
|
265
|
+
if sort_by == '':
|
|
266
|
+
sort_by = []
|
|
267
|
+
else:
|
|
268
|
+
sort_by = [ sort_by ]
|
|
269
|
+
|
|
270
|
+
elif not isinstance(sort_by, list):
|
|
271
|
+
|
|
272
|
+
# Also detect any disallowed incoming data types and complain if we find any.
|
|
273
|
+
|
|
274
|
+
log.error( f"columns(): ERROR: 'sort_by' must be a string or a list of strings; you used '{sort_by}', which is neither." )
|
|
275
|
+
return
|
|
276
|
+
|
|
277
|
+
# Enumerate all allowed values that a user can specify using the `sort_by` parameter. ( 'X:asc' will be aliased immediately to just 'X'. )
|
|
278
|
+
|
|
279
|
+
allowed_sort_by_arguments = [
|
|
280
|
+
'table',
|
|
281
|
+
'table:desc',
|
|
282
|
+
'column',
|
|
283
|
+
'column:desc',
|
|
284
|
+
'data_type',
|
|
285
|
+
'data_type:desc',
|
|
286
|
+
'nullable',
|
|
287
|
+
'nullable:desc',
|
|
288
|
+
]
|
|
289
|
+
|
|
290
|
+
# Build two lists to pass to `DataFrame.sort_values()` to direct the sorting of our result data
|
|
291
|
+
# according to user specifications:
|
|
292
|
+
#
|
|
293
|
+
# * `by_list` will contain exact field names on which to sort, in order of precedence.
|
|
294
|
+
#
|
|
295
|
+
# * `ascending_list` will be a series of boolean values, one for each field name in
|
|
296
|
+
# `by_list`, where each `False` value indicates that the corresponding field
|
|
297
|
+
# in `by_list` is to be sorted in reverse.
|
|
298
|
+
|
|
299
|
+
by_list = list()
|
|
300
|
+
|
|
301
|
+
ascending_list = list()
|
|
302
|
+
|
|
303
|
+
seen_so_far = dict()
|
|
304
|
+
|
|
305
|
+
for field_code in sort_by:
|
|
306
|
+
|
|
307
|
+
if not isinstance(field_code, str):
|
|
308
|
+
|
|
309
|
+
# Complain if we receive any unexpected data types instead of string directives.
|
|
310
|
+
|
|
311
|
+
log.critical( f"columns(): ERROR: 'sort_by' must be a string or a list of strings; you used '{sort_by}', which is neither." )
|
|
312
|
+
return
|
|
313
|
+
|
|
314
|
+
# Let's not care about case.
|
|
315
|
+
|
|
316
|
+
field_code = field_code.lower()
|
|
317
|
+
|
|
318
|
+
# ':asc' is redundant. Remove it (politely).
|
|
319
|
+
|
|
320
|
+
field_code = re.sub( r':asc$', r'', field_code )
|
|
321
|
+
|
|
322
|
+
if field_code not in allowed_sort_by_arguments:
|
|
323
|
+
|
|
324
|
+
# Complain if we receive any unexpected sort_by directives.
|
|
325
|
+
|
|
326
|
+
log.critical( f"columns(): ERROR: '{field_code}' is not a valid directive for the 'sort_by' parameter. Please use one of [ '"
|
|
327
|
+
+ "', '".join(allowed_sort_by_arguments)
|
|
328
|
+
+ "' ] instead." )
|
|
329
|
+
return
|
|
330
|
+
|
|
331
|
+
code_basename = field_code
|
|
332
|
+
|
|
333
|
+
if re.search( r':desc$', field_code ) is not None:
|
|
334
|
+
code_basename = re.sub( r':desc$', '', field_code )
|
|
335
|
+
|
|
336
|
+
if code_basename not in seen_so_far:
|
|
337
|
+
|
|
338
|
+
seen_so_far[code_basename] = field_code
|
|
339
|
+
|
|
340
|
+
else:
|
|
341
|
+
|
|
342
|
+
# Complain if we receive multiple sort_by directives for the same output column.
|
|
343
|
+
|
|
344
|
+
log.critical( f"columns(): ERROR: Multiple sort_by directives received for the same output column, including '{seen_so_far[code_basename]}' and '{field_code}': please specify only one directive per output column." )
|
|
345
|
+
return
|
|
346
|
+
|
|
347
|
+
by_list.append( code_basename )
|
|
348
|
+
|
|
349
|
+
if re.search( r':desc$', field_code ) is not None:
|
|
350
|
+
|
|
351
|
+
# Reverse the sort on this column.
|
|
352
|
+
ascending_list.append( False )
|
|
353
|
+
|
|
354
|
+
else:
|
|
355
|
+
|
|
356
|
+
# Sort this column normally.
|
|
357
|
+
ascending_list.append( True )
|
|
358
|
+
|
|
359
|
+
# Report details of the final parsed sort logic.
|
|
360
|
+
|
|
361
|
+
sort_dataframe = pd.DataFrame( { 'sort_by': by_list, 'ascending?': ascending_list } )
|
|
362
|
+
|
|
363
|
+
if not sort_dataframe.empty:
|
|
364
|
+
log.debug( f"Processed sort directives: {sort_dataframe}" )
|
|
365
|
+
else:
|
|
366
|
+
log.debug( f"Processed sort directives: <default>" )
|
|
367
|
+
|
|
368
|
+
#############################################################################################################################
|
|
369
|
+
# Process user-supplied filter directives.
|
|
370
|
+
|
|
371
|
+
# Enumerate all allowed filters that a user can specify with named parameters.
|
|
372
|
+
|
|
373
|
+
allowed_filter_arguments = [
|
|
374
|
+
'table',
|
|
375
|
+
'column',
|
|
376
|
+
'data_type',
|
|
377
|
+
'nullable',
|
|
378
|
+
'description',
|
|
379
|
+
'exclude_table'
|
|
380
|
+
]
|
|
381
|
+
|
|
382
|
+
# Validate filter argument content.
|
|
383
|
+
|
|
384
|
+
for filter_argument_name in filter_arguments:
|
|
385
|
+
|
|
386
|
+
if filter_argument_name not in allowed_filter_arguments:
|
|
387
|
+
|
|
388
|
+
# Complain if we receive any unexpected filter arguments.
|
|
389
|
+
|
|
390
|
+
log.critical( f"columns(): ERROR: Received unexpected argument {filter_argument_name}; aborting." )
|
|
391
|
+
return
|
|
392
|
+
|
|
393
|
+
elif filter_argument_name == 'nullable':
|
|
394
|
+
|
|
395
|
+
if not isinstance( filter_arguments[filter_argument_name], bool ):
|
|
396
|
+
|
|
397
|
+
# Complain if we got a parameter value of the wrong data type.
|
|
398
|
+
|
|
399
|
+
log.critical( f"columns(): ERROR: 'nullable' must be a Boolean value (True or False); you used '{filter_arguments[filter_argument_name]}', which is not." )
|
|
400
|
+
return
|
|
401
|
+
|
|
402
|
+
elif not ( isinstance( filter_arguments[filter_argument_name], str ) or isinstance( filter_arguments[filter_argument_name], list ) ):
|
|
403
|
+
|
|
404
|
+
# Complain if we got a parameter value of the wrong data type.
|
|
405
|
+
|
|
406
|
+
log.critical( f"columns(): ERROR: '{filter_argument_name}' must be a string or a list of strings; you used '{filter_arguments[filter_argument_name]}', which is neither." )
|
|
407
|
+
return
|
|
408
|
+
|
|
409
|
+
elif isinstance( filter_arguments[filter_argument_name], list ):
|
|
410
|
+
|
|
411
|
+
for pattern in filter_arguments[filter_argument_name]:
|
|
412
|
+
|
|
413
|
+
if not isinstance( pattern, str ):
|
|
414
|
+
|
|
415
|
+
# Complain if we receive any unexpected data types inside a filter list (i.e. anything but strings).
|
|
416
|
+
|
|
417
|
+
log.critical( f"columns(): ERROR: '{filter_argument_name}' must be a string or a list of strings; you used '{filter_arguments[filter_argument_name]}', which is neither." )
|
|
418
|
+
return
|
|
419
|
+
|
|
420
|
+
# Report details of fully processed user directives prior to querying.
|
|
421
|
+
|
|
422
|
+
status_report = f"return_data_as='{return_data_as}', output_file='{output_file}'"
|
|
423
|
+
|
|
424
|
+
status_report = status_report + f", sort_by={sort_by}"
|
|
425
|
+
|
|
426
|
+
for filter_argument_name in filter_arguments:
|
|
427
|
+
|
|
428
|
+
if isinstance( filter_arguments[filter_argument_name], str ):
|
|
429
|
+
status_report = status_report + f", {filter_argument_name}='{filter_arguments[filter_argument_name]}'"
|
|
430
|
+
else:
|
|
431
|
+
status_report = status_report + f", {filter_argument_name}={filter_arguments[filter_argument_name]}"
|
|
432
|
+
|
|
433
|
+
log.debug( f"Processed user directives: '{status_report}'" )
|
|
434
|
+
|
|
435
|
+
#############################################################################################################################
|
|
436
|
+
# Fetch data from the API.
|
|
437
|
+
|
|
438
|
+
query_api_instance = cda_client.Client( base_url=get_api_url(), raise_on_unexpected_status=True )
|
|
439
|
+
|
|
440
|
+
# Ask the columns endpoint for information. (It has no parameters.)
|
|
441
|
+
|
|
442
|
+
try:
|
|
443
|
+
columns_response_data_object = cda_client.api.columns.columns_endpoint_columns_get.sync( client=query_api_instance )
|
|
444
|
+
except UnexpectedStatus as error:
|
|
445
|
+
log.error( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
|
|
446
|
+
return
|
|
447
|
+
except Exception as error:
|
|
448
|
+
log.error( f"{type(error)}: {error}" )
|
|
449
|
+
return
|
|
450
|
+
|
|
451
|
+
#############################################################################################################################
|
|
452
|
+
# Postprocess API result data.
|
|
453
|
+
|
|
454
|
+
# columns_response_data_object['result'] is an array of dicts, with
|
|
455
|
+
# each dict containing a few named fields of metadata describing one column.
|
|
456
|
+
#
|
|
457
|
+
# Example:
|
|
458
|
+
#
|
|
459
|
+
# "result": [
|
|
460
|
+
# {
|
|
461
|
+
# "table": "file",
|
|
462
|
+
# "column": "file_id",
|
|
463
|
+
# "data_type": "text",
|
|
464
|
+
# "nullable": false,
|
|
465
|
+
# "description": "A unique identifier for this file minted by CDA. May change release-to-release. Contains no semantically reliable content with one exception: in the case of a DICOM series from IDC, this field will contain the crdc_series_uuid assigned by IDC to that DICOM series. Note that this crdc_series_uuid may change from one IDC release version to the next, according to IDC's data processing and identification rules."
|
|
466
|
+
# },
|
|
467
|
+
# ...
|
|
468
|
+
# ]
|
|
469
|
+
|
|
470
|
+
# Make a DataFrame from this array of dicts using DataFrame.from_records(), and explicitly specify the
|
|
471
|
+
# column ordering for the resulting DataFrame using the `columns=[]` parameter.
|
|
472
|
+
|
|
473
|
+
result_dataframe = pd.DataFrame.from_records( columns_response_data_object.to_dict()['result'], columns=[ 'table', 'column', 'data_type', 'nullable', 'description' ] )
|
|
474
|
+
|
|
475
|
+
# Remove `table`_data_source_count and *_alias columns from output.
|
|
476
|
+
#
|
|
477
|
+
# TO DO: Maybe put this list somewhere easier to find.
|
|
478
|
+
|
|
479
|
+
banned_column_name_patterns = {
|
|
480
|
+
r'_crdc_id$',
|
|
481
|
+
r'^[^_]+_data_source_count$',
|
|
482
|
+
r'^[^_]+_data_at_[^_]+$',
|
|
483
|
+
r'_alias$'
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
banned_columns = set()
|
|
487
|
+
|
|
488
|
+
for column_name in result_dataframe['column'].unique():
|
|
489
|
+
for banned_pattern in banned_column_name_patterns:
|
|
490
|
+
if re.search( banned_pattern, column_name ) is not None:
|
|
491
|
+
banned_columns.add( column_name )
|
|
492
|
+
|
|
493
|
+
for banned_column in banned_columns:
|
|
494
|
+
result_dataframe = result_dataframe.loc[ result_dataframe['column'] != banned_column ]
|
|
495
|
+
|
|
496
|
+
log.debug( 'Created result DataFrame' )
|
|
497
|
+
|
|
498
|
+
#############################################################################################################################
|
|
499
|
+
# Execute sorting directives, if we got any; otherwise perform the default sort on the result DataFrame.
|
|
500
|
+
|
|
501
|
+
if len( sort_by ) == 0:
|
|
502
|
+
|
|
503
|
+
# By default, we sort column records by column name, gathered into groups by table,
|
|
504
|
+
# to facilitate predictable output patterns. For easy access, we'd also like each
|
|
505
|
+
# table's ID column to show up first in its table's group.
|
|
506
|
+
#
|
|
507
|
+
# Temporarily prepend a '.' to `table`_id column names, so they float to the top of each
|
|
508
|
+
# table's list of columns when we sort.
|
|
509
|
+
|
|
510
|
+
result_dataframe = result_dataframe.replace( to_replace=r'^([^_]+_id)$', value=r'.\1', regex=True )
|
|
511
|
+
|
|
512
|
+
# Sort all column records, first on table and then on column name.
|
|
513
|
+
|
|
514
|
+
result_dataframe = result_dataframe.sort_values( by=['table', 'column'], ascending=[True, True] )
|
|
515
|
+
|
|
516
|
+
# Remove the '.' characters we temporarily prepended to `table`_id column names
|
|
517
|
+
# to force the sorting algorithm to place all such columns first within each
|
|
518
|
+
# table's group of column records.
|
|
519
|
+
|
|
520
|
+
result_dataframe = result_dataframe.replace( to_replace=r'^\.(.*)$', value=r'\1', regex=True )
|
|
521
|
+
|
|
522
|
+
else:
|
|
523
|
+
|
|
524
|
+
# Sort all column records according to the user-specified directives we've processed.
|
|
525
|
+
|
|
526
|
+
result_dataframe = result_dataframe.sort_values( by=by_list, ascending=ascending_list )
|
|
527
|
+
|
|
528
|
+
log.debug( 'Applied sort_by directives' )
|
|
529
|
+
|
|
530
|
+
#############################################################################################################################
|
|
531
|
+
# Iterate through whatever filters the user passed us and
|
|
532
|
+
# apply them to the result data before sending it back.
|
|
533
|
+
#
|
|
534
|
+
# The value of filter_name, here, will be one of
|
|
535
|
+
# 'table', 'column', 'data_type', 'nullable',
|
|
536
|
+
# 'description' or 'exclude_table'.
|
|
537
|
+
|
|
538
|
+
for filter_name in filter_arguments:
|
|
539
|
+
|
|
540
|
+
# Grab the filters the user sent us.
|
|
541
|
+
|
|
542
|
+
# Default behavior: all result values must be exact matches to at least one
|
|
543
|
+
# filter (ignoring case). To match end-to-end, we use a ^ to represent
|
|
544
|
+
# the beginning of each value and a $ to indicate the end. If the user
|
|
545
|
+
# specifies wildcards on one or both ends of a filter, we'll remove one or both
|
|
546
|
+
# restrictions as instructed for that filter.
|
|
547
|
+
#
|
|
548
|
+
# EXCEPTION ONE: the `nullable` filter argument will be a single Boolean
|
|
549
|
+
# value, and we handle it separately.
|
|
550
|
+
#
|
|
551
|
+
# EXCEPTION TWO: For the `description` field, we've modified the
|
|
552
|
+
# argument processing (because in this case users are searching an
|
|
553
|
+
# abstract-sized block of text, not a short string representing
|
|
554
|
+
# a name or a concept) so that filters will always be processed
|
|
555
|
+
# as if they have wildcards on both ends. (Search will still be
|
|
556
|
+
# case-insensitive at all times.)
|
|
557
|
+
#
|
|
558
|
+
# EXCEPTION THREE: In the case of `exclude_table`, all result values must
|
|
559
|
+
# _not_ match _any_ of the specified filters.
|
|
560
|
+
|
|
561
|
+
if filter_name == 'nullable':
|
|
562
|
+
|
|
563
|
+
return_if_nullable = filter_arguments[filter_name]
|
|
564
|
+
|
|
565
|
+
if not isinstance( return_if_nullable, bool ):
|
|
566
|
+
log.critical( f"columns(): ERROR: Please specify either nullable=True or nullable=False, not (what you sent) nullable='{return_if_nullable}'." )
|
|
567
|
+
return
|
|
568
|
+
|
|
569
|
+
result_dataframe = result_dataframe.loc[ result_dataframe['nullable'] == return_if_nullable ]
|
|
570
|
+
|
|
571
|
+
else:
|
|
572
|
+
|
|
573
|
+
filters = filter_arguments[filter_name]
|
|
574
|
+
|
|
575
|
+
filter_patterns = list()
|
|
576
|
+
|
|
577
|
+
# If the filter list wasn't a list at all but a (nonempty) string, we just have
|
|
578
|
+
# one filter. Listify it (so we don't have to care downstream about how many there are).
|
|
579
|
+
# Otherwise, just start with the list they sent us.
|
|
580
|
+
|
|
581
|
+
if isinstance( filters, str ) and filters != '':
|
|
582
|
+
filter_patterns = [filters]
|
|
583
|
+
elif isinstance( filters, list ):
|
|
584
|
+
filter_patterns = filters
|
|
585
|
+
|
|
586
|
+
# (If neither of the above conditions was met, `filter_patterns` will remain an
|
|
587
|
+
# empty list, and the rest of this filter-processing section will (by design) have no effect.
|
|
588
|
+
|
|
589
|
+
# Parse filter_name to establish which columns() field is being targeted
|
|
590
|
+
# for filtration and adjust default filtration logic as necessary according to the result.
|
|
591
|
+
|
|
592
|
+
target_field = filter_name
|
|
593
|
+
|
|
594
|
+
if filter_name == 'description':
|
|
595
|
+
|
|
596
|
+
# Never match end-to-end for query strings applied to description text (see discussion above).
|
|
597
|
+
|
|
598
|
+
updated_pattern_list = list()
|
|
599
|
+
|
|
600
|
+
for original_filter_pattern in filter_patterns:
|
|
601
|
+
|
|
602
|
+
updated_filter_pattern = f"*{original_filter_pattern}*"
|
|
603
|
+
updated_pattern_list.append( updated_filter_pattern )
|
|
604
|
+
|
|
605
|
+
filter_patterns = updated_pattern_list
|
|
606
|
+
|
|
607
|
+
elif filter_name == 'exclude_table':
|
|
608
|
+
|
|
609
|
+
target_field = 'table'
|
|
610
|
+
|
|
611
|
+
match_pattern_string = ''
|
|
612
|
+
|
|
613
|
+
for filter_pattern in filter_patterns:
|
|
614
|
+
|
|
615
|
+
# Process wildcard characters.
|
|
616
|
+
|
|
617
|
+
if re.search( r'^\*', filter_pattern ) is not None:
|
|
618
|
+
|
|
619
|
+
# Any prefix will do, now.
|
|
620
|
+
#
|
|
621
|
+
# Strip leading '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
|
|
622
|
+
|
|
623
|
+
filter_pattern = re.sub( r'^\*+', r'', filter_pattern )
|
|
624
|
+
|
|
625
|
+
else:
|
|
626
|
+
|
|
627
|
+
# No wildcard at the beginning of `filter_pattern` --> require all successful matches to _begin_ with `filter_pattern` by prepending a ^ character to `filter_pattern`:
|
|
628
|
+
#
|
|
629
|
+
# ...I know this looks weird, but it's just tacking a '^' character onto the beginning of `filter_pattern`.
|
|
630
|
+
|
|
631
|
+
filter_pattern = re.sub( r'^', r'^', filter_pattern )
|
|
632
|
+
|
|
633
|
+
if re.search( r'\*$', filter_pattern ) is not None:
|
|
634
|
+
|
|
635
|
+
# Any suffix will do, now.
|
|
636
|
+
#
|
|
637
|
+
# Strip trailing '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
|
|
638
|
+
|
|
639
|
+
filter_pattern = re.sub( r'\*+$', r'', filter_pattern )
|
|
640
|
+
|
|
641
|
+
else:
|
|
642
|
+
|
|
643
|
+
# No wildcard at the end of `filter_pattern` --> require all successful matches to _end_ with `filter_pattern` by appending a '$' character to `filter_pattern`:
|
|
644
|
+
#
|
|
645
|
+
# ...I know this looks weird, but it's just tacking a '$' character onto the end of `filter_pattern`.
|
|
646
|
+
|
|
647
|
+
filter_pattern = re.sub( r'$', r'$', filter_pattern )
|
|
648
|
+
|
|
649
|
+
# Build the overall match pattern as we go, one (processed) `filter_pattern` at a time.
|
|
650
|
+
|
|
651
|
+
match_pattern_string = match_pattern_string + filter_pattern + '|'
|
|
652
|
+
|
|
653
|
+
# Strip final trailing |.
|
|
654
|
+
|
|
655
|
+
match_pattern_string = re.sub( r'\|$', r'', match_pattern_string )
|
|
656
|
+
|
|
657
|
+
if filter_name == 'exclude_table':
|
|
658
|
+
|
|
659
|
+
# Retain all rows where the value of `target_field` (in this case, the value of `table`) does _not_ match any of the given filter patterns.
|
|
660
|
+
|
|
661
|
+
result_dataframe = result_dataframe.loc[ ~( result_dataframe[target_field].str.contains( match_pattern_string, case=False ) ) ]
|
|
662
|
+
|
|
663
|
+
else:
|
|
664
|
+
|
|
665
|
+
# Retain all rows where the value of `target_field` matches any of the given filter patterns.
|
|
666
|
+
|
|
667
|
+
result_dataframe = result_dataframe.loc[ result_dataframe[target_field].str.contains( match_pattern_string, case=False ) ]
|
|
668
|
+
|
|
669
|
+
log.debug( 'Applied value-filtration directives' )
|
|
670
|
+
|
|
671
|
+
#############################################################################################################################
|
|
672
|
+
# Send the results back to the user.
|
|
673
|
+
|
|
674
|
+
# Reindex DataFrame rows to match their final sort order.
|
|
675
|
+
|
|
676
|
+
result_dataframe = result_dataframe.reset_index( drop=True )
|
|
677
|
+
|
|
678
|
+
if return_data_as == '':
|
|
679
|
+
|
|
680
|
+
# Right now, the default is the same as if the user had
|
|
681
|
+
# specified return_data_as='dataframe'.
|
|
682
|
+
|
|
683
|
+
# The following, for the dubiously useful record, is a somewhat worse alternative default thing to do.
|
|
684
|
+
#
|
|
685
|
+
# print( result_dataframe.to_string( index=False, justify='right', max_rows=25, max_colwidth=50 ), file=sys.stdout )
|
|
686
|
+
|
|
687
|
+
log.debug( 'Returning results in default form (pandas.DataFrame)' )
|
|
688
|
+
|
|
689
|
+
return result_dataframe
|
|
690
|
+
|
|
691
|
+
elif return_data_as == 'dataframe':
|
|
692
|
+
|
|
693
|
+
# Give the user back the results DataFrame.
|
|
694
|
+
|
|
695
|
+
log.debug( 'Returning results as pandas.DataFrame' )
|
|
696
|
+
|
|
697
|
+
return result_dataframe
|
|
698
|
+
|
|
699
|
+
elif return_data_as == 'list':
|
|
700
|
+
|
|
701
|
+
# Give the user back a list of column names.
|
|
702
|
+
|
|
703
|
+
log.debug( 'Returning results as list of column names' )
|
|
704
|
+
|
|
705
|
+
return result_dataframe['column'].to_list()
|
|
706
|
+
|
|
707
|
+
else:
|
|
708
|
+
|
|
709
|
+
# Write the results DataFrame to a user-specified TSV file.
|
|
710
|
+
|
|
711
|
+
log.debug( f"Printing results to TSV file '{output_file}'" )
|
|
712
|
+
|
|
713
|
+
try:
|
|
714
|
+
|
|
715
|
+
result_dataframe.to_csv( output_file, sep='\t', index=False )
|
|
716
|
+
return
|
|
717
|
+
|
|
718
|
+
except Exception as error:
|
|
719
|
+
raise RuntimeError( f"Couldn't write to requested output file '{output_file}': got error of type '{type(error)}', with error message '{error}'." )
|
|
720
|
+
|
|
721
|
+
#############################################################################################################################
|
|
722
|
+
#
|
|
723
|
+
# END columns()
|
|
724
|
+
#
|
|
725
|
+
#############################################################################################################################
|
|
726
|
+
|
|
727
|
+
#############################################################################################################################
|
|
728
|
+
#
|
|
729
|
+
# column_values( column=`column` ): Show all distinct values present in `column`, along with a count of occurrences for each value.
|
|
730
|
+
#
|
|
731
|
+
#############################################################################################################################
|
|
732
|
+
|
|
733
|
+
def column_values(
|
|
734
|
+
column='',
|
|
735
|
+
*,
|
|
736
|
+
return_data_as='',
|
|
737
|
+
output_file='',
|
|
738
|
+
sort_by='',
|
|
739
|
+
filters=None,
|
|
740
|
+
data_source='',
|
|
741
|
+
force=False
|
|
742
|
+
):
|
|
743
|
+
"""
|
|
744
|
+
Show all distinct values present in `column`, along with a count
|
|
745
|
+
of occurrences for each value.
|
|
746
|
+
|
|
747
|
+
Arguments:
|
|
748
|
+
column ( string; required ):
|
|
749
|
+
The column to fetch values from.
|
|
750
|
+
|
|
751
|
+
return_data_as ( string; optional: 'dataframe' or 'list' or 'tsv' ):
|
|
752
|
+
Specify how column_values() should return results: as a pandas
|
|
753
|
+
DataFrame, a Python list, or as output written to a TSV file named
|
|
754
|
+
by the user. If this argument is omitted, column_values() will default
|
|
755
|
+
to returning results as a DataFrame.
|
|
756
|
+
|
|
757
|
+
output_file( string; optional ):
|
|
758
|
+
If return_data_as='tsv' is specified, output_file should contain a
|
|
759
|
+
resolvable path to a file into which column_values() will write
|
|
760
|
+
tab-delimited results.
|
|
761
|
+
|
|
762
|
+
sort_by( string; optional:
|
|
763
|
+
'count' ( default for return_data_as='dataframe' and
|
|
764
|
+
return_data_as='tsv' ) or 'value' ( default for
|
|
765
|
+
return_data_as='list' ) or 'count:desc' or 'value:desc'
|
|
766
|
+
or 'count:asc' or 'value:asc' ):
|
|
767
|
+
Specify the primary column to sort when preparing result data: on
|
|
768
|
+
values, or on counts of values.
|
|
769
|
+
|
|
770
|
+
A column name with a suffix of ':desc' appended to it will be
|
|
771
|
+
sorted in reverse order; adding ':asc' will ensure ascending sort
|
|
772
|
+
order. Example: sort_by='value:desc'
|
|
773
|
+
|
|
774
|
+
Secondary sort order is automatic: if the results are to be
|
|
775
|
+
primarily sorted by count, then the automatic behavior will be to
|
|
776
|
+
also (alphabetically) sort by value within each group of values
|
|
777
|
+
that all share the same count. If results are primarily sorted by
|
|
778
|
+
value, then there is no secondary sort -- each value is unique by
|
|
779
|
+
design, so results don't contain groups with the same value but
|
|
780
|
+
different counts, so there's nothing to arrange once the primary
|
|
781
|
+
sort has been applied.
|
|
782
|
+
|
|
783
|
+
filters ( string or list of strings; optional ):
|
|
784
|
+
Restrict returned values to those matching any of the given strings.
|
|
785
|
+
A wildcard (asterisk) at either end (or both ends) of each string
|
|
786
|
+
will allow partial matches. Case will be ignored. Specify an empty
|
|
787
|
+
filter string '' to match and count missing (null) values.
|
|
788
|
+
|
|
789
|
+
data_source ( string; optional ):
|
|
790
|
+
Restrict returned values to the given upstream data source. Current
|
|
791
|
+
valid values are 'GDC', 'PDC', 'IDC', 'CDS' and 'ICDC'.
|
|
792
|
+
Defaults to '' (no filter).
|
|
793
|
+
|
|
794
|
+
force( boolean; optional ):
|
|
795
|
+
Force execution of high-overhead queries on columns (like IDs)
|
|
796
|
+
flagged as having large numbers of values. Defaults to False,
|
|
797
|
+
in which case attempts to retrieve values for flagged columns
|
|
798
|
+
will result in a warning.
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
Returns:
|
|
802
|
+
pandas.DataFrame OR list OR returns nothing, but writes retrieved
|
|
803
|
+
data to a user-specified TSV file
|
|
804
|
+
"""
|
|
805
|
+
|
|
806
|
+
log = get_logger()
|
|
807
|
+
|
|
808
|
+
#############################################################################################################################
|
|
809
|
+
# Check for our one required parameter: column.
|
|
810
|
+
|
|
811
|
+
if ( not isinstance( column, str ) ) or column == '':
|
|
812
|
+
log.critical( 'column_values(): ERROR: parameter \'column\' cannot be omitted. Please specify a column from which to fetch a list of distinct values.')
|
|
813
|
+
return
|
|
814
|
+
|
|
815
|
+
# Let's not care about case, and if there's whitespace in our column name, remove it before it does any damage.
|
|
816
|
+
column = re.sub( r'\s+', r'', column ).lower()
|
|
817
|
+
|
|
818
|
+
# See if columns() agrees that the requested column exists. Note: cdapython.columns() and
|
|
819
|
+
# the API's /columns endpoint give different sets of columns, by design. Here we want our output
|
|
820
|
+
# to match the former, because we postprocess some of the columns offered by the API instead of
|
|
821
|
+
# exposing them directly.
|
|
822
|
+
|
|
823
|
+
if len( columns( column=column, return_data_as='list' ) ) == 0:
|
|
824
|
+
log.critical( f"column_values(): ERROR: parameter 'column' must be a searchable CDA column name. You supplied '{column}', which is not." )
|
|
825
|
+
return
|
|
826
|
+
|
|
827
|
+
#############################################################################################################################
|
|
828
|
+
# Check the data_source parameter.
|
|
829
|
+
|
|
830
|
+
if not isinstance( data_source, str ):
|
|
831
|
+
log.error( f"The 'data_source' parameter must be a string (e.g. 'GDC'); you specified '{data_source}', which is not." )
|
|
832
|
+
return
|
|
833
|
+
|
|
834
|
+
# Let's not care about case, and remove any whitespace before it can do any damage.
|
|
835
|
+
data_source = re.sub( r'\s+', r'', data_source ).upper()
|
|
836
|
+
|
|
837
|
+
# TEMPORARY: enumerate valid `data_source` values and warn the user if they supplied something else.
|
|
838
|
+
#
|
|
839
|
+
# This should be replaced ASAP with a fetch from the /release_metadata endpoint.
|
|
840
|
+
|
|
841
|
+
allowed_data_source_values = {
|
|
842
|
+
'GDC',
|
|
843
|
+
'PDC',
|
|
844
|
+
'IDC',
|
|
845
|
+
'CDS',
|
|
846
|
+
'ICDC'
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
if data_source != '' and data_source not in allowed_data_source_values:
|
|
850
|
+
log.error( f"The 'data_source' parameter must be one of [ 'GDC', 'PDC', 'IDC', 'CDS', 'ICDC' ]. You supplied '{data_source}', which is not." )
|
|
851
|
+
return
|
|
852
|
+
|
|
853
|
+
#############################################################################################################################
|
|
854
|
+
# Check in advance for columns flagged as high-overhead.
|
|
855
|
+
|
|
856
|
+
expensive_columns = {
|
|
857
|
+
'file_id',
|
|
858
|
+
'description',
|
|
859
|
+
'drs_uri',
|
|
860
|
+
'file_name',
|
|
861
|
+
'size',
|
|
862
|
+
'case_id',
|
|
863
|
+
'hgnc_id',
|
|
864
|
+
'transcript_id',
|
|
865
|
+
'aliquot_barcode_normal',
|
|
866
|
+
'aliquot_barcode_tumor',
|
|
867
|
+
'case_barcode',
|
|
868
|
+
'dbsnp_rs',
|
|
869
|
+
'entrez_gene_id',
|
|
870
|
+
'gene',
|
|
871
|
+
'hugo_symbol',
|
|
872
|
+
'matched_norm_aliquot_uuid',
|
|
873
|
+
'normal_submitter_uuid',
|
|
874
|
+
'reference_allele',
|
|
875
|
+
'sample_barcode_normal',
|
|
876
|
+
'sample_barcode_tumor',
|
|
877
|
+
'tumor_aliquot_uuid',
|
|
878
|
+
'tumor_seq_allele1',
|
|
879
|
+
'tumor_seq_allele2',
|
|
880
|
+
'tumor_submitter_uuid',
|
|
881
|
+
'subject_id'
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
# Warn the user if an override hasn't been requested.
|
|
885
|
+
|
|
886
|
+
if not force and column in expensive_columns:
|
|
887
|
+
log.warning( f"'{column}' has a very large number of values; retrieval is blocked by default. To perform this query, use column_values( ..., 'force=True' )." )
|
|
888
|
+
return
|
|
889
|
+
|
|
890
|
+
#############################################################################################################################
|
|
891
|
+
# Listify `filters`, if it's a string, so we can process it in a uniform way later on.
|
|
892
|
+
|
|
893
|
+
if filters is None:
|
|
894
|
+
filters = list()
|
|
895
|
+
elif isinstance( filters, str ):
|
|
896
|
+
filters = [ filters ]
|
|
897
|
+
|
|
898
|
+
#############################################################################################################################
|
|
899
|
+
# Process return_data_as and output_file directives.
|
|
900
|
+
|
|
901
|
+
allowed_return_types = {
|
|
902
|
+
'',
|
|
903
|
+
'dataframe',
|
|
904
|
+
'tsv',
|
|
905
|
+
'list'
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
if not isinstance( return_data_as, str ):
|
|
909
|
+
log.critical( f"column_values(): ERROR: unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
|
|
910
|
+
return
|
|
911
|
+
|
|
912
|
+
# Let's not be picky if someone wants to give us return_data_as='DataFrame' or return_data_as='TSV'
|
|
913
|
+
return_data_as = return_data_as.lower()
|
|
914
|
+
|
|
915
|
+
# We can't do much validation on filenames. If `output_file` isn't
|
|
916
|
+
# a locally writeable path, it'll fail when we try to open it for
|
|
917
|
+
# writing. Strip trailing whitespace from both ends and wrap the
|
|
918
|
+
# file-access operation (later, below) in a try{} block.
|
|
919
|
+
|
|
920
|
+
if not isinstance( output_file, str ):
|
|
921
|
+
log.critical( f"column_values(): ERROR: the `output_file` parameter, if not omitted, should be a string containing a path to the desired output file. You supplied '{output_file}', which is not a string, let alone a valid path." )
|
|
922
|
+
return
|
|
923
|
+
|
|
924
|
+
output_file = output_file.strip()
|
|
925
|
+
|
|
926
|
+
if return_data_as not in allowed_return_types:
|
|
927
|
+
|
|
928
|
+
log.critical( f"column_values(): ERROR: unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
|
|
929
|
+
return
|
|
930
|
+
|
|
931
|
+
elif return_data_as == 'tsv' and output_file == '':
|
|
932
|
+
|
|
933
|
+
log.critical( 'column_values(): ERROR: return type \'tsv\' requested, but \'output_file\' not specified. Please specify output_file=\'some/path/string/to/write/your/tsv/to\'.')
|
|
934
|
+
return
|
|
935
|
+
|
|
936
|
+
elif return_data_as != 'tsv' and output_file != '':
|
|
937
|
+
|
|
938
|
+
# If the user put something in the `output_file` parameter but didn't specify `result_data_as='tsv'`,
|
|
939
|
+
# they most likely want their data saved to a file (so ignoring the parameter misconfiguration
|
|
940
|
+
# isn't safe), but ultimately we can't be sure what they meant (so taking an action isn't safe),
|
|
941
|
+
# so we complain and ask them to clarify.
|
|
942
|
+
|
|
943
|
+
log.error( f"'output_file' was specified, but this is only meaningful if 'return_data_as' is set to 'tsv'. You requested return_data_as='{return_data_as}'.\n(Note that if you don't specify any value for 'return_data_as', it defaults to 'dataframe'.)." )
|
|
944
|
+
return
|
|
945
|
+
|
|
946
|
+
#############################################################################################################################
|
|
947
|
+
# Process sorting directives.
|
|
948
|
+
|
|
949
|
+
# Enumerate all allowed values that a user can specify using the `sort_by` parameter. ( 'X:asc' will be aliased immediately to just 'X'. )
|
|
950
|
+
|
|
951
|
+
allowed_sort_by_options = {
|
|
952
|
+
'list': {
|
|
953
|
+
'',
|
|
954
|
+
'value',
|
|
955
|
+
'value:desc'
|
|
956
|
+
},
|
|
957
|
+
'dataframe_or_tsv': {
|
|
958
|
+
'',
|
|
959
|
+
'count',
|
|
960
|
+
'count:desc',
|
|
961
|
+
'value',
|
|
962
|
+
'value:desc'
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
if not isinstance( sort_by, str ):
|
|
967
|
+
|
|
968
|
+
# Complain if we receive any unexpected data types instead of string directives.
|
|
969
|
+
|
|
970
|
+
log.critical( f"column_values(): ERROR: 'sort_by' must be a string; you used '{sort_by}', which is not." )
|
|
971
|
+
return
|
|
972
|
+
|
|
973
|
+
# Let's not care about case. Also, ':asc' is redundant: remove it (politely).
|
|
974
|
+
sort_by = re.sub( r':asc$', r'', sort_by ).lower()
|
|
975
|
+
|
|
976
|
+
if return_data_as == 'list':
|
|
977
|
+
|
|
978
|
+
# Restrict sorting options for lists.
|
|
979
|
+
|
|
980
|
+
if sort_by == '':
|
|
981
|
+
sort_by = 'value'
|
|
982
|
+
elif sort_by not in allowed_sort_by_options['list']:
|
|
983
|
+
log.critical( f"column_values(): ERROR: return_data_as='list' can only be processed with sort_by='value' or sort_by='value:desc' (or omitting sort_by altogether). Please modify unsupported sort_by directive '{sort_by}' and try again." )
|
|
984
|
+
return
|
|
985
|
+
|
|
986
|
+
else:
|
|
987
|
+
|
|
988
|
+
# For TSV output files and DataFrames, we support more user-configurable options (defaulting to sort_by='count:desc'):
|
|
989
|
+
|
|
990
|
+
if sort_by == '':
|
|
991
|
+
sort_by = 'count:desc'
|
|
992
|
+
elif sort_by not in allowed_sort_by_options['dataframe_or_tsv']:
|
|
993
|
+
log.critical( f"column_values(): ERROR: unrecognized sort_by '{sort_by}'. Please use one of 'count', 'value', 'count:desc', 'value:desc', 'count:asc' or 'value:asc' (or omit the sort_by parameter altogether)." )
|
|
994
|
+
return
|
|
995
|
+
|
|
996
|
+
#############################################################################################################################
|
|
997
|
+
# Report details of the final parsed parameters.
|
|
998
|
+
|
|
999
|
+
parameter_dict = {
|
|
1000
|
+
'column': column,
|
|
1001
|
+
'return_data_as': return_data_as,
|
|
1002
|
+
'output_file': output_file,
|
|
1003
|
+
'sort_by': sort_by,
|
|
1004
|
+
'filters': filters,
|
|
1005
|
+
'data_source': data_source,
|
|
1006
|
+
'force': force,
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
log.debug( f"Processed all parameter directives. Calling API to fetch data for\n{json.dumps( parameter_dict, indent=4 )}\n" )
|
|
1010
|
+
|
|
1011
|
+
#############################################################################################################################
|
|
1012
|
+
# Fetch data from the API.
|
|
1013
|
+
|
|
1014
|
+
query_api_instance = cda_client.Client( base_url=get_api_url(), raise_on_unexpected_status=True )
|
|
1015
|
+
|
|
1016
|
+
starting_offset = 0
|
|
1017
|
+
records_per_page = 500000
|
|
1018
|
+
|
|
1019
|
+
try:
|
|
1020
|
+
paged_response_data_object = (
|
|
1021
|
+
cda_client.api.column_values.column_values_endpoint_column_values_column_post.sync(
|
|
1022
|
+
client=query_api_instance,
|
|
1023
|
+
column=column,
|
|
1024
|
+
data_source=data_source,
|
|
1025
|
+
limit=records_per_page,
|
|
1026
|
+
offset=starting_offset
|
|
1027
|
+
)
|
|
1028
|
+
)
|
|
1029
|
+
except UnexpectedStatus as error:
|
|
1030
|
+
log.error( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
|
|
1031
|
+
return
|
|
1032
|
+
except Exception as error:
|
|
1033
|
+
log.error( f"{type(error)}: {error}" )
|
|
1034
|
+
return
|
|
1035
|
+
|
|
1036
|
+
log.debug( f"Sending query to API:\n{json.dumps( { 'columnname': column, 'system': data_source, 'count': True, 'total_count': True, 'limit': records_per_page, 'offset': starting_offset }, indent=4 )}\n" )
|
|
1037
|
+
|
|
1038
|
+
# Report some metadata about the results we got back.
|
|
1039
|
+
|
|
1040
|
+
log.debug( f"Number of result rows: {paged_response_data_object.total_row_count}" )
|
|
1041
|
+
|
|
1042
|
+
log.debug( f"Query SQL: '{paged_response_data_object.query_sql}'" )
|
|
1043
|
+
|
|
1044
|
+
# Make a Pandas DataFrame out of the first batch of results.
|
|
1045
|
+
#
|
|
1046
|
+
# The API returns responses in JSON format: convert that JSON into a DataFrame
|
|
1047
|
+
# using pandas' json_normalize() function. Example JSON response:
|
|
1048
|
+
#
|
|
1049
|
+
# {
|
|
1050
|
+
# "result": [
|
|
1051
|
+
# {
|
|
1052
|
+
# "sex": "female",
|
|
1053
|
+
# "value_count": 31215
|
|
1054
|
+
# },
|
|
1055
|
+
# {
|
|
1056
|
+
# "sex": "male",
|
|
1057
|
+
# "value_count": 28571
|
|
1058
|
+
# },
|
|
1059
|
+
# {
|
|
1060
|
+
# "sex": null,
|
|
1061
|
+
# "value_count": 64240
|
|
1062
|
+
# }
|
|
1063
|
+
# ],
|
|
1064
|
+
# "query_sql": "SELECT row_to_json(column_json) AS row_to_json_1 FROM (SELECT observation.sex AS sex, count(*) AS value_count FROM observation WHERE observation.data_at_gdc IS true GROUP BY observation.sex ORDER BY observation.sex) AS column_json",
|
|
1065
|
+
# "total_row_count": 3,
|
|
1066
|
+
# "next_url": null
|
|
1067
|
+
# }
|
|
1068
|
+
|
|
1069
|
+
log.debug( f"Page one results ( NOTE: pattern filters have not yet been applied ):\n{json.dumps( paged_response_data_object.to_dict(), indent=4 )}\n" )
|
|
1070
|
+
|
|
1071
|
+
result_dataframe = pd.json_normalize( paged_response_data_object.to_dict()['result'] )
|
|
1072
|
+
|
|
1073
|
+
# The data we've fetched so far might be just the first page (if the total number
|
|
1074
|
+
# of results is greater than `records_per_page`).
|
|
1075
|
+
#
|
|
1076
|
+
# Get the rest of the result pages, if there are any, and add each page's data
|
|
1077
|
+
# onto the end of our results DataFrame.
|
|
1078
|
+
|
|
1079
|
+
incremented_offset = starting_offset + records_per_page
|
|
1080
|
+
|
|
1081
|
+
more_than_one_result_page = False
|
|
1082
|
+
|
|
1083
|
+
if paged_response_data_object.next_url is not None:
|
|
1084
|
+
log.debug( 'Fetching remaining results in pages...' )
|
|
1085
|
+
more_than_one_result_page = True
|
|
1086
|
+
|
|
1087
|
+
while paged_response_data_object.next_url is not None and len( paged_response_data_object.next_url ) > 0:
|
|
1088
|
+
|
|
1089
|
+
log.debug( f" ...fetching {paged_response_data_object.next_url}..." )
|
|
1090
|
+
|
|
1091
|
+
try:
|
|
1092
|
+
paged_response_data_object = (
|
|
1093
|
+
cda_client.api.column_values.column_values_endpoint_column_values_column_post.sync(
|
|
1094
|
+
client=query_api_instance,
|
|
1095
|
+
column=column,
|
|
1096
|
+
data_source=data_source,
|
|
1097
|
+
limit=records_per_page,
|
|
1098
|
+
offset=incremented_offset
|
|
1099
|
+
)
|
|
1100
|
+
)
|
|
1101
|
+
except UnexpectedStatus as error:
|
|
1102
|
+
log.error( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
|
|
1103
|
+
return
|
|
1104
|
+
except Exception as error:
|
|
1105
|
+
log.error( f"{type(error)}: {error}" )
|
|
1106
|
+
return
|
|
1107
|
+
|
|
1108
|
+
next_result_batch = pd.json_normalize( paged_response_data_object.to_dict()['result'] )
|
|
1109
|
+
|
|
1110
|
+
if not result_dataframe.empty and not next_result_batch.empty:
|
|
1111
|
+
|
|
1112
|
+
# Silence a future deprecation warning about pd.concat and empty DataFrame columns.
|
|
1113
|
+
|
|
1114
|
+
next_result_batch = next_result_batch.astype( result_dataframe.dtypes )
|
|
1115
|
+
|
|
1116
|
+
result_dataframe = pd.concat( [result_dataframe, next_result_batch] )
|
|
1117
|
+
|
|
1118
|
+
incremented_offset = incremented_offset + records_per_page
|
|
1119
|
+
|
|
1120
|
+
if more_than_one_result_page:
|
|
1121
|
+
log.debug( '...done.' )
|
|
1122
|
+
|
|
1123
|
+
#############################################################################################################################
|
|
1124
|
+
# Postprocess API result data, if there is any.
|
|
1125
|
+
|
|
1126
|
+
if len( result_dataframe ) == 0:
|
|
1127
|
+
return result_dataframe
|
|
1128
|
+
|
|
1129
|
+
log.debug( 'Postprocessing results' )
|
|
1130
|
+
|
|
1131
|
+
log.debug( 'Casting counts to integers and fixing symmetry for returned column labels...' )
|
|
1132
|
+
|
|
1133
|
+
# Sanity check on expected result column 'value_count':
|
|
1134
|
+
|
|
1135
|
+
if 'value_count' not in result_dataframe.columns:
|
|
1136
|
+
log.error( 'Expected column \'value_count\' not present in API response.' )
|
|
1137
|
+
return
|
|
1138
|
+
|
|
1139
|
+
# Term-count values come in as floats. Make them not that.
|
|
1140
|
+
|
|
1141
|
+
result_dataframe['value_count'] = result_dataframe['value_count'].astype( int )
|
|
1142
|
+
|
|
1143
|
+
log.debug( 'Handling missing values...' )
|
|
1144
|
+
|
|
1145
|
+
# CDA has no float values. If the API gives us some, cast them to integers.
|
|
1146
|
+
|
|
1147
|
+
if result_dataframe[column].dtype == 'float64':
|
|
1148
|
+
|
|
1149
|
+
# Columns of type `float64` can contain NaN (missing) values, which cannot (for some reason)
|
|
1150
|
+
# be stored in Pandas Series objects (i.e., DataFrame columns) of type `int` or `int64`.
|
|
1151
|
+
# Pandas workaround: use extension type 'Int64' (note initial capital), which supports the
|
|
1152
|
+
# storage of missing values. These will print as '<NA>'.
|
|
1153
|
+
|
|
1154
|
+
result_dataframe[column] = result_dataframe[column].round().astype( 'Int64' )
|
|
1155
|
+
|
|
1156
|
+
elif result_dataframe[column].dtype == 'object':
|
|
1157
|
+
|
|
1158
|
+
# String data comes through as a column with dtype 'object', based on something involving
|
|
1159
|
+
# the variability inherent in string lengths.
|
|
1160
|
+
#
|
|
1161
|
+
# See https://stackoverflow.com/questions/33957720/how-to-convert-column-with-dtype-as-object-to-string-in-pandas-dataframe
|
|
1162
|
+
|
|
1163
|
+
# Replace term values that are None (== null) with empty strings.
|
|
1164
|
+
|
|
1165
|
+
result_dataframe = result_dataframe.fillna( '' )
|
|
1166
|
+
|
|
1167
|
+
elif result_dataframe[column].dtype == 'bool':
|
|
1168
|
+
|
|
1169
|
+
result_dataframe = result_dataframe.fillna( '' )
|
|
1170
|
+
|
|
1171
|
+
else:
|
|
1172
|
+
|
|
1173
|
+
# This isn't anticipated. Yell if we get something unexpected.
|
|
1174
|
+
|
|
1175
|
+
log.critical( f"column_values(): ERROR: Unexpected data type `{result_dataframe[column].dtype}` received; aborting. Please report this event to the CDA development team." )
|
|
1176
|
+
return
|
|
1177
|
+
|
|
1178
|
+
#############################################################################################################################
|
|
1179
|
+
# Filter returned values according to user specifications.
|
|
1180
|
+
|
|
1181
|
+
# Default behavior: all result values must be exact matches to at least one
|
|
1182
|
+
# filter (ignoring case). To match end-to-end, we use a ^ to represent
|
|
1183
|
+
# the beginning of each value and a $ to indicate the end. If the user
|
|
1184
|
+
# specifies wildcards on one or both ends of a filter, we'll remove one or both
|
|
1185
|
+
# restrictions as instructed for that filter.
|
|
1186
|
+
|
|
1187
|
+
match_pattern_string = ''
|
|
1188
|
+
|
|
1189
|
+
# If the user includes an empty string in the filters list, make sure we return
|
|
1190
|
+
# a count for empty (null) values in addition to any values matching other filters.
|
|
1191
|
+
|
|
1192
|
+
include_null_count = False
|
|
1193
|
+
|
|
1194
|
+
for filter_pattern in filters:
|
|
1195
|
+
|
|
1196
|
+
if filter_pattern == '':
|
|
1197
|
+
|
|
1198
|
+
include_null_count = True
|
|
1199
|
+
|
|
1200
|
+
else:
|
|
1201
|
+
|
|
1202
|
+
# Process wildcard characters.
|
|
1203
|
+
|
|
1204
|
+
if re.search( r'^\*', filter_pattern ) is not None:
|
|
1205
|
+
|
|
1206
|
+
# Any prefix will do, now.
|
|
1207
|
+
#
|
|
1208
|
+
# Strip leading '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
|
|
1209
|
+
|
|
1210
|
+
filter_pattern = re.sub( r'^\*+', r'', filter_pattern )
|
|
1211
|
+
|
|
1212
|
+
else:
|
|
1213
|
+
|
|
1214
|
+
# No wildcard at the beginning of `filter_pattern` --> require all successful matches to _begin_ with `filter_pattern` by prepending a ^ character to `filter_pattern`:
|
|
1215
|
+
#
|
|
1216
|
+
# ...I know this looks weird, but it's just tacking a '^' character onto the beginning of `filter_pattern`.
|
|
1217
|
+
|
|
1218
|
+
filter_pattern = re.sub( r'^', r'^', filter_pattern )
|
|
1219
|
+
|
|
1220
|
+
if re.search( r'\*$', filter_pattern ) is not None:
|
|
1221
|
+
|
|
1222
|
+
# Any suffix will do, now.
|
|
1223
|
+
#
|
|
1224
|
+
# Strip trailing '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
|
|
1225
|
+
|
|
1226
|
+
filter_pattern = re.sub( r'\*+$', r'', filter_pattern )
|
|
1227
|
+
|
|
1228
|
+
else:
|
|
1229
|
+
|
|
1230
|
+
# No wildcard at the end of `filter_pattern` --> require all successful matches to _end_ with `filter_pattern` by appending a '$' character to `filter_pattern`:
|
|
1231
|
+
#
|
|
1232
|
+
# ...I know this looks weird, but it's just tacking a '$' character onto the end of `filter_pattern`.
|
|
1233
|
+
|
|
1234
|
+
filter_pattern = re.sub( r'$', r'$', filter_pattern )
|
|
1235
|
+
|
|
1236
|
+
# Build the overall match pattern as we go, one (processed) `filter_pattern` at a time.
|
|
1237
|
+
|
|
1238
|
+
match_pattern_string = match_pattern_string + filter_pattern + '|'
|
|
1239
|
+
|
|
1240
|
+
# Strip the final trailing '|' character from the end of the last `filter_pattern`.
|
|
1241
|
+
|
|
1242
|
+
match_pattern_string = re.sub( r'\|$', r'', match_pattern_string )
|
|
1243
|
+
|
|
1244
|
+
print_regex = match_pattern_string
|
|
1245
|
+
|
|
1246
|
+
if include_null_count:
|
|
1247
|
+
if print_regex == '':
|
|
1248
|
+
print_regex = '(missing values)'
|
|
1249
|
+
else:
|
|
1250
|
+
print_regex = print_regex + '|(missing values)'
|
|
1251
|
+
|
|
1252
|
+
if print_regex == '':
|
|
1253
|
+
print_regex = '(none)'
|
|
1254
|
+
else:
|
|
1255
|
+
print_regex = f"/{print_regex}/"
|
|
1256
|
+
|
|
1257
|
+
log.debug( f"Applying pattern filters: {print_regex}" )
|
|
1258
|
+
|
|
1259
|
+
# Filter results to match the full aggregated regular expression in `match_pattern_string`.
|
|
1260
|
+
|
|
1261
|
+
if include_null_count and match_pattern_string != '':
|
|
1262
|
+
|
|
1263
|
+
result_dataframe = result_dataframe.loc[
|
|
1264
|
+
result_dataframe[column].astype( str ).str.contains( match_pattern_string, case=False )
|
|
1265
|
+
| result_dataframe[column].astype( str ).str.contains( r'^$' )
|
|
1266
|
+
| result_dataframe[column].isna()
|
|
1267
|
+
]
|
|
1268
|
+
|
|
1269
|
+
elif include_null_count:
|
|
1270
|
+
|
|
1271
|
+
result_dataframe = result_dataframe.loc[ result_dataframe[column].astype( str ).str.contains( r'^$' ) | result_dataframe[column].isna() ]
|
|
1272
|
+
|
|
1273
|
+
else:
|
|
1274
|
+
|
|
1275
|
+
# This will return unfiltered results if `match_pattern_string` is empty (i.e. if the user asked for no filters to be applied),
|
|
1276
|
+
# and will filter results according to `match_pattern_string` if not.
|
|
1277
|
+
|
|
1278
|
+
result_dataframe = result_dataframe.loc[ result_dataframe[column].astype( str ).str.contains( match_pattern_string, case=False ) ]
|
|
1279
|
+
|
|
1280
|
+
# Sort results. Default (note that the final value of `sort_by` is determined earlier in this function) is to sort by term count, descending.
|
|
1281
|
+
|
|
1282
|
+
log.debug( f"Applying sort directive '{sort_by}'..." )
|
|
1283
|
+
|
|
1284
|
+
if sort_by == 'count':
|
|
1285
|
+
|
|
1286
|
+
# Sort by count; break ties among groups of values with identical counts by sub-sorting each such group alphabetically by value.
|
|
1287
|
+
result_dataframe = result_dataframe.sort_values( by=['value_count', column], ascending=[True, True] )
|
|
1288
|
+
|
|
1289
|
+
elif sort_by == 'count:desc':
|
|
1290
|
+
|
|
1291
|
+
# Sort by count, descending; break ties among groups of values with identical counts by sub-sorting each such group alphabetically by value.
|
|
1292
|
+
result_dataframe = result_dataframe.sort_values( by=['value_count', column], ascending=[False, True] )
|
|
1293
|
+
|
|
1294
|
+
elif sort_by == 'value':
|
|
1295
|
+
|
|
1296
|
+
# No need for a sub-sort, here, since values aren't repeated.
|
|
1297
|
+
result_dataframe = result_dataframe.sort_values( by=column, ascending=True )
|
|
1298
|
+
|
|
1299
|
+
elif sort_by == 'value:desc':
|
|
1300
|
+
|
|
1301
|
+
# No need for a sub-sort, here, since values aren't repeated.
|
|
1302
|
+
result_dataframe = result_dataframe.sort_values( by=column, ascending=False )
|
|
1303
|
+
|
|
1304
|
+
else:
|
|
1305
|
+
|
|
1306
|
+
log.error( 'Something has gone horribly wrong; we should never get here.' )
|
|
1307
|
+
return
|
|
1308
|
+
|
|
1309
|
+
#############################################################################################################################
|
|
1310
|
+
# Send the results back to the user.
|
|
1311
|
+
|
|
1312
|
+
# Reindex DataFrame rows to match their final sort order.
|
|
1313
|
+
|
|
1314
|
+
result_dataframe = result_dataframe.reset_index( drop=True )
|
|
1315
|
+
|
|
1316
|
+
# Pretty-print missing values.
|
|
1317
|
+
|
|
1318
|
+
if result_dataframe[column].dtype == 'object':
|
|
1319
|
+
|
|
1320
|
+
# String data comes through as a column with dtype 'object', based on something involving
|
|
1321
|
+
# the variability inherent in string lengths.
|
|
1322
|
+
#
|
|
1323
|
+
# See https://stackoverflow.com/questions/33957720/how-to-convert-column-with-dtype-as-object-to-string-in-pandas-dataframe
|
|
1324
|
+
|
|
1325
|
+
# Replace null string values with <NA> to match what we['re forced to] use for numeric data.
|
|
1326
|
+
|
|
1327
|
+
result_dataframe = result_dataframe.replace( r'^$', r'<NA>', regex=True )
|
|
1328
|
+
|
|
1329
|
+
elif result_dataframe[column].dtype == 'bool':
|
|
1330
|
+
|
|
1331
|
+
# Replace null boolean values with <NA> to match what we['re forced to] use for numeric data.
|
|
1332
|
+
result_dataframe = result_dataframe.replace(r'^$', r'<NA>', regex=True)
|
|
1333
|
+
|
|
1334
|
+
if return_data_as == '':
|
|
1335
|
+
|
|
1336
|
+
# Right now, the default is the same as if the user had
|
|
1337
|
+
# specified return_data_as='dataframe'.
|
|
1338
|
+
|
|
1339
|
+
# The following, for the dubiously useful record, is a somewhat worse alternative default thing to do.
|
|
1340
|
+
#
|
|
1341
|
+
# print( result_dataframe.to_string( index=False, justify='right', max_rows=25, max_colwidth=50 ), file=sys.stdout )
|
|
1342
|
+
|
|
1343
|
+
log.debug( 'Returning results in default form (pandas.DataFrame)' )
|
|
1344
|
+
|
|
1345
|
+
return result_dataframe
|
|
1346
|
+
|
|
1347
|
+
elif return_data_as == 'dataframe':
|
|
1348
|
+
|
|
1349
|
+
# Give the user back the results DataFrame.
|
|
1350
|
+
|
|
1351
|
+
log.debug( 'Returning results as pandas.DataFrame' )
|
|
1352
|
+
return result_dataframe
|
|
1353
|
+
|
|
1354
|
+
elif return_data_as == 'list':
|
|
1355
|
+
|
|
1356
|
+
# Strip the term-values column out of the results DataFrame and give them to the user as a Python list.
|
|
1357
|
+
|
|
1358
|
+
log.debug( 'Returning results as list of column values' )
|
|
1359
|
+
return result_dataframe[column].to_list()
|
|
1360
|
+
|
|
1361
|
+
else:
|
|
1362
|
+
|
|
1363
|
+
# Write the results DataFrame to a user-specified TSV file.
|
|
1364
|
+
|
|
1365
|
+
log.debug( f"Printing results to TSV file '{output_file}'" )
|
|
1366
|
+
|
|
1367
|
+
try:
|
|
1368
|
+
result_dataframe.to_csv( output_file, sep='\t', index=False )
|
|
1369
|
+
return
|
|
1370
|
+
|
|
1371
|
+
except Exception as error:
|
|
1372
|
+
raise RuntimeError( f"Couldn't write to requested output file '{output_file}': got error of type '{type(error)}', with error message '{error}'." )
|
|
1373
|
+
|
|
1374
|
+
#############################################################################################################################
|
|
1375
|
+
#
|
|
1376
|
+
# END column_values()
|
|
1377
|
+
#
|
|
1378
|
+
#############################################################################################################################
|
|
1379
|
+
|
|
1380
|
+
#############################################################################################################################
|
|
1381
|
+
#
|
|
1382
|
+
# release_metadata(): Return a list of metadata dicts describing the current CDA release, columnwise.
|
|
1383
|
+
#
|
|
1384
|
+
#############################################################################################################################
|
|
1385
|
+
|
|
1386
|
+
def release_metadata():
|
|
1387
|
+
"""
|
|
1388
|
+
Return a list of metadata dicts describing the current CDA release, columnwise.
|
|
1389
|
+
"""
|
|
1390
|
+
|
|
1391
|
+
log = get_logger()
|
|
1392
|
+
|
|
1393
|
+
log.debug( 'Querying /release_metadata endpoint' )
|
|
1394
|
+
|
|
1395
|
+
query_api_instance = cda_client.Client( base_url=get_api_url(), raise_on_unexpected_status=True )
|
|
1396
|
+
|
|
1397
|
+
try:
|
|
1398
|
+
release_metadata_response_data_object = cda_client.api.release_metadata.release_metadata_endpoint_release_metadata_get.sync( client=query_api_instance )
|
|
1399
|
+
except UnexpectedStatus as error:
|
|
1400
|
+
raise RuntimeError( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
|
|
1401
|
+
except Exception as error:
|
|
1402
|
+
raise RuntimeError( f"{type(error)}: {error}" )
|
|
1403
|
+
|
|
1404
|
+
return release_metadata_response_data_object.to_dict()['result']
|
|
1405
|
+
|
|
1406
|
+
#############################################################################################################################
|
|
1407
|
+
#
|
|
1408
|
+
# END release_metadata()
|
|
1409
|
+
#
|
|
1410
|
+
#############################################################################################################################
|
|
1411
|
+
|
|
1412
|
+
|