cdapython 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cdapython/discover.py ADDED
@@ -0,0 +1,1412 @@
1
+ import json
2
+ import os
3
+ import pandas as pd
4
+ import re
5
+
6
+ import cda_client
7
+ import cda_client.api.columns.columns_endpoint_columns_get
8
+ import cda_client.api.release_metadata.release_metadata_endpoint_release_metadata_get
9
+ import cda_client.api.column_values.column_values_endpoint_column_values_column_post
10
+
11
+ from cdapython.application_utilities import get_api_url
12
+ from cda_client.errors import UnexpectedStatus
13
+ from cdapython.logging_wrappers import get_logger
14
+
15
+ #############################################################################################################################
16
+ #############################################################################################################################
17
+ # Nomenclature notes:
18
+ #
19
+ # * try to standardize all potential user-facing synonyms for basic database data structures
20
+ # (field, entity, endpoint, cell, value, term, etc.) to 'table', 'column', 'row' and 'value'.
21
+ #############################################################################################################################
22
+ #############################################################################################################################
23
+
24
+
25
+ #############################################################################################################################
26
+ #
27
+ # cda_functions(): Return a list of user-facing cdapython functions useful for both scripting and interactive data sessions.
28
+ #
29
+ #############################################################################################################################
30
+
31
+ def cda_functions():
32
+ """
33
+ Returns a list of cdapython functions useful for both scripting and interactive data sessions.
34
+ """
35
+
36
+ # There doesn't seem to be a need to check invocation context before just printing to standard output,
37
+ # here, as this function is primarily designed to be consulted in interactive sessions. If anyone ever
38
+ # finds themselves checking this function's output programmatically, for some reason, and is annoyed
39
+ # by the print statement, we will entertain pull requests to be more sensitive about our assumptions.
40
+
41
+ print( '\nYou can get complete usage details for each function by calling "help( function_name )".', end='\n\n' )
42
+
43
+ return [
44
+ 'tables',
45
+ 'columns',
46
+ 'column_values',
47
+ 'summarize_subjects',
48
+ 'summarize_files',
49
+ 'get_subject_data',
50
+ 'get_file_data',
51
+ 'intersect_subject_results',
52
+ 'intersect_file_results',
53
+ 'expand_subject_results',
54
+ 'expand_file_results',
55
+ 'get_valid_log_levels',
56
+ 'get_log_level',
57
+ 'set_log_level',
58
+ 'enable_console_logging',
59
+ 'disable_console_logging',
60
+ 'enable_file_logging',
61
+ 'disable_file_logging',
62
+ 'get_api_url',
63
+ 'set_api_url',
64
+ 'cda_functions'
65
+ ]
66
+
67
+ #############################################################################################################################
68
+ #
69
+ # END cda_functions()
70
+ #
71
+ #############################################################################################################################
72
+
73
+ #############################################################################################################################
74
+ #
75
+ # tables(): Return a list of all searchable CDA data tables.
76
+ #
77
+ #############################################################################################################################
78
+
79
+ def tables():
80
+ """
81
+ Get a list of all searchable CDA data tables.
82
+
83
+ Returns:
84
+ list of strings: names of searchable CDA tables.
85
+ """
86
+
87
+ log = get_logger()
88
+
89
+ # Call columns(), extract unique values from the `table` column of the
90
+ # resulting DataFrame, and return those values to the user as a list.
91
+
92
+ log.debug( 'Calling columns()' )
93
+
94
+ columns_result_df = columns( return_data_as='dataframe' )
95
+
96
+ if columns_result_df is None:
97
+ log.error( 'Something went fatally wrong with columns( return_data_as="dataframe" ): got a null DataFrame back.' )
98
+ return
99
+ else:
100
+ return sorted( columns_result_df['table'].unique() )
101
+
102
+ #############################################################################################################################
103
+ #
104
+ # END tables()
105
+ #
106
+ #############################################################################################################################
107
+
108
+ #############################################################################################################################
109
+ #
110
+ # columns(): Provide user with structured metadata describing searchable CDA columns:
111
+ #
112
+ # (containing) table
113
+ # column (name)
114
+ # data_type (stored in column)
115
+ # (values are) nullable(?)
116
+ # (prose) description
117
+ #
118
+ #############################################################################################################################
119
+
120
+
121
+ def columns(
122
+ *,
123
+ return_data_as='',
124
+ output_file='',
125
+ sort_by='',
126
+ **filter_arguments
127
+ ):
128
+ """
129
+ Get structured metadata describing searchable CDA columns.
130
+
131
+ Arguments:
132
+ return_data_as ( string; optional: 'dataframe' or 'list' or 'tsv' ):
133
+ Specify how columns() should return results: as a pandas DataFrame,
134
+ a Python list, or as output written to a TSV file named by the user.
135
+ If this argument is omitted, columns() will default to returning
136
+ results as a DataFrame.
137
+
138
+ output_file( string; optional ):
139
+ If return_data_as='tsv' is specified, output_file should contain a
140
+ resolvable path to a file into which columns() will write
141
+ tab-delimited results.
142
+
143
+ sort_by( string or list of strings; optional:
144
+ any combination of 'table', 'column', 'data_type',
145
+ and/or 'nullable'):
146
+ Specify the column metadata field(s) on which to sort result data.
147
+ Results will be sorted first by the first named field; groups of
148
+ records sharing the same value in the first field will then be
149
+ sub-sorted by the second field, and so on.
150
+
151
+ Any field with a suffix of ':desc' appended to it will be sorted
152
+ in reverse order; adding ':asc' will ensure ascending sort order.
153
+ Example: sort_by=[ 'table', 'nullable:desc', 'column:asc' ]
154
+
155
+ Filter arguments:
156
+ table ( string or list of strings; optional ):
157
+ Restrict returned data to columns from tables whose names match any
158
+ of the given strings. A wildcard (asterisk) at either end (or both
159
+ ends) of each string will allow partial matches. Case will be
160
+ ignored.
161
+
162
+ column ( string or list of strings; optional ):
163
+ Restrict returned data to columns whose name matches any of the
164
+ given strings. A wildcard (asterisk) at either end (or both ends)
165
+ of each string will allow partial matches. Case will be ignored.
166
+
167
+ data_type ( string or list of strings; optional ):
168
+ Restrict returned data to columns whose data type matches any of
169
+ the given strings. A wildcard (asterisk) at either end (or both
170
+ ends) of each string will allow partial matches. Case will be
171
+ ignored.
172
+
173
+ nullable ( boolean; optional ):
174
+ If set to True, restrict returned data to columns whose values are
175
+ allowed to be empty; if False, return data only for columns
176
+ requiring nonempty values.
177
+
178
+ description ( string or list of strings; optional ):
179
+ Restrict returned data to columns whose `description` field matches
180
+ any of the given strings. Wildcards will be automatically applied
181
+ if not provided, to support straightforward keyword searching of this
182
+ field without requiring too much extra punctuation. Case will be
183
+ ignored.
184
+
185
+ exclude_table ( string or list of strings; optional ):
186
+ Restrict returned data to columns from tables whose names do _not_
187
+ match any of the given strings. A wildcard (asterisk) at either end
188
+ (or both ends) of each string will allow partial matches. Case will
189
+ be ignored.
190
+
191
+ Returns:
192
+ pandas.DataFrame where each row is a metadata record describing one
193
+ searchable CDA column and is comprised of the following fields:
194
+
195
+ `table` (string: name of the CDA table containing this column)
196
+ `column` (string: name of this column)
197
+ `data_type` (string: data type of this column)
198
+ `nullable` (boolean: if True, this column can contain null values)`
199
+ `description` (string: prose description of this column)
200
+
201
+ OR list of column names
202
+
203
+ OR returns nothing, but writes results to a user-specified TSV file
204
+ """
205
+
206
+ log = get_logger()
207
+
208
+ #############################################################################################################################
209
+ # Process return-type directives `return_data_as` and `output_file`.
210
+
211
+ if not isinstance(return_data_as, str):
212
+
213
+ log.error( f"Unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
214
+ return
215
+
216
+ # Let's not be picky if someone wants to give us return_data_as='DataFrame' or return_data_as='TSV'
217
+
218
+ return_data_as = return_data_as.lower()
219
+
220
+ # We can't do much validation on filenames. If `output_file` isn't
221
+ # a locally writeable path, it'll fail when we try to open it for
222
+ # writing. Strip trailing whitespace from both ends and wrap the
223
+ # file-access operation (later, below) in a try{} block.
224
+
225
+ if not isinstance( output_file, str ):
226
+
227
+ log.error( f"The `output_file` parameter, if not omitted, should be a string containing a path to the desired output file. You supplied '{output_file}', which is not a string, let alone a valid path." )
228
+ return
229
+
230
+ output_file = output_file.strip()
231
+
232
+ allowed_return_types = { '', 'dataframe', 'tsv', 'list' }
233
+
234
+ if return_data_as not in allowed_return_types:
235
+
236
+ # Complain if we receive an unexpected `return_data_as` value.
237
+ log.error( f"Unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
238
+ return
239
+
240
+ elif return_data_as == 'tsv' and output_file == '':
241
+
242
+ # If the user asks for a TSV, they also have to give us a path for that TSV. If they didn't, complain.
243
+ log.error( "Return type 'tsv' was requested, but 'output_file' was not specified. Please specify output_file='some/path/string/to/write/your/tsv/to/your_tsv_output_file.tsv'." )
244
+ return
245
+
246
+ elif return_data_as != 'tsv' and output_file != '':
247
+
248
+ # If the user put something in the `output_file` parameter but didn't specify `result_data_as='tsv'`,
249
+ # they most likely want their data saved to a file (so ignoring the parameter misconfiguration
250
+ # isn't safe), but ultimately we can't be sure what they meant (so taking an action isn't safe),
251
+ # so we complain and ask them to clarify.
252
+
253
+ log.error( f"'output_file' was specified, but this is only meaningful if 'return_data_as' is set to 'tsv'. You requested return_data_as='{return_data_as}'." )
254
+ log.error( '(Note that if you don\'t specify any value for \'return_data_as\', it defaults to \'dataframe\'.).' )
255
+ return
256
+
257
+ #############################################################################################################################
258
+ # Process `sort_by` directives.
259
+
260
+ if isinstance(sort_by, str):
261
+
262
+ # Make `sort_by` a list, if it's not, so we don't have to split the way we
263
+ # process this information into parallel distinct branches.
264
+
265
+ if sort_by == '':
266
+ sort_by = []
267
+ else:
268
+ sort_by = [ sort_by ]
269
+
270
+ elif not isinstance(sort_by, list):
271
+
272
+ # Also detect any disallowed incoming data types and complain if we find any.
273
+
274
+ log.error( f"columns(): ERROR: 'sort_by' must be a string or a list of strings; you used '{sort_by}', which is neither." )
275
+ return
276
+
277
+ # Enumerate all allowed values that a user can specify using the `sort_by` parameter. ( 'X:asc' will be aliased immediately to just 'X'. )
278
+
279
+ allowed_sort_by_arguments = [
280
+ 'table',
281
+ 'table:desc',
282
+ 'column',
283
+ 'column:desc',
284
+ 'data_type',
285
+ 'data_type:desc',
286
+ 'nullable',
287
+ 'nullable:desc',
288
+ ]
289
+
290
+ # Build two lists to pass to `DataFrame.sort_values()` to direct the sorting of our result data
291
+ # according to user specifications:
292
+ #
293
+ # * `by_list` will contain exact field names on which to sort, in order of precedence.
294
+ #
295
+ # * `ascending_list` will be a series of boolean values, one for each field name in
296
+ # `by_list`, where each `False` value indicates that the corresponding field
297
+ # in `by_list` is to be sorted in reverse.
298
+
299
+ by_list = list()
300
+
301
+ ascending_list = list()
302
+
303
+ seen_so_far = dict()
304
+
305
+ for field_code in sort_by:
306
+
307
+ if not isinstance(field_code, str):
308
+
309
+ # Complain if we receive any unexpected data types instead of string directives.
310
+
311
+ log.critical( f"columns(): ERROR: 'sort_by' must be a string or a list of strings; you used '{sort_by}', which is neither." )
312
+ return
313
+
314
+ # Let's not care about case.
315
+
316
+ field_code = field_code.lower()
317
+
318
+ # ':asc' is redundant. Remove it (politely).
319
+
320
+ field_code = re.sub( r':asc$', r'', field_code )
321
+
322
+ if field_code not in allowed_sort_by_arguments:
323
+
324
+ # Complain if we receive any unexpected sort_by directives.
325
+
326
+ log.critical( f"columns(): ERROR: '{field_code}' is not a valid directive for the 'sort_by' parameter. Please use one of [ '"
327
+ + "', '".join(allowed_sort_by_arguments)
328
+ + "' ] instead." )
329
+ return
330
+
331
+ code_basename = field_code
332
+
333
+ if re.search( r':desc$', field_code ) is not None:
334
+ code_basename = re.sub( r':desc$', '', field_code )
335
+
336
+ if code_basename not in seen_so_far:
337
+
338
+ seen_so_far[code_basename] = field_code
339
+
340
+ else:
341
+
342
+ # Complain if we receive multiple sort_by directives for the same output column.
343
+
344
+ log.critical( f"columns(): ERROR: Multiple sort_by directives received for the same output column, including '{seen_so_far[code_basename]}' and '{field_code}': please specify only one directive per output column." )
345
+ return
346
+
347
+ by_list.append( code_basename )
348
+
349
+ if re.search( r':desc$', field_code ) is not None:
350
+
351
+ # Reverse the sort on this column.
352
+ ascending_list.append( False )
353
+
354
+ else:
355
+
356
+ # Sort this column normally.
357
+ ascending_list.append( True )
358
+
359
+ # Report details of the final parsed sort logic.
360
+
361
+ sort_dataframe = pd.DataFrame( { 'sort_by': by_list, 'ascending?': ascending_list } )
362
+
363
+ if not sort_dataframe.empty:
364
+ log.debug( f"Processed sort directives: {sort_dataframe}" )
365
+ else:
366
+ log.debug( f"Processed sort directives: <default>" )
367
+
368
+ #############################################################################################################################
369
+ # Process user-supplied filter directives.
370
+
371
+ # Enumerate all allowed filters that a user can specify with named parameters.
372
+
373
+ allowed_filter_arguments = [
374
+ 'table',
375
+ 'column',
376
+ 'data_type',
377
+ 'nullable',
378
+ 'description',
379
+ 'exclude_table'
380
+ ]
381
+
382
+ # Validate filter argument content.
383
+
384
+ for filter_argument_name in filter_arguments:
385
+
386
+ if filter_argument_name not in allowed_filter_arguments:
387
+
388
+ # Complain if we receive any unexpected filter arguments.
389
+
390
+ log.critical( f"columns(): ERROR: Received unexpected argument {filter_argument_name}; aborting." )
391
+ return
392
+
393
+ elif filter_argument_name == 'nullable':
394
+
395
+ if not isinstance( filter_arguments[filter_argument_name], bool ):
396
+
397
+ # Complain if we got a parameter value of the wrong data type.
398
+
399
+ log.critical( f"columns(): ERROR: 'nullable' must be a Boolean value (True or False); you used '{filter_arguments[filter_argument_name]}', which is not." )
400
+ return
401
+
402
+ elif not ( isinstance( filter_arguments[filter_argument_name], str ) or isinstance( filter_arguments[filter_argument_name], list ) ):
403
+
404
+ # Complain if we got a parameter value of the wrong data type.
405
+
406
+ log.critical( f"columns(): ERROR: '{filter_argument_name}' must be a string or a list of strings; you used '{filter_arguments[filter_argument_name]}', which is neither." )
407
+ return
408
+
409
+ elif isinstance( filter_arguments[filter_argument_name], list ):
410
+
411
+ for pattern in filter_arguments[filter_argument_name]:
412
+
413
+ if not isinstance( pattern, str ):
414
+
415
+ # Complain if we receive any unexpected data types inside a filter list (i.e. anything but strings).
416
+
417
+ log.critical( f"columns(): ERROR: '{filter_argument_name}' must be a string or a list of strings; you used '{filter_arguments[filter_argument_name]}', which is neither." )
418
+ return
419
+
420
+ # Report details of fully processed user directives prior to querying.
421
+
422
+ status_report = f"return_data_as='{return_data_as}', output_file='{output_file}'"
423
+
424
+ status_report = status_report + f", sort_by={sort_by}"
425
+
426
+ for filter_argument_name in filter_arguments:
427
+
428
+ if isinstance( filter_arguments[filter_argument_name], str ):
429
+ status_report = status_report + f", {filter_argument_name}='{filter_arguments[filter_argument_name]}'"
430
+ else:
431
+ status_report = status_report + f", {filter_argument_name}={filter_arguments[filter_argument_name]}"
432
+
433
+ log.debug( f"Processed user directives: '{status_report}'" )
434
+
435
+ #############################################################################################################################
436
+ # Fetch data from the API.
437
+
438
+ query_api_instance = cda_client.Client( base_url=get_api_url(), raise_on_unexpected_status=True )
439
+
440
+ # Ask the columns endpoint for information. (It has no parameters.)
441
+
442
+ try:
443
+ columns_response_data_object = cda_client.api.columns.columns_endpoint_columns_get.sync( client=query_api_instance )
444
+ except UnexpectedStatus as error:
445
+ log.error( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
446
+ return
447
+ except Exception as error:
448
+ log.error( f"{type(error)}: {error}" )
449
+ return
450
+
451
+ #############################################################################################################################
452
+ # Postprocess API result data.
453
+
454
+ # columns_response_data_object['result'] is an array of dicts, with
455
+ # each dict containing a few named fields of metadata describing one column.
456
+ #
457
+ # Example:
458
+ #
459
+ # "result": [
460
+ # {
461
+ # "table": "file",
462
+ # "column": "file_id",
463
+ # "data_type": "text",
464
+ # "nullable": false,
465
+ # "description": "A unique identifier for this file minted by CDA. May change release-to-release. Contains no semantically reliable content with one exception: in the case of a DICOM series from IDC, this field will contain the crdc_series_uuid assigned by IDC to that DICOM series. Note that this crdc_series_uuid may change from one IDC release version to the next, according to IDC's data processing and identification rules."
466
+ # },
467
+ # ...
468
+ # ]
469
+
470
+ # Make a DataFrame from this array of dicts using DataFrame.from_records(), and explicitly specify the
471
+ # column ordering for the resulting DataFrame using the `columns=[]` parameter.
472
+
473
+ result_dataframe = pd.DataFrame.from_records( columns_response_data_object.to_dict()['result'], columns=[ 'table', 'column', 'data_type', 'nullable', 'description' ] )
474
+
475
+ # Remove `table`_data_source_count and *_alias columns from output.
476
+ #
477
+ # TO DO: Maybe put this list somewhere easier to find.
478
+
479
+ banned_column_name_patterns = {
480
+ r'_crdc_id$',
481
+ r'^[^_]+_data_source_count$',
482
+ r'^[^_]+_data_at_[^_]+$',
483
+ r'_alias$'
484
+ }
485
+
486
+ banned_columns = set()
487
+
488
+ for column_name in result_dataframe['column'].unique():
489
+ for banned_pattern in banned_column_name_patterns:
490
+ if re.search( banned_pattern, column_name ) is not None:
491
+ banned_columns.add( column_name )
492
+
493
+ for banned_column in banned_columns:
494
+ result_dataframe = result_dataframe.loc[ result_dataframe['column'] != banned_column ]
495
+
496
+ log.debug( 'Created result DataFrame' )
497
+
498
+ #############################################################################################################################
499
+ # Execute sorting directives, if we got any; otherwise perform the default sort on the result DataFrame.
500
+
501
+ if len( sort_by ) == 0:
502
+
503
+ # By default, we sort column records by column name, gathered into groups by table,
504
+ # to facilitate predictable output patterns. For easy access, we'd also like each
505
+ # table's ID column to show up first in its table's group.
506
+ #
507
+ # Temporarily prepend a '.' to `table`_id column names, so they float to the top of each
508
+ # table's list of columns when we sort.
509
+
510
+ result_dataframe = result_dataframe.replace( to_replace=r'^([^_]+_id)$', value=r'.\1', regex=True )
511
+
512
+ # Sort all column records, first on table and then on column name.
513
+
514
+ result_dataframe = result_dataframe.sort_values( by=['table', 'column'], ascending=[True, True] )
515
+
516
+ # Remove the '.' characters we temporarily prepended to `table`_id column names
517
+ # to force the sorting algorithm to place all such columns first within each
518
+ # table's group of column records.
519
+
520
+ result_dataframe = result_dataframe.replace( to_replace=r'^\.(.*)$', value=r'\1', regex=True )
521
+
522
+ else:
523
+
524
+ # Sort all column records according to the user-specified directives we've processed.
525
+
526
+ result_dataframe = result_dataframe.sort_values( by=by_list, ascending=ascending_list )
527
+
528
+ log.debug( 'Applied sort_by directives' )
529
+
530
+ #############################################################################################################################
531
+ # Iterate through whatever filters the user passed us and
532
+ # apply them to the result data before sending it back.
533
+ #
534
+ # The value of filter_name, here, will be one of
535
+ # 'table', 'column', 'data_type', 'nullable',
536
+ # 'description' or 'exclude_table'.
537
+
538
+ for filter_name in filter_arguments:
539
+
540
+ # Grab the filters the user sent us.
541
+
542
+ # Default behavior: all result values must be exact matches to at least one
543
+ # filter (ignoring case). To match end-to-end, we use a ^ to represent
544
+ # the beginning of each value and a $ to indicate the end. If the user
545
+ # specifies wildcards on one or both ends of a filter, we'll remove one or both
546
+ # restrictions as instructed for that filter.
547
+ #
548
+ # EXCEPTION ONE: the `nullable` filter argument will be a single Boolean
549
+ # value, and we handle it separately.
550
+ #
551
+ # EXCEPTION TWO: For the `description` field, we've modified the
552
+ # argument processing (because in this case users are searching an
553
+ # abstract-sized block of text, not a short string representing
554
+ # a name or a concept) so that filters will always be processed
555
+ # as if they have wildcards on both ends. (Search will still be
556
+ # case-insensitive at all times.)
557
+ #
558
+ # EXCEPTION THREE: In the case of `exclude_table`, all result values must
559
+ # _not_ match _any_ of the specified filters.
560
+
561
+ if filter_name == 'nullable':
562
+
563
+ return_if_nullable = filter_arguments[filter_name]
564
+
565
+ if not isinstance( return_if_nullable, bool ):
566
+ log.critical( f"columns(): ERROR: Please specify either nullable=True or nullable=False, not (what you sent) nullable='{return_if_nullable}'." )
567
+ return
568
+
569
+ result_dataframe = result_dataframe.loc[ result_dataframe['nullable'] == return_if_nullable ]
570
+
571
+ else:
572
+
573
+ filters = filter_arguments[filter_name]
574
+
575
+ filter_patterns = list()
576
+
577
+ # If the filter list wasn't a list at all but a (nonempty) string, we just have
578
+ # one filter. Listify it (so we don't have to care downstream about how many there are).
579
+ # Otherwise, just start with the list they sent us.
580
+
581
+ if isinstance( filters, str ) and filters != '':
582
+ filter_patterns = [filters]
583
+ elif isinstance( filters, list ):
584
+ filter_patterns = filters
585
+
586
+ # (If neither of the above conditions was met, `filter_patterns` will remain an
587
+ # empty list, and the rest of this filter-processing section will (by design) have no effect.
588
+
589
+ # Parse filter_name to establish which columns() field is being targeted
590
+ # for filtration and adjust default filtration logic as necessary according to the result.
591
+
592
+ target_field = filter_name
593
+
594
+ if filter_name == 'description':
595
+
596
+ # Never match end-to-end for query strings applied to description text (see discussion above).
597
+
598
+ updated_pattern_list = list()
599
+
600
+ for original_filter_pattern in filter_patterns:
601
+
602
+ updated_filter_pattern = f"*{original_filter_pattern}*"
603
+ updated_pattern_list.append( updated_filter_pattern )
604
+
605
+ filter_patterns = updated_pattern_list
606
+
607
+ elif filter_name == 'exclude_table':
608
+
609
+ target_field = 'table'
610
+
611
+ match_pattern_string = ''
612
+
613
+ for filter_pattern in filter_patterns:
614
+
615
+ # Process wildcard characters.
616
+
617
+ if re.search( r'^\*', filter_pattern ) is not None:
618
+
619
+ # Any prefix will do, now.
620
+ #
621
+ # Strip leading '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
622
+
623
+ filter_pattern = re.sub( r'^\*+', r'', filter_pattern )
624
+
625
+ else:
626
+
627
+ # No wildcard at the beginning of `filter_pattern` --> require all successful matches to _begin_ with `filter_pattern` by prepending a ^ character to `filter_pattern`:
628
+ #
629
+ # ...I know this looks weird, but it's just tacking a '^' character onto the beginning of `filter_pattern`.
630
+
631
+ filter_pattern = re.sub( r'^', r'^', filter_pattern )
632
+
633
+ if re.search( r'\*$', filter_pattern ) is not None:
634
+
635
+ # Any suffix will do, now.
636
+ #
637
+ # Strip trailing '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
638
+
639
+ filter_pattern = re.sub( r'\*+$', r'', filter_pattern )
640
+
641
+ else:
642
+
643
+ # No wildcard at the end of `filter_pattern` --> require all successful matches to _end_ with `filter_pattern` by appending a '$' character to `filter_pattern`:
644
+ #
645
+ # ...I know this looks weird, but it's just tacking a '$' character onto the end of `filter_pattern`.
646
+
647
+ filter_pattern = re.sub( r'$', r'$', filter_pattern )
648
+
649
+ # Build the overall match pattern as we go, one (processed) `filter_pattern` at a time.
650
+
651
+ match_pattern_string = match_pattern_string + filter_pattern + '|'
652
+
653
+ # Strip final trailing |.
654
+
655
+ match_pattern_string = re.sub( r'\|$', r'', match_pattern_string )
656
+
657
+ if filter_name == 'exclude_table':
658
+
659
+ # Retain all rows where the value of `target_field` (in this case, the value of `table`) does _not_ match any of the given filter patterns.
660
+
661
+ result_dataframe = result_dataframe.loc[ ~( result_dataframe[target_field].str.contains( match_pattern_string, case=False ) ) ]
662
+
663
+ else:
664
+
665
+ # Retain all rows where the value of `target_field` matches any of the given filter patterns.
666
+
667
+ result_dataframe = result_dataframe.loc[ result_dataframe[target_field].str.contains( match_pattern_string, case=False ) ]
668
+
669
+ log.debug( 'Applied value-filtration directives' )
670
+
671
+ #############################################################################################################################
672
+ # Send the results back to the user.
673
+
674
+ # Reindex DataFrame rows to match their final sort order.
675
+
676
+ result_dataframe = result_dataframe.reset_index( drop=True )
677
+
678
+ if return_data_as == '':
679
+
680
+ # Right now, the default is the same as if the user had
681
+ # specified return_data_as='dataframe'.
682
+
683
+ # The following, for the dubiously useful record, is a somewhat worse alternative default thing to do.
684
+ #
685
+ # print( result_dataframe.to_string( index=False, justify='right', max_rows=25, max_colwidth=50 ), file=sys.stdout )
686
+
687
+ log.debug( 'Returning results in default form (pandas.DataFrame)' )
688
+
689
+ return result_dataframe
690
+
691
+ elif return_data_as == 'dataframe':
692
+
693
+ # Give the user back the results DataFrame.
694
+
695
+ log.debug( 'Returning results as pandas.DataFrame' )
696
+
697
+ return result_dataframe
698
+
699
+ elif return_data_as == 'list':
700
+
701
+ # Give the user back a list of column names.
702
+
703
+ log.debug( 'Returning results as list of column names' )
704
+
705
+ return result_dataframe['column'].to_list()
706
+
707
+ else:
708
+
709
+ # Write the results DataFrame to a user-specified TSV file.
710
+
711
+ log.debug( f"Printing results to TSV file '{output_file}'" )
712
+
713
+ try:
714
+
715
+ result_dataframe.to_csv( output_file, sep='\t', index=False )
716
+ return
717
+
718
+ except Exception as error:
719
+ raise RuntimeError( f"Couldn't write to requested output file '{output_file}': got error of type '{type(error)}', with error message '{error}'." )
720
+
721
+ #############################################################################################################################
722
+ #
723
+ # END columns()
724
+ #
725
+ #############################################################################################################################
726
+
727
+ #############################################################################################################################
728
+ #
729
+ # column_values( column=`column` ): Show all distinct values present in `column`, along with a count of occurrences for each value.
730
+ #
731
+ #############################################################################################################################
732
+
733
+ def column_values(
734
+ column='',
735
+ *,
736
+ return_data_as='',
737
+ output_file='',
738
+ sort_by='',
739
+ filters=None,
740
+ data_source='',
741
+ force=False
742
+ ):
743
+ """
744
+ Show all distinct values present in `column`, along with a count
745
+ of occurrences for each value.
746
+
747
+ Arguments:
748
+ column ( string; required ):
749
+ The column to fetch values from.
750
+
751
+ return_data_as ( string; optional: 'dataframe' or 'list' or 'tsv' ):
752
+ Specify how column_values() should return results: as a pandas
753
+ DataFrame, a Python list, or as output written to a TSV file named
754
+ by the user. If this argument is omitted, column_values() will default
755
+ to returning results as a DataFrame.
756
+
757
+ output_file( string; optional ):
758
+ If return_data_as='tsv' is specified, output_file should contain a
759
+ resolvable path to a file into which column_values() will write
760
+ tab-delimited results.
761
+
762
+ sort_by( string; optional:
763
+ 'count' ( default for return_data_as='dataframe' and
764
+ return_data_as='tsv' ) or 'value' ( default for
765
+ return_data_as='list' ) or 'count:desc' or 'value:desc'
766
+ or 'count:asc' or 'value:asc' ):
767
+ Specify the primary column to sort when preparing result data: on
768
+ values, or on counts of values.
769
+
770
+ A column name with a suffix of ':desc' appended to it will be
771
+ sorted in reverse order; adding ':asc' will ensure ascending sort
772
+ order. Example: sort_by='value:desc'
773
+
774
+ Secondary sort order is automatic: if the results are to be
775
+ primarily sorted by count, then the automatic behavior will be to
776
+ also (alphabetically) sort by value within each group of values
777
+ that all share the same count. If results are primarily sorted by
778
+ value, then there is no secondary sort -- each value is unique by
779
+ design, so results don't contain groups with the same value but
780
+ different counts, so there's nothing to arrange once the primary
781
+ sort has been applied.
782
+
783
+ filters ( string or list of strings; optional ):
784
+ Restrict returned values to those matching any of the given strings.
785
+ A wildcard (asterisk) at either end (or both ends) of each string
786
+ will allow partial matches. Case will be ignored. Specify an empty
787
+ filter string '' to match and count missing (null) values.
788
+
789
+ data_source ( string; optional ):
790
+ Restrict returned values to the given upstream data source. Current
791
+ valid values are 'GDC', 'PDC', 'IDC', 'CDS' and 'ICDC'.
792
+ Defaults to '' (no filter).
793
+
794
+ force( boolean; optional ):
795
+ Force execution of high-overhead queries on columns (like IDs)
796
+ flagged as having large numbers of values. Defaults to False,
797
+ in which case attempts to retrieve values for flagged columns
798
+ will result in a warning.
799
+
800
+
801
+ Returns:
802
+ pandas.DataFrame OR list OR returns nothing, but writes retrieved
803
+ data to a user-specified TSV file
804
+ """
805
+
806
+ log = get_logger()
807
+
808
+ #############################################################################################################################
809
+ # Check for our one required parameter: column.
810
+
811
+ if ( not isinstance( column, str ) ) or column == '':
812
+ log.critical( 'column_values(): ERROR: parameter \'column\' cannot be omitted. Please specify a column from which to fetch a list of distinct values.')
813
+ return
814
+
815
+ # Let's not care about case, and if there's whitespace in our column name, remove it before it does any damage.
816
+ column = re.sub( r'\s+', r'', column ).lower()
817
+
818
+ # See if columns() agrees that the requested column exists. Note: cdapython.columns() and
819
+ # the API's /columns endpoint give different sets of columns, by design. Here we want our output
820
+ # to match the former, because we postprocess some of the columns offered by the API instead of
821
+ # exposing them directly.
822
+
823
+ if len( columns( column=column, return_data_as='list' ) ) == 0:
824
+ log.critical( f"column_values(): ERROR: parameter 'column' must be a searchable CDA column name. You supplied '{column}', which is not." )
825
+ return
826
+
827
+ #############################################################################################################################
828
+ # Check the data_source parameter.
829
+
830
+ if not isinstance( data_source, str ):
831
+ log.error( f"The 'data_source' parameter must be a string (e.g. 'GDC'); you specified '{data_source}', which is not." )
832
+ return
833
+
834
+ # Let's not care about case, and remove any whitespace before it can do any damage.
835
+ data_source = re.sub( r'\s+', r'', data_source ).upper()
836
+
837
+ # TEMPORARY: enumerate valid `data_source` values and warn the user if they supplied something else.
838
+ #
839
+ # This should be replaced ASAP with a fetch from the /release_metadata endpoint.
840
+
841
+ allowed_data_source_values = {
842
+ 'GDC',
843
+ 'PDC',
844
+ 'IDC',
845
+ 'CDS',
846
+ 'ICDC'
847
+ }
848
+
849
+ if data_source != '' and data_source not in allowed_data_source_values:
850
+ log.error( f"The 'data_source' parameter must be one of [ 'GDC', 'PDC', 'IDC', 'CDS', 'ICDC' ]. You supplied '{data_source}', which is not." )
851
+ return
852
+
853
+ #############################################################################################################################
854
+ # Check in advance for columns flagged as high-overhead.
855
+
856
+ expensive_columns = {
857
+ 'file_id',
858
+ 'description',
859
+ 'drs_uri',
860
+ 'file_name',
861
+ 'size',
862
+ 'case_id',
863
+ 'hgnc_id',
864
+ 'transcript_id',
865
+ 'aliquot_barcode_normal',
866
+ 'aliquot_barcode_tumor',
867
+ 'case_barcode',
868
+ 'dbsnp_rs',
869
+ 'entrez_gene_id',
870
+ 'gene',
871
+ 'hugo_symbol',
872
+ 'matched_norm_aliquot_uuid',
873
+ 'normal_submitter_uuid',
874
+ 'reference_allele',
875
+ 'sample_barcode_normal',
876
+ 'sample_barcode_tumor',
877
+ 'tumor_aliquot_uuid',
878
+ 'tumor_seq_allele1',
879
+ 'tumor_seq_allele2',
880
+ 'tumor_submitter_uuid',
881
+ 'subject_id'
882
+ }
883
+
884
+ # Warn the user if an override hasn't been requested.
885
+
886
+ if not force and column in expensive_columns:
887
+ log.warning( f"'{column}' has a very large number of values; retrieval is blocked by default. To perform this query, use column_values( ..., 'force=True' )." )
888
+ return
889
+
890
+ #############################################################################################################################
891
+ # Listify `filters`, if it's a string, so we can process it in a uniform way later on.
892
+
893
+ if filters is None:
894
+ filters = list()
895
+ elif isinstance( filters, str ):
896
+ filters = [ filters ]
897
+
898
+ #############################################################################################################################
899
+ # Process return_data_as and output_file directives.
900
+
901
+ allowed_return_types = {
902
+ '',
903
+ 'dataframe',
904
+ 'tsv',
905
+ 'list'
906
+ }
907
+
908
+ if not isinstance( return_data_as, str ):
909
+ log.critical( f"column_values(): ERROR: unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
910
+ return
911
+
912
+ # Let's not be picky if someone wants to give us return_data_as='DataFrame' or return_data_as='TSV'
913
+ return_data_as = return_data_as.lower()
914
+
915
+ # We can't do much validation on filenames. If `output_file` isn't
916
+ # a locally writeable path, it'll fail when we try to open it for
917
+ # writing. Strip trailing whitespace from both ends and wrap the
918
+ # file-access operation (later, below) in a try{} block.
919
+
920
+ if not isinstance( output_file, str ):
921
+ log.critical( f"column_values(): ERROR: the `output_file` parameter, if not omitted, should be a string containing a path to the desired output file. You supplied '{output_file}', which is not a string, let alone a valid path." )
922
+ return
923
+
924
+ output_file = output_file.strip()
925
+
926
+ if return_data_as not in allowed_return_types:
927
+
928
+ log.critical( f"column_values(): ERROR: unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
929
+ return
930
+
931
+ elif return_data_as == 'tsv' and output_file == '':
932
+
933
+ log.critical( 'column_values(): ERROR: return type \'tsv\' requested, but \'output_file\' not specified. Please specify output_file=\'some/path/string/to/write/your/tsv/to\'.')
934
+ return
935
+
936
+ elif return_data_as != 'tsv' and output_file != '':
937
+
938
+ # If the user put something in the `output_file` parameter but didn't specify `result_data_as='tsv'`,
939
+ # they most likely want their data saved to a file (so ignoring the parameter misconfiguration
940
+ # isn't safe), but ultimately we can't be sure what they meant (so taking an action isn't safe),
941
+ # so we complain and ask them to clarify.
942
+
943
+ log.error( f"'output_file' was specified, but this is only meaningful if 'return_data_as' is set to 'tsv'. You requested return_data_as='{return_data_as}'.\n(Note that if you don't specify any value for 'return_data_as', it defaults to 'dataframe'.)." )
944
+ return
945
+
946
+ #############################################################################################################################
947
+ # Process sorting directives.
948
+
949
+ # Enumerate all allowed values that a user can specify using the `sort_by` parameter. ( 'X:asc' will be aliased immediately to just 'X'. )
950
+
951
+ allowed_sort_by_options = {
952
+ 'list': {
953
+ '',
954
+ 'value',
955
+ 'value:desc'
956
+ },
957
+ 'dataframe_or_tsv': {
958
+ '',
959
+ 'count',
960
+ 'count:desc',
961
+ 'value',
962
+ 'value:desc'
963
+ }
964
+ }
965
+
966
+ if not isinstance( sort_by, str ):
967
+
968
+ # Complain if we receive any unexpected data types instead of string directives.
969
+
970
+ log.critical( f"column_values(): ERROR: 'sort_by' must be a string; you used '{sort_by}', which is not." )
971
+ return
972
+
973
+ # Let's not care about case. Also, ':asc' is redundant: remove it (politely).
974
+ sort_by = re.sub( r':asc$', r'', sort_by ).lower()
975
+
976
+ if return_data_as == 'list':
977
+
978
+ # Restrict sorting options for lists.
979
+
980
+ if sort_by == '':
981
+ sort_by = 'value'
982
+ elif sort_by not in allowed_sort_by_options['list']:
983
+ log.critical( f"column_values(): ERROR: return_data_as='list' can only be processed with sort_by='value' or sort_by='value:desc' (or omitting sort_by altogether). Please modify unsupported sort_by directive '{sort_by}' and try again." )
984
+ return
985
+
986
+ else:
987
+
988
+ # For TSV output files and DataFrames, we support more user-configurable options (defaulting to sort_by='count:desc'):
989
+
990
+ if sort_by == '':
991
+ sort_by = 'count:desc'
992
+ elif sort_by not in allowed_sort_by_options['dataframe_or_tsv']:
993
+ log.critical( f"column_values(): ERROR: unrecognized sort_by '{sort_by}'. Please use one of 'count', 'value', 'count:desc', 'value:desc', 'count:asc' or 'value:asc' (or omit the sort_by parameter altogether)." )
994
+ return
995
+
996
+ #############################################################################################################################
997
+ # Report details of the final parsed parameters.
998
+
999
+ parameter_dict = {
1000
+ 'column': column,
1001
+ 'return_data_as': return_data_as,
1002
+ 'output_file': output_file,
1003
+ 'sort_by': sort_by,
1004
+ 'filters': filters,
1005
+ 'data_source': data_source,
1006
+ 'force': force,
1007
+ }
1008
+
1009
+ log.debug( f"Processed all parameter directives. Calling API to fetch data for\n{json.dumps( parameter_dict, indent=4 )}\n" )
1010
+
1011
+ #############################################################################################################################
1012
+ # Fetch data from the API.
1013
+
1014
+ query_api_instance = cda_client.Client( base_url=get_api_url(), raise_on_unexpected_status=True )
1015
+
1016
+ starting_offset = 0
1017
+ records_per_page = 500000
1018
+
1019
+ try:
1020
+ paged_response_data_object = (
1021
+ cda_client.api.column_values.column_values_endpoint_column_values_column_post.sync(
1022
+ client=query_api_instance,
1023
+ column=column,
1024
+ data_source=data_source,
1025
+ limit=records_per_page,
1026
+ offset=starting_offset
1027
+ )
1028
+ )
1029
+ except UnexpectedStatus as error:
1030
+ log.error( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
1031
+ return
1032
+ except Exception as error:
1033
+ log.error( f"{type(error)}: {error}" )
1034
+ return
1035
+
1036
+ log.debug( f"Sending query to API:\n{json.dumps( { 'columnname': column, 'system': data_source, 'count': True, 'total_count': True, 'limit': records_per_page, 'offset': starting_offset }, indent=4 )}\n" )
1037
+
1038
+ # Report some metadata about the results we got back.
1039
+
1040
+ log.debug( f"Number of result rows: {paged_response_data_object.total_row_count}" )
1041
+
1042
+ log.debug( f"Query SQL: '{paged_response_data_object.query_sql}'" )
1043
+
1044
+ # Make a Pandas DataFrame out of the first batch of results.
1045
+ #
1046
+ # The API returns responses in JSON format: convert that JSON into a DataFrame
1047
+ # using pandas' json_normalize() function. Example JSON response:
1048
+ #
1049
+ # {
1050
+ # "result": [
1051
+ # {
1052
+ # "sex": "female",
1053
+ # "value_count": 31215
1054
+ # },
1055
+ # {
1056
+ # "sex": "male",
1057
+ # "value_count": 28571
1058
+ # },
1059
+ # {
1060
+ # "sex": null,
1061
+ # "value_count": 64240
1062
+ # }
1063
+ # ],
1064
+ # "query_sql": "SELECT row_to_json(column_json) AS row_to_json_1 FROM (SELECT observation.sex AS sex, count(*) AS value_count FROM observation WHERE observation.data_at_gdc IS true GROUP BY observation.sex ORDER BY observation.sex) AS column_json",
1065
+ # "total_row_count": 3,
1066
+ # "next_url": null
1067
+ # }
1068
+
1069
+ log.debug( f"Page one results ( NOTE: pattern filters have not yet been applied ):\n{json.dumps( paged_response_data_object.to_dict(), indent=4 )}\n" )
1070
+
1071
+ result_dataframe = pd.json_normalize( paged_response_data_object.to_dict()['result'] )
1072
+
1073
+ # The data we've fetched so far might be just the first page (if the total number
1074
+ # of results is greater than `records_per_page`).
1075
+ #
1076
+ # Get the rest of the result pages, if there are any, and add each page's data
1077
+ # onto the end of our results DataFrame.
1078
+
1079
+ incremented_offset = starting_offset + records_per_page
1080
+
1081
+ more_than_one_result_page = False
1082
+
1083
+ if paged_response_data_object.next_url is not None:
1084
+ log.debug( 'Fetching remaining results in pages...' )
1085
+ more_than_one_result_page = True
1086
+
1087
+ while paged_response_data_object.next_url is not None and len( paged_response_data_object.next_url ) > 0:
1088
+
1089
+ log.debug( f" ...fetching {paged_response_data_object.next_url}..." )
1090
+
1091
+ try:
1092
+ paged_response_data_object = (
1093
+ cda_client.api.column_values.column_values_endpoint_column_values_column_post.sync(
1094
+ client=query_api_instance,
1095
+ column=column,
1096
+ data_source=data_source,
1097
+ limit=records_per_page,
1098
+ offset=incremented_offset
1099
+ )
1100
+ )
1101
+ except UnexpectedStatus as error:
1102
+ log.error( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
1103
+ return
1104
+ except Exception as error:
1105
+ log.error( f"{type(error)}: {error}" )
1106
+ return
1107
+
1108
+ next_result_batch = pd.json_normalize( paged_response_data_object.to_dict()['result'] )
1109
+
1110
+ if not result_dataframe.empty and not next_result_batch.empty:
1111
+
1112
+ # Silence a future deprecation warning about pd.concat and empty DataFrame columns.
1113
+
1114
+ next_result_batch = next_result_batch.astype( result_dataframe.dtypes )
1115
+
1116
+ result_dataframe = pd.concat( [result_dataframe, next_result_batch] )
1117
+
1118
+ incremented_offset = incremented_offset + records_per_page
1119
+
1120
+ if more_than_one_result_page:
1121
+ log.debug( '...done.' )
1122
+
1123
+ #############################################################################################################################
1124
+ # Postprocess API result data, if there is any.
1125
+
1126
+ if len( result_dataframe ) == 0:
1127
+ return result_dataframe
1128
+
1129
+ log.debug( 'Postprocessing results' )
1130
+
1131
+ log.debug( 'Casting counts to integers and fixing symmetry for returned column labels...' )
1132
+
1133
+ # Sanity check on expected result column 'value_count':
1134
+
1135
+ if 'value_count' not in result_dataframe.columns:
1136
+ log.error( 'Expected column \'value_count\' not present in API response.' )
1137
+ return
1138
+
1139
+ # Term-count values come in as floats. Make them not that.
1140
+
1141
+ result_dataframe['value_count'] = result_dataframe['value_count'].astype( int )
1142
+
1143
+ log.debug( 'Handling missing values...' )
1144
+
1145
+ # CDA has no float values. If the API gives us some, cast them to integers.
1146
+
1147
+ if result_dataframe[column].dtype == 'float64':
1148
+
1149
+ # Columns of type `float64` can contain NaN (missing) values, which cannot (for some reason)
1150
+ # be stored in Pandas Series objects (i.e., DataFrame columns) of type `int` or `int64`.
1151
+ # Pandas workaround: use extension type 'Int64' (note initial capital), which supports the
1152
+ # storage of missing values. These will print as '<NA>'.
1153
+
1154
+ result_dataframe[column] = result_dataframe[column].round().astype( 'Int64' )
1155
+
1156
+ elif result_dataframe[column].dtype == 'object':
1157
+
1158
+ # String data comes through as a column with dtype 'object', based on something involving
1159
+ # the variability inherent in string lengths.
1160
+ #
1161
+ # See https://stackoverflow.com/questions/33957720/how-to-convert-column-with-dtype-as-object-to-string-in-pandas-dataframe
1162
+
1163
+ # Replace term values that are None (== null) with empty strings.
1164
+
1165
+ result_dataframe = result_dataframe.fillna( '' )
1166
+
1167
+ elif result_dataframe[column].dtype == 'bool':
1168
+
1169
+ result_dataframe = result_dataframe.fillna( '' )
1170
+
1171
+ else:
1172
+
1173
+ # This isn't anticipated. Yell if we get something unexpected.
1174
+
1175
+ log.critical( f"column_values(): ERROR: Unexpected data type `{result_dataframe[column].dtype}` received; aborting. Please report this event to the CDA development team." )
1176
+ return
1177
+
1178
+ #############################################################################################################################
1179
+ # Filter returned values according to user specifications.
1180
+
1181
+ # Default behavior: all result values must be exact matches to at least one
1182
+ # filter (ignoring case). To match end-to-end, we use a ^ to represent
1183
+ # the beginning of each value and a $ to indicate the end. If the user
1184
+ # specifies wildcards on one or both ends of a filter, we'll remove one or both
1185
+ # restrictions as instructed for that filter.
1186
+
1187
+ match_pattern_string = ''
1188
+
1189
+ # If the user includes an empty string in the filters list, make sure we return
1190
+ # a count for empty (null) values in addition to any values matching other filters.
1191
+
1192
+ include_null_count = False
1193
+
1194
+ for filter_pattern in filters:
1195
+
1196
+ if filter_pattern == '':
1197
+
1198
+ include_null_count = True
1199
+
1200
+ else:
1201
+
1202
+ # Process wildcard characters.
1203
+
1204
+ if re.search( r'^\*', filter_pattern ) is not None:
1205
+
1206
+ # Any prefix will do, now.
1207
+ #
1208
+ # Strip leading '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
1209
+
1210
+ filter_pattern = re.sub( r'^\*+', r'', filter_pattern )
1211
+
1212
+ else:
1213
+
1214
+ # No wildcard at the beginning of `filter_pattern` --> require all successful matches to _begin_ with `filter_pattern` by prepending a ^ character to `filter_pattern`:
1215
+ #
1216
+ # ...I know this looks weird, but it's just tacking a '^' character onto the beginning of `filter_pattern`.
1217
+
1218
+ filter_pattern = re.sub( r'^', r'^', filter_pattern )
1219
+
1220
+ if re.search( r'\*$', filter_pattern ) is not None:
1221
+
1222
+ # Any suffix will do, now.
1223
+ #
1224
+ # Strip trailing '*' characters off of `filter_pattern` so we don't confuse the downstream matching function.
1225
+
1226
+ filter_pattern = re.sub( r'\*+$', r'', filter_pattern )
1227
+
1228
+ else:
1229
+
1230
+ # No wildcard at the end of `filter_pattern` --> require all successful matches to _end_ with `filter_pattern` by appending a '$' character to `filter_pattern`:
1231
+ #
1232
+ # ...I know this looks weird, but it's just tacking a '$' character onto the end of `filter_pattern`.
1233
+
1234
+ filter_pattern = re.sub( r'$', r'$', filter_pattern )
1235
+
1236
+ # Build the overall match pattern as we go, one (processed) `filter_pattern` at a time.
1237
+
1238
+ match_pattern_string = match_pattern_string + filter_pattern + '|'
1239
+
1240
+ # Strip the final trailing '|' character from the end of the last `filter_pattern`.
1241
+
1242
+ match_pattern_string = re.sub( r'\|$', r'', match_pattern_string )
1243
+
1244
+ print_regex = match_pattern_string
1245
+
1246
+ if include_null_count:
1247
+ if print_regex == '':
1248
+ print_regex = '(missing values)'
1249
+ else:
1250
+ print_regex = print_regex + '|(missing values)'
1251
+
1252
+ if print_regex == '':
1253
+ print_regex = '(none)'
1254
+ else:
1255
+ print_regex = f"/{print_regex}/"
1256
+
1257
+ log.debug( f"Applying pattern filters: {print_regex}" )
1258
+
1259
+ # Filter results to match the full aggregated regular expression in `match_pattern_string`.
1260
+
1261
+ if include_null_count and match_pattern_string != '':
1262
+
1263
+ result_dataframe = result_dataframe.loc[
1264
+ result_dataframe[column].astype( str ).str.contains( match_pattern_string, case=False )
1265
+ | result_dataframe[column].astype( str ).str.contains( r'^$' )
1266
+ | result_dataframe[column].isna()
1267
+ ]
1268
+
1269
+ elif include_null_count:
1270
+
1271
+ result_dataframe = result_dataframe.loc[ result_dataframe[column].astype( str ).str.contains( r'^$' ) | result_dataframe[column].isna() ]
1272
+
1273
+ else:
1274
+
1275
+ # This will return unfiltered results if `match_pattern_string` is empty (i.e. if the user asked for no filters to be applied),
1276
+ # and will filter results according to `match_pattern_string` if not.
1277
+
1278
+ result_dataframe = result_dataframe.loc[ result_dataframe[column].astype( str ).str.contains( match_pattern_string, case=False ) ]
1279
+
1280
+ # Sort results. Default (note that the final value of `sort_by` is determined earlier in this function) is to sort by term count, descending.
1281
+
1282
+ log.debug( f"Applying sort directive '{sort_by}'..." )
1283
+
1284
+ if sort_by == 'count':
1285
+
1286
+ # Sort by count; break ties among groups of values with identical counts by sub-sorting each such group alphabetically by value.
1287
+ result_dataframe = result_dataframe.sort_values( by=['value_count', column], ascending=[True, True] )
1288
+
1289
+ elif sort_by == 'count:desc':
1290
+
1291
+ # Sort by count, descending; break ties among groups of values with identical counts by sub-sorting each such group alphabetically by value.
1292
+ result_dataframe = result_dataframe.sort_values( by=['value_count', column], ascending=[False, True] )
1293
+
1294
+ elif sort_by == 'value':
1295
+
1296
+ # No need for a sub-sort, here, since values aren't repeated.
1297
+ result_dataframe = result_dataframe.sort_values( by=column, ascending=True )
1298
+
1299
+ elif sort_by == 'value:desc':
1300
+
1301
+ # No need for a sub-sort, here, since values aren't repeated.
1302
+ result_dataframe = result_dataframe.sort_values( by=column, ascending=False )
1303
+
1304
+ else:
1305
+
1306
+ log.error( 'Something has gone horribly wrong; we should never get here.' )
1307
+ return
1308
+
1309
+ #############################################################################################################################
1310
+ # Send the results back to the user.
1311
+
1312
+ # Reindex DataFrame rows to match their final sort order.
1313
+
1314
+ result_dataframe = result_dataframe.reset_index( drop=True )
1315
+
1316
+ # Pretty-print missing values.
1317
+
1318
+ if result_dataframe[column].dtype == 'object':
1319
+
1320
+ # String data comes through as a column with dtype 'object', based on something involving
1321
+ # the variability inherent in string lengths.
1322
+ #
1323
+ # See https://stackoverflow.com/questions/33957720/how-to-convert-column-with-dtype-as-object-to-string-in-pandas-dataframe
1324
+
1325
+ # Replace null string values with <NA> to match what we['re forced to] use for numeric data.
1326
+
1327
+ result_dataframe = result_dataframe.replace( r'^$', r'<NA>', regex=True )
1328
+
1329
+ elif result_dataframe[column].dtype == 'bool':
1330
+
1331
+ # Replace null boolean values with <NA> to match what we['re forced to] use for numeric data.
1332
+ result_dataframe = result_dataframe.replace(r'^$', r'<NA>', regex=True)
1333
+
1334
+ if return_data_as == '':
1335
+
1336
+ # Right now, the default is the same as if the user had
1337
+ # specified return_data_as='dataframe'.
1338
+
1339
+ # The following, for the dubiously useful record, is a somewhat worse alternative default thing to do.
1340
+ #
1341
+ # print( result_dataframe.to_string( index=False, justify='right', max_rows=25, max_colwidth=50 ), file=sys.stdout )
1342
+
1343
+ log.debug( 'Returning results in default form (pandas.DataFrame)' )
1344
+
1345
+ return result_dataframe
1346
+
1347
+ elif return_data_as == 'dataframe':
1348
+
1349
+ # Give the user back the results DataFrame.
1350
+
1351
+ log.debug( 'Returning results as pandas.DataFrame' )
1352
+ return result_dataframe
1353
+
1354
+ elif return_data_as == 'list':
1355
+
1356
+ # Strip the term-values column out of the results DataFrame and give them to the user as a Python list.
1357
+
1358
+ log.debug( 'Returning results as list of column values' )
1359
+ return result_dataframe[column].to_list()
1360
+
1361
+ else:
1362
+
1363
+ # Write the results DataFrame to a user-specified TSV file.
1364
+
1365
+ log.debug( f"Printing results to TSV file '{output_file}'" )
1366
+
1367
+ try:
1368
+ result_dataframe.to_csv( output_file, sep='\t', index=False )
1369
+ return
1370
+
1371
+ except Exception as error:
1372
+ raise RuntimeError( f"Couldn't write to requested output file '{output_file}': got error of type '{type(error)}', with error message '{error}'." )
1373
+
1374
+ #############################################################################################################################
1375
+ #
1376
+ # END column_values()
1377
+ #
1378
+ #############################################################################################################################
1379
+
1380
+ #############################################################################################################################
1381
+ #
1382
+ # release_metadata(): Return a list of metadata dicts describing the current CDA release, columnwise.
1383
+ #
1384
+ #############################################################################################################################
1385
+
1386
+ def release_metadata():
1387
+ """
1388
+ Return a list of metadata dicts describing the current CDA release, columnwise.
1389
+ """
1390
+
1391
+ log = get_logger()
1392
+
1393
+ log.debug( 'Querying /release_metadata endpoint' )
1394
+
1395
+ query_api_instance = cda_client.Client( base_url=get_api_url(), raise_on_unexpected_status=True )
1396
+
1397
+ try:
1398
+ release_metadata_response_data_object = cda_client.api.release_metadata.release_metadata_endpoint_release_metadata_get.sync( client=query_api_instance )
1399
+ except UnexpectedStatus as error:
1400
+ raise RuntimeError( f"UnexpectedStatus error from API, status code {error.status_code}: {json.loads( error.content )['message']}" )
1401
+ except Exception as error:
1402
+ raise RuntimeError( f"{type(error)}: {error}" )
1403
+
1404
+ return release_metadata_response_data_object.to_dict()['result']
1405
+
1406
+ #############################################################################################################################
1407
+ #
1408
+ # END release_metadata()
1409
+ #
1410
+ #############################################################################################################################
1411
+
1412
+