cdapython 2.0.8__tar.gz → 2.0.10__tar.gz

{cdapython-2.0.8 → cdapython-2.0.10}/CHANGELOG.md

@@ -1,3 +1,7 @@
+2025-10-27: 2.0.10: integrate column\_values() parameter validation with shared validation library; expand support for column\_values( data\_source ) to accept list values in addition to single strings
+
+2025-10-22: 2.0.9: parametrize data\_source check in column\_values()
+
 2025-10-20: 2.0.8: rename CDS to GC throughout
 
 2025-10-08: 2.0.7: fix malformed column metadata selector inside `collate=True` provenance management in get\_data()

{cdapython-2.0.8 → cdapython-2.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cdapython
-Version: 2.0.8
+Version: 2.0.10
 Summary: NCI CRDC CDA front-end user interface library
 License-File: LICENSE
 Requires-Python: >=3.9,<4.0

{cdapython-2.0.8 → cdapython-2.0.10}/cdapython/discover.py

@@ -48,10 +48,11 @@ def cda_functions():
             'summarize_files',
             'get_subject_data',
             'get_file_data',
-            'intersect_subject_results',
-            'intersect_file_results',
             'expand_subject_results',
             'expand_file_results',
+            'intersect_subject_results',
+            'intersect_file_results',
+            'release_metadata',
             'get_valid_log_levels',
             'get_log_level',
             'set_log_level',
@@ -737,7 +738,7 @@ def column_values(
     output_file='',
     sort_by='',
     filters=None,
-    data_source='',
+    data_source=None,
     force=False
 ):
     """
@@ -789,7 +790,7 @@ def column_values(
         data_source ( string; optional ):
             Restrict returned values to the given upstream data source. Current
             valid values are 'GC', 'GDC', 'PDC', 'IDC' and 'ICDC'.
-            Defaults to '' (no filter).
+            Defaults to None (no filter).
 
         force( boolean; optional ):
             Force execution of high-overhead queries on columns (like IDs)
@@ -806,57 +807,93 @@ def column_values(
     log = get_logger()
 
     #############################################################################################################################
-    # Check for our one required parameter: column.
-
-    if ( not isinstance( column, str ) ) or column == '':
-        log.critical( 'column_values(): ERROR: parameter \'column\' cannot be omitted. Please specify a column from which to fetch a list of distinct values.')
-        return
+    # Validate parameter inputs.
+    #############################################################################################################################
 
-    # Let's not care about case, and if there's whitespace in our column name, remove it before it does any damage.
-    column = re.sub( r'\s+', r'', column ).lower()
+    # Normalize user-supplied parameter data so we can assume from here on out that these are always lists of values:
+    # convert any of the following that come in as single values (instead of lists of values) into one-element lists,
+    # and leave the rest unmodified. Also convert everything to lowercase.
 
-    # See if columns() agrees that the requested column exists. Note: cdapython.columns() and
-    # the API's /columns endpoint give different sets of columns, by design. Here we want our output
-    # to match the former, because we postprocess some of the columns offered by the API instead of
-    # exposing them directly.
+    # If someone can devise a way to do this with a control loop, I'm all ears. I gave up after 20 minutes
+    # of fiddling with `locals()`.
 
-    if len( columns( column=column, return_data_as='list' ) ) == 0:
-        log.critical( f"column_values(): ERROR: parameter 'column' must be a searchable CDA column name. You supplied '{column}', which is not." )
+    try:
+        filters = normalize_to_list( 'filters', filters, str )
+        data_source = normalize_to_list( 'data_source', data_source, str )
+    except Exception as e:
+        log.error( e )
         return
 
-    #############################################################################################################################
-    # Check the data_source parameter.
+    # Cache CDA table and column metadata from the API for downstream reuse without further
+    # network disturbance. The data structure coming back from columns() is a DataFrame
+    # with columns [ 'table', 'column', 'data_type', 'nullable', 'description' ].
 
-    if not isinstance( data_source, str ):
-        log.error( f"The 'data_source' parameter must be a string (e.g. 'GDC'); you specified '{data_source}', which is not." )
+    cached_column_metadata = columns()
+
+    # Cache valid labels for upstream data sources. The data structure coming back from
+    # release_metadata() is a list of dicts, with each dict looking like
+    # 
+    # {
+    #     'cda_table': 'file',
+    #     'cda_column': 'access',
+    #     'data_source': 'GDC',
+    #     'data_source_version': 'March 2025',
+    #     'data_source_extraction_date': '2025-03-21',
+    #     'data_source_row_count': 3025352,
+    #     'data_source_unique_value_count': 4,
+    #     'data_source_null_count': 407714
+    # }
+
+    try:
+        cached_release_metadata = release_metadata()
+    except Exception as e:
+        log.critical( e )
         return
 
-    # Let's not care about case, and remove any whitespace before it can do any damage.
-    data_source = re.sub( r'\s+', r'', data_source ).upper()
+    valid_data_sources = set()
 
-    # TEMPORARY: enumerate valid `data_source` values and warn the user if they supplied something else.
-    #
-    # This should be replaced ASAP with a fetch from the /release_metadata endpoint.
-
-    allowed_data_source_values = {
-        'GC',
-        'GDC',
-        'PDC',
-        'IDC',
-        'ICDC'
-    }
+    for column_record in cached_release_metadata:
+        record_data_source = column_record['data_source']
+        # Let's not care about case.
+        if record_data_source.upper() != 'CDA':
+            valid_data_sources.add( record_data_source.upper() )
 
-    if data_source != '' and data_source not in allowed_data_source_values:
-        log.error( f"The 'data_source' parameter must be one of [ 'GC', 'GDC', 'PDC', 'IDC', 'ICDC' ]. You supplied '{data_source}', which is not." )
+    # Validate user-supplied parameter data.
+
+    try:
+        validate_parameter_values(
+            called_function='column_values',
+            cached_column_metadata=cached_column_metadata,
+            valid_data_sources=valid_data_sources,
+            table=None,
+            column=column,
+            match_from_file=None,
+            data_source=data_source,
+            add_columns=None,
+            exclude_columns=None,
+            collate_results=None,
+            include_external_refs=None,
+            return_data_as=return_data_as,
+            output_file=output_file,
+            sort_by=sort_by,
+            log=log
+        )
+    except Exception as e:
+        log.error( e )
         return
 
+    # Normalize sort_by case. Also, ':asc' is redundant: remove it (politely).
+    sort_by = re.sub( r':asc$', r'', sort_by ).lower()
+
     #############################################################################################################################
     # Check in advance for columns flagged as high-overhead.
 
+    # TO DO: Maybe store this somewhere better than here.
+
     expensive_columns = {
         'file_id',
-        'description',
         'drs_uri',
+        'file_description',
         'file_name',
         'size',
         'case_id',
@@ -878,121 +915,20 @@ def column_values(
         'tumor_seq_allele1',
         'tumor_seq_allele2',
         'tumor_submitter_uuid',
+        'upstream_id',
         'subject_id'
     }
 
-    # Warn the user if an override hasn't been requested.
+    # Make sure the `force` parameter is a Boolean.
+    if not isinstance( force, bool ):
+        log.warning( f"The `force` parameter must be True or False; you supplied {force}, which is neither." )
+        return
 
+    # Warn the user if an override hasn't been requested.
     if not force and column in expensive_columns:
         log.warning( f"'{column}' has a very large number of values; retrieval is blocked by default. To perform this query, use column_values( ..., 'force=True' )." )
         return
 
-    #############################################################################################################################
-    # Listify `filters`, if it's a string, so we can process it in a uniform way later on.
-
-    if filters is None:
-        filters = list()
-    elif isinstance( filters, str ):
-        filters = [ filters ]
-
-    #############################################################################################################################
-    # Process return_data_as and output_file directives.
-
-    allowed_return_types = {
-        '',
-        'dataframe',
-        'tsv',
-        'list'
-    }
-
-    if not isinstance( return_data_as, str ):
-        log.critical( f"column_values(): ERROR: unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
-        return
-
-    # Let's not be picky if someone wants to give us return_data_as='DataFrame' or return_data_as='TSV'
-    return_data_as = return_data_as.lower()
-
-    # We can't do much validation on filenames. If `output_file` isn't
-    # a locally writeable path, it'll fail when we try to open it for
-    # writing. Strip trailing whitespace from both ends and wrap the
-    # file-access operation (later, below) in a try{} block.
-
-    if not isinstance( output_file, str ):
-        log.critical( f"column_values(): ERROR: the `output_file` parameter, if not omitted, should be a string containing a path to the desired output file. You supplied '{output_file}', which is not a string, let alone a valid path." )
-        return
-
-    output_file = output_file.strip()
-
-    if return_data_as not in allowed_return_types:
-        
-        log.critical( f"column_values(): ERROR: unrecognized return type '{return_data_as}' requested. Please use one of 'dataframe', 'list' or 'tsv'." )
-        return
-
-    elif return_data_as == 'tsv' and output_file == '':
-        
-        log.critical( 'column_values(): ERROR: return type \'tsv\' requested, but \'output_file\' not specified. Please specify output_file=\'some/path/string/to/write/your/tsv/to\'.')
-        return
-
-    elif return_data_as != 'tsv' and output_file != '':
-        
-        # If the user put something in the `output_file` parameter but didn't specify `result_data_as='tsv'`,
-        # they most likely want their data saved to a file (so ignoring the parameter misconfiguration
-        # isn't safe), but ultimately we can't be sure what they meant (so taking an action isn't safe),
-        # so we complain and ask them to clarify.
-
-        log.error( f"'output_file' was specified, but this is only meaningful if 'return_data_as' is set to 'tsv'. You requested return_data_as='{return_data_as}'.\n(Note that if you don't specify any value for 'return_data_as', it defaults to 'dataframe'.)." )
-        return
-
-    #############################################################################################################################
-    # Process sorting directives.
-
-    # Enumerate all allowed values that a user can specify using the `sort_by` parameter. ( 'X:asc' will be aliased immediately to just 'X'. )
-
-    allowed_sort_by_options = {
-        'list': {
-            '',
-            'value',
-            'value:desc'
-        },
-        'dataframe_or_tsv': {
-            '',
-            'count',
-            'count:desc',
-            'value',
-            'value:desc'
-        }
-    }
-
-    if not isinstance( sort_by, str ):
-        
-        # Complain if we receive any unexpected data types instead of string directives.
-
-        log.critical( f"column_values(): ERROR: 'sort_by' must be a string; you used '{sort_by}', which is not." )
-        return
-
-    # Let's not care about case. Also, ':asc' is redundant: remove it (politely).
-    sort_by = re.sub( r':asc$', r'', sort_by ).lower()
-
-    if return_data_as == 'list':
-        
-        # Restrict sorting options for lists.
-
-        if sort_by == '':
-            sort_by = 'value'
-        elif sort_by not in allowed_sort_by_options['list']:
-            log.critical( f"column_values(): ERROR: return_data_as='list' can only be processed with sort_by='value' or sort_by='value:desc' (or omitting sort_by altogether). Please modify unsupported sort_by directive '{sort_by}' and try again." )
-            return
-
-    else:
-        
-        # For TSV output files and DataFrames, we support more user-configurable options (defaulting to sort_by='count:desc'):
-
-        if sort_by == '':
-            sort_by = 'count:desc'
-        elif sort_by not in allowed_sort_by_options['dataframe_or_tsv']:
-            log.critical( f"column_values(): ERROR: unrecognized sort_by '{sort_by}'. Please use one of 'count', 'value', 'count:desc', 'value:desc', 'count:asc' or 'value:asc' (or omit the sort_by parameter altogether)." )
-            return
-
     #############################################################################################################################
     # Report details of the final parsed parameters.
 
@@ -1021,7 +957,8 @@ def column_values(
             cda_client.api.column_values.column_values_endpoint_column_values_column_post.sync(
                 client=query_api_instance,
                 column=column,
-                data_source=data_source,
+                # This is how the API expects the data_source parameter to be encoded.
+                data_source=', '.join( data_source ),
                 limit=records_per_page,
                 offset=starting_offset
             )
@@ -1033,7 +970,7 @@ def column_values(
         log.error( f"{type(error)}: {error}" )
         return
 
-    log.debug( f"Sending query to API:\n{json.dumps( { 'columnname': column, 'system': data_source, 'count': True, 'total_count': True, 'limit': records_per_page, 'offset': starting_offset }, indent=4 )}\n" )
+    log.debug( f"Sending query to API:\n{json.dumps( { 'column': column, 'data_source': ', '.join( data_source ), 'limit': records_per_page, 'offset': starting_offset }, indent=4 )}\n" )
 
     # Report some metadata about the results we got back.
 
@@ -1093,7 +1030,7 @@ def column_values(
                 cda_client.api.column_values.column_values_endpoint_column_values_column_post.sync(
                     client=query_api_instance,
                     column=column,
-                    data_source=data_source,
+                    data_source=', '.join( data_source ),
                     limit=records_per_page,
                     offset=incremented_offset
                 )
@@ -1409,4 +1346,7 @@ def release_metadata():
 #
 #############################################################################################################################
 
+# This is down here to avoid a circular import problem.
+from cdapython.validation import normalize_to_list, validate_parameter_values
+
 

{cdapython-2.0.8 → cdapython-2.0.10}/cdapython/get_data.py

@@ -457,6 +457,7 @@ def get_data(
             cached_column_metadata=cached_column_metadata,
             valid_data_sources=valid_data_sources,
             table=table,
+            column=None,
             match_from_file=match_from_file,
             data_source=data_source,
             add_columns=add_columns,
@@ -465,6 +466,7 @@ def get_data(
             include_external_refs=include_external_refs,
             return_data_as=return_data_as,
             output_file=output_file,
+            sort_by=None,
             log=log
         )
     except Exception as e:

{cdapython-2.0.8 → cdapython-2.0.10}/cdapython/summarize.py

@@ -587,14 +587,14 @@ def summarize_files(
         Users can require partial matches to string VALUEs by adding * to either or
         both ends. For example:
 
-            primary_disease_type = *duct*
+            diagnosis = *duct*
             sex = F*
             size < 100
 
         String VALUEs need not be quoted inside of filter strings. For example, to include
         the filters specified just above in the `match_all` argument, we can write:
 
-            summarize_files( match_all=[ 'primary_disease_type = *duct*', 'sex = F*', 'size < 100' ] )
+            summarize_files( match_all=[ 'diagnosis = *duct*', 'sex = F*', 'size < 100' ] )
 
         NULL is a special VALUE which can be used to match missing data. For
         example, to get a summary report for CDA files where the `access` field
@@ -728,14 +728,14 @@ def summarize_subjects(
         Users can require partial matches to string VALUEs by adding * to either or
         both ends. For example:
 
-            primary_disease_type = *duct*
+            diagnosis = *duct*
             sex = F*
             size < 100
 
         String VALUEs need not be quoted inside of filter strings. For example, to include
         the filters specified just above in the `match_all` argument, we can write:
 
-            summarize_subjects( match_all=[ 'primary_disease_type = *duct*', 'sex = F*' ] )
+            summarize_subjects( match_all=[ 'diagnosis = *duct*', 'sex = F*' ] )
 
         NULL is a special VALUE which can be used to match missing data. For
         example, to get a summary report for CDA subjects where the `year_of_birth` field
@@ -876,13 +876,13 @@ def summarize(
         Users can require partial matches to string VALUEs by adding * to either or
         both ends. For example:
 
-            primary_disease_type = *duct*
+            diagnosis = *duct*
             sex = F*
 
         String VALUEs need not be quoted inside of filter strings. For example, to include
         the filters specified just above in the `match_all` argument, we can write:
 
-            summarize( table='subject', match_all=[ 'primary_disease_type = *duct*', 'sex = F*' ] )
+            summarize( table='subject', match_all=[ 'diagnosis = *duct*', 'sex = F*' ] )
 
         NULL is a special VALUE which can be used to match missing data. For
         example, to get a count summary for rows where the `sex` field is missing data,
@@ -993,6 +993,7 @@ def summarize(
             cached_column_metadata=cached_column_metadata,
             valid_data_sources=valid_data_sources,
             table=table,
+            column=None,
             match_from_file=match_from_file,
             data_source=data_source,
             add_columns=add_columns,
@@ -1001,6 +1002,7 @@ def summarize(
             include_external_refs=None,
             return_data_as=return_data_as,
             output_file=output_file,
+            sort_by=None,
             log=log
         )
     except Exception as e:

{cdapython-2.0.8 → cdapython-2.0.10}/cdapython/validation.py

@@ -1,8 +1,6 @@
 import re
 import sys
 
-from cdapython.discover import tables
-
 #############################################################################################################################
 #
 # normalize_to_list( parameter_name, user_supplied_parameter_value, value_type ):
@@ -406,7 +404,8 @@ def validate_and_transform_match_from_file_values( cda_column_to_match, target_d
 # Fail if:
 # 
 #     * `called_function` is not in [ 'column_values', 'get_data', 'summarize' ]
-#     * `table` is not a CDA table
+#     * `table` is not a CDA table (unless `called_function` is 'column_values', in which case `table` must be None)
+#     * `column` is not a CDA column (unless `called_function` is 'get_data' or 'summarize', in which case `column` must be None)
 #     * `match_from_file` isn't a dict which (if non-null) specifies an accessible input file
 #       containing a user-specified column, whose values are to be matched against a CDA column
 #       that exists
@@ -417,6 +416,7 @@ def validate_and_transform_match_from_file_values( cda_column_to_match, target_d
 #     * `include_external_refs` isn't a boolean value or None, depending on `called_function`
 #     * `return_data_as` isn't one of the allowable types for `called_function`
 #     * The value of `output_file` isn't consistent with the directive in `return_data_as` for `called_function`
+#     * `sort_by` isn't an allowable string value (or None, for callers get_data() and summarize())
 #
 #############################################################################################################################
 
@@ -425,6 +425,7 @@ def validate_parameter_values(
     cached_column_metadata,
     valid_data_sources,
     table,
+    column,
     match_from_file,
     data_source,
     add_columns,
@@ -433,39 +434,44 @@ def validate_parameter_values(
     include_external_refs,
     return_data_as,
     output_file,
+    sort_by,
     log
 ):
     # The data structure coming back from columns() is a DataFrame with columns [ 'table', 'column', 'data_type', 'nullable', 'description' ].
 
-    # Make sure `table` exists.
-
-    if table is None or not isinstance( table, str ) or table not in cached_column_metadata['table'].unique():
-        raise RuntimeError( f"The required parameter 'table' must be a searchable CDA table; you supplied '{table}', which is not. Please run tables() for a list." )
+    # Make sure `table` exists, unless we're called by column_values().
+    if called_function in [ 'column_values' ]:
+        if table is not None:
+            raise RuntimeError( f"'table' cannot be non-null for column_values() caller: please alert the CDA devs to this event, something is misconfigured in our code." )
+    elif called_function in [ 'get_data', 'summarize' ]:
+        if table is None or not isinstance( table, str ) or table not in cached_column_metadata['table'].unique():
+            raise RuntimeError( f"The required parameter 'table' must be a searchable CDA table; you supplied '{table}', which is not. Please run tables() for a list." )
 
+    # Make sure `column` exists, unless we're called by get_data() or summarize().
     if called_function in [ 'get_data', 'summarize' ]:
-        
-        # Check `match_from_file` data for sanity.
+        if column is not None:
+            raise RuntimeError( f"'column' cannot be non-null for get_data() or summarize() caller: please alert the CDA devs to this event, something is misconfigured in our code." )
+    elif called_function in [ 'column_values' ]:
+        if column is None or not isinstance( column, str ) or column not in cached_column_metadata['column'].unique():
+            raise RuntimeError( f"The required parameter 'column' must be a searchable CDA column; you supplied '{column}', which is not. Please run columns() for a list." )
 
+    # Check `match_from_file` data for sanity.
+    if called_function in [ 'get_data', 'summarize' ]:
         # Is `match_from_file` a dict with the expected set of keys?
         if not isinstance( match_from_file, dict ) or set( match_from_file.keys() ) != { 'input_file', 'input_column', 'cda_column_to_match' }:
             raise RuntimeError( f"'match_from_file' must be a 3-element dictionary with keys ['input_file', 'input_column', 'cda_column_to_match']; you specified '{match_from_file}', which is not." )
-
         # Does `match_from_file`['cda_column_to_match'] exist?
         if match_from_file['cda_column_to_match'] != '' and match_from_file['cda_column_to_match'] not in cached_column_metadata['column'].unique():
             raise RuntimeError( f"'match_from_file['cda_column_to_match']' must be a valid CDA column; you supplied {match_from_file['cda_column_to_match']}, which is not." )
-
         # Does `match_from_file`['input_file'] exist and does it have a column named `match_from_file`['input_column']?
         if match_from_file['input_file'] != '':
-            
             try:
                 with open( match_from_file['input_file'] ) as IN:
                     input_file_column_names = next( IN ).rstrip( '\n' ).split( '\t' )
                     if match_from_file['input_column'] not in input_file_column_names:
                         raise RuntimeError( f"'match_from_file['input_column']' must specify a column that exists in 'match_from_file['input_file']'. You specified '{match_from_file['input_column']}', which is not present in '{match_from_file['input_file']}'." )
-
             except Exception as error:
                 raise RuntimeError( f"Couldn't read from match_from_file input file '{match_from_file['input_file']}': got error of type '{type(error)}', with error message '{error}'." )
-
         # Are the values given in `match_from_file` internally consistent? (Strange results might occur if not.)
         if match_from_file['cda_column_to_match'] == '':
             if match_from_file['input_file'] != '' or match_from_file['input_column'] != '':
@@ -476,19 +482,18 @@ def validate_parameter_values(
         elif match_from_file['input_column'] == '':
             if match_from_file['cda_column_to_match'] != '' or match_from_file['input_file'] != '':
                 raise RuntimeError( f"If the 'match_from_file' parameter is used, it must be a 3-element dictionary with keys ['input_file', 'input_column', 'cda_column_to_match'] pointing to non-empty values. You specified '{match_from_file}', which is not that." )
-
-    if match_from_file['input_file'] != '' and match_from_file['input_file'] == output_file:
-        raise RuntimeError( f"You specified the same file ('{output_file}') as both a source of filter values (via 'match_from_file') and the target output file ( via 'output_file'). Please make sure these two files are different." )
-
-    # Check that `data_source` is a single valid upstream data source label (for `called_function`=='column_values')
-    # or a list of valid upstream data source labels (for `called_function` in [ 'get_data', 'summarize' ]).
+        if match_from_file['input_file'] != '' and match_from_file['input_file'] == output_file:
+            raise RuntimeError( f"You specified the same file ('{output_file}') as both a source of filter values (via 'match_from_file') and the target output file ( via 'output_file'). Please make sure these two files are different." )
+    elif called_function in [ 'column_values' ]:
+        if match_from_file is not None:
+            raise RuntimeError( f"'match_from_file' cannot be non-null for column_values() caller: please alert the CDA devs to this event, something is misconfigured in our code." )
+
+    # Check that `data_source` is a list of valid upstream data source labels (for
+    # `called_function` in [ 'column_values', 'get_data', 'summarize' ]).
     # `valid_data_sources` was normalized to uppercase when it was constructed by the caller.
 
     normalized_data_source = list()
 
-    if called_function == 'column_values' and len( data_source ) > 1:
-        raise RuntimeError( f"The 'data_source' parameter must be one valid data source name (e.g. 'GDC'); you specified '{data_source}', which is not." )
-
     for ds in data_source:
         if ds.upper() not in valid_data_sources:
             raise RuntimeError( f"The 'data_source' parameter must be a list containing one or more of [ {', '.join( sorted( valid_data_sources ) )} ]. You supplied '{data_source}', which is not that." )
@@ -496,31 +501,30 @@ def validate_parameter_values(
 
     data_source = normalized_data_source
 
-    # Make sure CDA columns named in `add_columns` exist.
-
-    for column_name in add_columns:
-        match_result = re.search( r'^(.+)\.\*$', column_name )
-
-        if match_result is not None:
-            foreign_table = match_result.group(1)
-            if foreign_table not in tables():
+    # Make sure CDA columns named in `add_columns` and `exclude_columns` exist, unless we're called by 'column_values'.
+
+    if called_function in [ 'column_values' ]:
+        if add_columns is not None or exclude_columns is not None:
+            raise RuntimeError( f"'add_columns' and 'exclude_columns' must be null for column_values() caller: please alert the CDA devs to this event, something is misconfigured in our code." )
+    elif called_function in [ 'get_data', 'summarize' ]:
+        for column_name in add_columns:
+            match_result = re.search( r'^(.+)\.\*$', column_name )
+            if match_result is not None:
+                foreign_table = match_result.group(1)
+                if foreign_table not in tables():
+                    raise RuntimeError( f"'add_columns' can only contain valid CDA column names, or macros for whole tables like 'treatment.*'. You specified '{column_name}', which is neither." )
+            elif column_name not in cached_column_metadata['column'].unique():
                 raise RuntimeError( f"'add_columns' can only contain valid CDA column names, or macros for whole tables like 'treatment.*'. You specified '{column_name}', which is neither." )
-        elif column_name not in cached_column_metadata['column'].unique():
-            raise RuntimeError( f"'add_columns' can only contain valid CDA column names, or macros for whole tables like 'treatment.*'. You specified '{column_name}', which is neither." )
-
-    # Make sure CDA columns named in `exclude_columns` exist.
-
-    for column_name in exclude_columns:
-        match_result = re.search( r'^(.+)\.\*$', column_name )
+        for column_name in exclude_columns:
+            match_result = re.search( r'^(.+)\.\*$', column_name )
+            if match_result is not None:
+                foreign_table = match_result.group(1)
+                if foreign_table not in tables():
+                    raise RuntimeError( f"'exclude_columns' can only contain valid CDA column names, or macros for whole tables like 'treatment.*'. You specified '{column_name}', which is neither." )
+            elif column_name not in cached_column_metadata['column'].unique():
+                raise RuntimeError( f"'exclude_columns' can only contain valid CDA column names. You specified '{column_name}', which is not that." )
 
-        if match_result is not None:
-            foreign_table = match_result.group(1)
-            if foreign_table not in tables():
-                raise RuntimeError( f"'exclude_columns' can only contain valid CDA column names, or macros for whole tables like 'treatment.*'. You specified '{column_name}', which is neither." )
-        elif column_name not in cached_column_metadata['column'].unique():
-            raise RuntimeError( f"'exclude_columns' can only contain valid CDA column names. You specified '{column_name}', which is not that." )
-
-    # Check that `collate_results` is a boolean (for get_data) or None (for column_values, summarize) and that `called_function` has an expected value.
+    # Check that `collate_results` and `include_external_refs` are booleans (for get_data) or None (for column_values, summarize) and that `called_function` has an expected value.
 
     if called_function == 'get_data':
         if collate_results != True and collate_results != False:
@@ -529,9 +533,9 @@ def validate_parameter_values(
             raise RuntimeError( f"The `include_external_refs` parameter must be set to True or False; you specified '{include_external_refs}', which is neither." )
     elif called_function == 'summarize' or called_function == 'column_values':
         if collate_results is not None:
-            raise RuntimeError( f"Something has gone horribly and unexpectedly wrong with respect to phantom collate_results values in summarize() calls; please notify the CDA devs of this event." )
+            raise RuntimeError( f"Something has gone horribly and unexpectedly wrong with respect to phantom collate_results values in summarize() or column_values() calls; please notify the CDA devs of this event." )
         if include_external_refs is not None:
-            raise RuntimeError( f"Something has gone horribly and unexpectedly wrong with respect to phantom include_external_refs values in summarize() calls; please notify the CDA devs of this event." )
+            raise RuntimeError( f"Something has gone horribly and unexpectedly wrong with respect to phantom include_external_refs values in summarize() or column_values() calls; please notify the CDA devs of this event." )
     else:
         raise RuntimeError( f"`called_function` must be one of [ 'column_values', 'get_data', 'summarize' ] -- '{called_function}' is none of those. Please notify the CDA devs of this event." )
 
@@ -542,6 +546,12 @@ def validate_parameter_values(
     # writing.
 
     allowed_return_types = {
+        'column_values': {
+            '',
+            'dataframe',
+            'tsv',
+            'list'
+        },
         'get_data': {
             'dataframe',
             'tsv'
@@ -595,9 +605,66 @@ def validate_parameter_values(
 
                 raise RuntimeError( f"'output_file' was specified, but this is only meaningful if 'return_data_as' is set to 'json'. You requested return_data_as='{return_data_as}'.\n(Note that if you don't specify any value for 'return_data_as', it defaults to printing tables to the standard output stream and not to an output file.)." )
 
+        elif called_function == 'column_values':
+            
+            if return_data_as == 'tsv' and output_file == '':
+                # If the user asks for TSV, they also have to give us a path for the output file. If they didn't, complain.
+                raise RuntimeError( 'Return type \'tsv\' was requested, but \'output_file\' was not specified. Please specify output_file=\'some/path/string/to/write/your/tsv/to/your_tsv_output_file.tsv\'.' )
+
+            elif return_data_as != 'tsv' and output_file != '':
+                
+                # If the user put something in the `output_file` parameter but didn't specify `result_data_as`='tsv',
+                # they most likely want their data saved to a file (so ignoring the parameter misconfiguration
+                # isn't safe), but ultimately we can't be sure what they meant (so taking an action isn't safe),
+                # so we complain and ask them to clarify.
+
+                raise RuntimeError( f"'output_file' was specified, but this is only meaningful if 'return_data_as' is set to 'tsv'. You requested return_data_as='{return_data_as}'.\n(Note that if you don't specify any value for 'return_data_as', it defaults to 'dataframe'.)." )
+
         else:
             raise RuntimeError( f"Got unpexpectedly non-null 'return_data_as' value '{return_data_as}' from function '{called_function}'. Please report this event to the CDA devs." )
 
+    if called_function in [ 'column_values' ]:
+
+        #############################################################################################################################
+        # Process sorting directives.
+
+        # Enumerate all allowed values that a user can specify using the `sort_by` parameter. ( 'X:asc' will be aliased immediately to just 'X'. )
+
+        allowed_sort_by_options = {
+            'list': {
+                '',
+                'value',
+                'value:desc'
+            },
+            'dataframe_or_tsv': {
+                '',
+                'count',
+                'count:desc',
+                'value',
+                'value:desc'
+            }
+        }
+
+        if not isinstance( sort_by, str ):
+            # Complain if we receive any unexpected data types instead of string directives.
+            raise RuntimeError( f"'sort_by' must be a string; you used '{sort_by}', which is not." )
+
+        if return_data_as == 'list':
+            # Restrict sorting options for lists.
+            if sort_by == '':
+                sort_by = 'value'
+            elif sort_by not in allowed_sort_by_options['list']:
+                raise RuntimeError( f"return_data_as='list' can only be processed with sort_by='value' or sort_by='value:desc' (or omitting sort_by altogether). Please modify unsupported sort_by directive '{sort_by}' and try again." )
+        else:
+            # For TSV output files and DataFrames, we support more user-configurable options (defaulting to sort_by='count:desc'):
+            if sort_by == '':
+                sort_by = 'count:desc'
+            elif sort_by not in allowed_sort_by_options['dataframe_or_tsv']:
+                raise RuntimeError( f"unrecognized sort_by '{sort_by}'. Please use one of 'count', 'value', 'count:desc', 'value:desc', 'count:asc' or 'value:asc' (or omit the sort_by parameter altogether)." )
+
+    elif sort_by is not None:
+        raise RuntimeError( f"'sort_by' cannot be non-null for get_data() or summarize() callers: please alert the CDA devs to this event, something is misconfigured in our code." )
+
     return
 
 #############################################################################################################################
@@ -606,4 +673,7 @@ def validate_parameter_values(
 # 
 #############################################################################################################################
 
+# This is down here to avoid a circular import problem.
+from cdapython.discover import tables
+
 

{cdapython-2.0.8 → cdapython-2.0.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cdapython"
-version = "2.0.8"
+version = "2.0.10"
 description = "NCI CRDC CDA front-end user interface library"
 authors = []
 readme = "README.md"