nci-cidc-api-modules 1.0.9__py3-none-any.whl → 1.0.12__py3-none-any.whl

cidc_api/models/models.py

@@ -1467,100 +1467,68 @@ class TrialMetadata(CommonColumns):
         }
 
     @staticmethod
-    @with_default_session
-    def get_summaries(session: Session) -> List[dict]:
-        """
-        Return a list of trial summaries, where each summary has structure like:
-        ```python
-            {
-                "trial_id": ...,
-                "expected_assays": ..., # list of assays the trial should have data for
-                "file_size_bytes": ..., # total file size for the trial
-                "clinical_participants": ..., # number of participants with clinical data
-                "total_participants": ..., # number of unique participants with assay data
-                "total_samples": ..., # number of samples with assay data
-                "cytof": ..., # cytof sample count
-                ... # other assays and analysis
-            }
-        ```
-        NOTE: if the metadata model for any existing assays substantially changes,
-        or if new assays are introduced that don't follow the typical structure
-        (batches containing sample-level records), then this method will need to
-        be updated to accommodate those changes.
-
-        Only the assays are used for calculating `"total_participants"` and `"total_samples"`,
-        as all analyses are derived from assay data.
-        Each assay/analysis subquery is expected to return a set with `trial_id`, `key`,
-        and `cimac_id` which are used for both assay-level and overall counting.
-
-        There is a bit of complexity with the way that WES samples are counted:
-            - `"wes"` only counts tumor samples slated for paired wes_analysis
-            - `"wes_tumor_only"` counts all tumor samples NOT slated for paired wes_analysis
-            - `"wes_analysis"` counts tumor samples with paired wes_analysis
-            - `"wes_tumor_only_analysis"` counts (tumor) samples with tumor-only analysis
-        For `"total_[participants/samples]"`, ALL (ie tumor AND normal) WES assay samples are included.
-        """
+    def get_summaries_query() -> str:
         # Compute the total amount of data in bytes stored for each trial
         files_subquery = """
-            select
-                trial_id,
-                sum(file_size_bytes) as value
-            from
-                downloadable_files
-            group by
-                trial_id
-        """
+                    select
+                        trial_id,
+                        sum(file_size_bytes) as value
+                    from
+                        downloadable_files
+                    group by
+                        trial_id
+                """
 
         # Count how many participants have associated clinical data. The same
         # participant may appear in multiple clinical data files, so deduplicate
         # participants before counting them.
         clinical_subquery = """
-            select
-                trial_id,
-                count(distinct participants) as value
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{clinical_data,records}') as records,
-                jsonb_array_elements(records#>'{clinical_file,participants}') as participants
-            group by
-                trial_id
-        """
+                    select
+                        trial_id,
+                        count(distinct participants) as value
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{clinical_data,records}') as records,
+                        jsonb_array_elements(records#>'{clinical_file,participants}') as participants
+                    group by
+                        trial_id
+                """
 
         # Find all samples associated with each assay type for
         # assays whose metadata follows the typical structure: an array of batches,
         # with each batch containing an array of records, where each record
         # corresponds to a unique sample with a cimac_id.
         generic_assay_subquery = """
-            select
-                trial_id,
-                case
-                    when key = 'hande' then 'h&e'
-                    else key
-                end as key,
-                record->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_each(metadata_json->'assays') assays,
-                jsonb_array_elements(value) batches,
-                jsonb_array_elements(batches->'records') record
-            where key not in ('olink', 'nanostring', 'elisa', 'wes', 'misc_data')
-        """
+                    select
+                        trial_id,
+                        case
+                            when key = 'hande' then 'h&e'
+                            else key
+                        end as key,
+                        record->>'cimac_id' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_each(metadata_json->'assays') assays,
+                        jsonb_array_elements(value) batches,
+                        jsonb_array_elements(batches->'records') record
+                    where key not in ('olink', 'nanostring', 'elisa', 'wes', 'misc_data')
+                """
 
         # Find all samples associated with nanostring uploads.
         # Nanostring metadata has a slightly different structure than typical
         # assays, where each batch has an array of runs, and each run has
         # an array of sample-level entries each with a cimac_id.
         nanostring_subquery = """
-            select
-                trial_id,
-                'nanostring' as key,
-                sample->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{assays,nanostring}') batches,
-                jsonb_array_elements(batches->'runs') runs,
-                jsonb_array_elements(runs->'samples') sample
-        """
+                    select
+                        trial_id,
+                        'nanostring' as key,
+                        sample->>'cimac_id' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{assays,nanostring}') batches,
+                        jsonb_array_elements(batches->'runs') runs,
+                        jsonb_array_elements(runs->'samples') sample
+                """
 
         # Find all samples associated with olink uploads.
         # Unlike other assays, olink metadata is an object at the top level
@@ -1570,222 +1538,222 @@ class TrialMetadata(CommonColumns):
         # the samples corresponding to a given record are stored
         # like: record["files"]["assay_npx"]["samples"].
         olink_subquery = """
-            select
-                trial_id,
-                'olink' as key,
-                sample as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{assays,olink,batches}') batches,
-                jsonb_array_elements(batches->'records') records,
-                jsonb_array_elements_text(records#>'{files,assay_npx,samples}') sample
-        """
+                    select
+                        trial_id,
+                        'olink' as key,
+                        sample as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{assays,olink,batches}') batches,
+                        jsonb_array_elements(batches->'records') records,
+                        jsonb_array_elements_text(records#>'{files,assay_npx,samples}') sample
+                """
 
         # Find all samples associated with elisa uploads.
         # Unlike other assays, elisa metadata is an array of entries, each containing a single data file.
         # The samples corresponding to a given entry are stored like:
         # entry["assay_xlsx"]["samples"].
         elisa_subquery = """
-            select
-                trial_id,
-                'elisa' as key,
-                sample as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{assays,elisa}') entry,
-                jsonb_array_elements_text(entry#>'{assay_xlsx,samples}') sample
-        """
+                    select
+                        trial_id,
+                        'elisa' as key,
+                        sample as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{assays,elisa}') entry,
+                        jsonb_array_elements_text(entry#>'{assay_xlsx,samples}') sample
+                """
 
         # Find the tumor samples that have associated paired-analysis data.
         wes_analysis_subquery = """
-            select
-                trial_id,
-                'wes_analysis' as key,
-                pair#>>'{tumor,cimac_id}' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis,pair_runs}') pair
-            where
-                pair#>>'{report,report}' is not null
-            union all
-            select
-                trial_id,
-                'wes_analysis' as key,
-                pair#>>'{tumor,cimac_id}' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis_old,pair_runs}') pair
-            where
-                pair#>>'{report,report}' is not null
-        """
+                    select
+                        trial_id,
+                        'wes_analysis' as key,
+                        pair#>>'{tumor,cimac_id}' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{analysis,wes_analysis,pair_runs}') pair
+                    where
+                        pair#>>'{report,report}' is not null
+                    union all
+                    select
+                        trial_id,
+                        'wes_analysis' as key,
+                        pair#>>'{tumor,cimac_id}' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{analysis,wes_analysis_old,pair_runs}') pair
+                    where
+                        pair#>>'{report,report}' is not null
+                """
 
         # Find the tumor samples that have associated tumor-only analysis data.
         wes_tumor_only_analysis_subquery = """
-            select
-                trial_id,
-                'wes_tumor_only_analysis' as key,
-                run#>>'{tumor,cimac_id}' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis,runs}') run
-            where
-                run#>>'{report,report}' is not null
-            union all
-            select
-                trial_id,
-                'wes_tumor_only_analysis' as key,
-                run#>>'{tumor,cimac_id}' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis_old,runs}') run
-            where
-                run#>>'{report,report}' is not null
-        """
+                    select
+                        trial_id,
+                        'wes_tumor_only_analysis' as key,
+                        run#>>'{tumor,cimac_id}' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis,runs}') run
+                    where
+                        run#>>'{report,report}' is not null
+                    union all
+                    select
+                        trial_id,
+                        'wes_tumor_only_analysis' as key,
+                        run#>>'{tumor,cimac_id}' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis_old,runs}') run
+                    where
+                        run#>>'{report,report}' is not null
+                """
 
         # Find the tumor samples that will have associated paired-analysis data.
         # We are asserting that a tumor sample will not be used for multiple analyses.
         # This is similar to the wes_analysis_subquery but without the requirement for a report,
         # which is the defining feature of analysis.
         wes_subquery = """
-            select
-                trial_id,
-                'wes' as key,
-                pair#>>'{tumor,cimac_id}' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis,pair_runs}') pair
-            union all
-            select
-                trial_id,
-                'wes' as key,
-                pair#>>'{tumor,cimac_id}' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis_old,pair_runs}') pair
-        """
-
-        # Find the tumor samples that WON'T have associated paired-analysis data.
-        # Get all tumor samples with WES data not in the equivalent of wes_subquery.
-        wes_tumor_assay_subquery = """
-            select
-                trial_metadata.trial_id,
-                'wes_tumor_only' as key,
-                record->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{assays,wes}') batch,
-                jsonb_array_elements(batch->'records') record
-            join (
-                select
-                    trial_id,
-                    sample->>'cimac_id' as cimac_id
-                from
-                    trial_metadata,
-                    jsonb_array_elements(metadata_json->'participants') participant,
-                    jsonb_array_elements(participant->'samples') sample
-
-                where
-                        sample->>'processed_sample_derivative' = 'Tumor DNA'
-                    or
-                        sample->>'processed_sample_derivative' = 'Tumor RNA'
-            ) sample_data
-            on
-                sample_data.cimac_id = record->>'cimac_id'
-            where
-                sample_data.trial_id = trial_metadata.trial_id
-                and
-                record->>'cimac_id' not in (
                     select
-                        pair#>>'{tumor,cimac_id}'
+                        trial_id,
+                        'wes' as key,
+                        pair#>>'{tumor,cimac_id}' as cimac_id
                     from
                         trial_metadata,
                         jsonb_array_elements(metadata_json#>'{analysis,wes_analysis,pair_runs}') pair
                     union all
                     select
-                        pair#>>'{tumor,cimac_id}'
+                        trial_id,
+                        'wes' as key,
+                        pair#>>'{tumor,cimac_id}' as cimac_id
                     from
                         trial_metadata,
                         jsonb_array_elements(metadata_json#>'{analysis,wes_analysis_old,pair_runs}') pair
-                )
-        """
+                """
+
+        # Find the tumor samples that WON'T have associated paired-analysis data.
+        # Get all tumor samples with WES data not in the equivalent of wes_subquery.
+        wes_tumor_assay_subquery = """
+                    select
+                        trial_metadata.trial_id,
+                        'wes_tumor_only' as key,
+                        record->>'cimac_id' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{assays,wes}') batch,
+                        jsonb_array_elements(batch->'records') record
+                    join (
+                        select
+                            trial_id,
+                            sample->>'cimac_id' as cimac_id
+                        from
+                            trial_metadata,
+                            jsonb_array_elements(metadata_json->'participants') participant,
+                            jsonb_array_elements(participant->'samples') sample
+
+                        where
+                                sample->>'processed_sample_derivative' = 'Tumor DNA'
+                            or
+                                sample->>'processed_sample_derivative' = 'Tumor RNA'
+                    ) sample_data
+                    on
+                        sample_data.cimac_id = record->>'cimac_id'
+                    where
+                        sample_data.trial_id = trial_metadata.trial_id
+                        and
+                        record->>'cimac_id' not in (
+                            select
+                                pair#>>'{tumor,cimac_id}'
+                            from
+                                trial_metadata,
+                                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis,pair_runs}') pair
+                            union all
+                            select
+                                pair#>>'{tumor,cimac_id}'
+                            from
+                                trial_metadata,
+                                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis_old,pair_runs}') pair
+                        )
+                """
 
         # Find ALL normal samples that have WES data.
         # This is included in counting for total_participants and total_samples,
         # but do not affect the assay-level counts which are tumor sample-specific for WES.
         wes_normal_assay_subquery = """
-            select
-                trial_id,
-                'wes_normal' as key,
-                record->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{assays,wes}') batch,
-                jsonb_array_elements(batch->'records') record
-            join (
                     select
-                        sample->>'cimac_id' as cimac_id
+                        trial_id,
+                        'wes_normal' as key,
+                        record->>'cimac_id' as cimac_id
                     from
                         trial_metadata,
-                        jsonb_array_elements(metadata_json->'participants') participant,
-                        jsonb_array_elements(participant->'samples') sample
-                    where
-                            sample->>'processed_sample_derivative' <> 'Tumor DNA'
-                        and
-                            sample->>'processed_sample_derivative' <> 'Tumor RNA'
-                ) sample_data
-            on
-                sample_data.cimac_id = record->>'cimac_id'
-        """
+                        jsonb_array_elements(metadata_json#>'{assays,wes}') batch,
+                        jsonb_array_elements(batch->'records') record
+                    join (
+                            select
+                                sample->>'cimac_id' as cimac_id
+                            from
+                                trial_metadata,
+                                jsonb_array_elements(metadata_json->'participants') participant,
+                                jsonb_array_elements(participant->'samples') sample
+                            where
+                                    sample->>'processed_sample_derivative' <> 'Tumor DNA'
+                                and
+                                    sample->>'processed_sample_derivative' <> 'Tumor RNA'
+                        ) sample_data
+                    on
+                        sample_data.cimac_id = record->>'cimac_id'
+                """
 
         # Find all samples associated with RNA analysis uploads.
         # There is ONLY level_1
         rna_level1_analysis_subquery = """
-            select
-                trial_id,
-                'rna_level1_analysis' as key,
-                run->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,rna_analysis,level_1}') run
-        """
+                    select
+                        trial_id,
+                        'rna_level1_analysis' as key,
+                        run->>'cimac_id' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{analysis,rna_analysis,level_1}') run
+                """
 
         # Find all samples associated with TCR analysis uploads.
         tcr_analysis_subquery = """
-            select
-                trial_id,
-                'tcr_analysis' as key,
-                record->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,tcr_analysis,batches}') batch,
-                jsonb_array_elements(batch->'records') record
-        """
+                    select
+                        trial_id,
+                        'tcr_analysis' as key,
+                        record->>'cimac_id' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{analysis,tcr_analysis,batches}') batch,
+                        jsonb_array_elements(batch->'records') record
+                """
 
         # Find all samples associated with CyTOF analysis uploads.
         cytof_analysis_subquery = """
-            select
-                trial_id,
-                'cytof_analysis' as key,
-                record->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{assays,cytof}') batch,
-                jsonb_array_elements(batch->'records') record
-            where
-                record->'output_files' is not null
-        """
+                    select
+                        trial_id,
+                        'cytof_analysis' as key,
+                        record->>'cimac_id' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{assays,cytof}') batch,
+                        jsonb_array_elements(batch->'records') record
+                    where
+                        record->'output_files' is not null
+                """
 
         # Find all samples associated with ATACseq analysis uploads.
         atacseq_analysis_subquery = """
-            select
-                trial_id,
-                'atacseq_analysis' as key,
-                record->>'cimac_id' as cimac_id
-            from
-                trial_metadata,
-                jsonb_array_elements(metadata_json#>'{analysis,atacseq_analysis}') batch,
-                jsonb_array_elements(batch->'records') record
-        """
+                    select
+                        trial_id,
+                        'atacseq_analysis' as key,
+                        record->>'cimac_id' as cimac_id
+                    from
+                        trial_metadata,
+                        jsonb_array_elements(metadata_json#>'{analysis,atacseq_analysis}') batch,
+                        jsonb_array_elements(batch->'records') record
+                """
 
         # Build up a JSON object mapping analysis types to arrays of excluded samples.
         # The resulting object will have structure like:
@@ -1795,79 +1763,79 @@ class TrialMetadata(CommonColumns):
         #   ...
         # }
         excluded_samples_subquery = """
-            select
-                trial_id,
-                jsonb_object_agg(key, value) as value
-            from (
-                select
-                    trial_id,
-                    key,
-                    jsonb_agg(sample) as value
-                from (
-                    select
-                        trial_id,
-                        'cytof_analysis' as key,
-                        jsonb_array_elements(batch->'excluded_samples') as sample
-                    from
-                        trial_metadata,
-                        jsonb_array_elements(metadata_json#>'{assays,cytof}') batch
-                    union all
-                    select
-                        trial_id,
-                        'wes_analysis' as key,
-                        jsonb_array_elements(metadata_json#>'{analysis,wes_analysis,excluded_samples}') as sample
-                    from
-                        trial_metadata
-                    union all
                     select
                         trial_id,
-                        'wes_analysis' as key,
-                        jsonb_array_elements(metadata_json#>'{analysis,wes_analysis_old,excluded_samples}') as sample
-                    from
-                        trial_metadata
-                    union all
-                    select
-                        trial_id,
-                        'wes_tumor_only_analysis' as key,
-                        jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis,excluded_samples}') as sample
-                    from
-                        trial_metadata
-                    union all
-                    select
-                        trial_id,
-                        'wes_tumor_only_analysis' as key,
-                        jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis_old,excluded_samples}') as sample
-                    from
-                        trial_metadata
-                    union all
+                        jsonb_object_agg(key, value) as value
+                    from (
+                        select
+                            trial_id,
+                            key,
+                            jsonb_agg(sample) as value
+                        from (
+                            select
+                                trial_id,
+                                'cytof_analysis' as key,
+                                jsonb_array_elements(batch->'excluded_samples') as sample
+                            from
+                                trial_metadata,
+                                jsonb_array_elements(metadata_json#>'{assays,cytof}') batch
+                            union all
+                            select
+                                trial_id,
+                                'wes_analysis' as key,
+                                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis,excluded_samples}') as sample
+                            from
+                                trial_metadata
+                            union all
+                            select
+                                trial_id,
+                                'wes_analysis' as key,
+                                jsonb_array_elements(metadata_json#>'{analysis,wes_analysis_old,excluded_samples}') as sample
+                            from
+                                trial_metadata
+                            union all
+                            select
+                                trial_id,
+                                'wes_tumor_only_analysis' as key,
+                                jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis,excluded_samples}') as sample
+                            from
+                                trial_metadata
+                            union all
+                            select
+                                trial_id,
+                                'wes_tumor_only_analysis' as key,
+                                jsonb_array_elements(metadata_json#>'{analysis,wes_tumor_only_analysis_old,excluded_samples}') as sample
+                            from
+                                trial_metadata
+                            union all
+                            select
+                                trial_id,
+                                'rna_level1_analysis' as key,
+                                jsonb_array_elements(metadata_json#>'{analysis,rna_analysis,excluded_samples}') as sample
+                            from
+                                trial_metadata
+                            union all
+                            select
+                                trial_id,
+                                'tcr_analysis' as key,
+                                jsonb_array_elements(batches->'excluded_samples') as sample
+                            from
+                                trial_metadata,
+                                jsonb_array_elements(metadata_json#>'{analysis,tcr_analysis,batches}') batches
+                        ) excluded_q1
+                        group by trial_id, key
+                    ) excluded_q2
+                    group by trial_id
+                """
+
+        # Extract an array of expected assays or an empty array if expected assays is null.
+        expected_assays_subquery = """
                     select
                         trial_id,
-                        'rna_level1_analysis' as key,
-                        jsonb_array_elements(metadata_json#>'{analysis,rna_analysis,excluded_samples}') as sample
+                        coalesce(metadata_json->'expected_assays', '[]'::jsonb) as expected_assays
                     from
                         trial_metadata
-                    union all
-                    select
-                        trial_id,
-                        'tcr_analysis' as key,
-                        jsonb_array_elements(batches->'excluded_samples') as sample
-                    from
-                        trial_metadata,
-                        jsonb_array_elements(metadata_json#>'{analysis,tcr_analysis,batches}') batches
-                ) excluded_q1
-                group by trial_id, key
-            ) excluded_q2
-            group by trial_id
-        """
-
-        # Extract an array of expected assays or an empty array if expected assays is null.
-        expected_assays_subquery = """
-            select
-                trial_id,
-                coalesce(metadata_json->'expected_assays', '[]'::jsonb) as expected_assays
-            from
-                trial_metadata
-        """
+                """
 
         # All the subqueries produce the same set of columns, so UNION ALL
         # them together into a single query, aggregating results into
@@ -1875,98 +1843,136 @@ class TrialMetadata(CommonColumns):
         # NOTE: we use UNION ALL for assay-level counts instead of just UNION to
         # prevent any unwanted de-duplication within subquery results.
         combined_query = f"""
-            select
-                jsonb_object_agg('trial_id', expected_assays.trial_id)
-                || jsonb_object_agg('excluded_samples', coalesce(excluded_sample_lists.value, '{{}}'::jsonb))
-                || jsonb_object_agg('expected_assays', coalesce(expected_assays, '[]'::jsonb))
-                || jsonb_object_agg('file_size_bytes', coalesce(file_sizes.value, 0))
-                || jsonb_object_agg('clinical_participants', coalesce(clinical_participants.value, 0))
-                || jsonb_build_object('total_participants', coalesce(total_participants, 0))
-                || jsonb_build_object('total_samples', coalesce(total_samples, 0))
-                || coalesce(sample_counts.sample_counts, '{{}}'::jsonb)
-            from ({expected_assays_subquery}) expected_assays
-            full join (
-                select
-                    trial_id,
-                    count(distinct cimac_id) as total_samples,
-                    count(distinct left(cimac_id, 7)) as total_participants
-                from (
-                    {generic_assay_subquery}
-                    union
-                    {nanostring_subquery}
-                    union
-                    {olink_subquery}
-                    union
-                    {elisa_subquery}
-                    union
-                    {wes_subquery}
-                    union
-                    {wes_tumor_assay_subquery}
-                    union
-                    {wes_normal_assay_subquery}
-                ) assays
-                group by
-                    trial_id
-            ) total_counts
-            on expected_assays.trial_id = total_counts.trial_id
-            full join (
-                select
-                    trial_id,
-                    jsonb_object_agg(key, num_sample) as sample_counts
-                from (
                     select
-                        trial_id,
-                        key,
-                        count(distinct cimac_id) as num_sample
-                    from (
-                        {generic_assay_subquery}
-                        union all
-                        {nanostring_subquery}
-                        union all
-                        {olink_subquery}
-                        union all
-                        {elisa_subquery}
-                        union all
-                        {wes_subquery}
-                        union all
-                        {wes_tumor_assay_subquery}
-                        union all
-                        {wes_analysis_subquery}
-                        union all
-                        {wes_tumor_only_analysis_subquery}
-                        union all
-                        {rna_level1_analysis_subquery}
-                        union all
-                        {tcr_analysis_subquery}
-                        union all
-                        {cytof_analysis_subquery}
-                        union all
-                        {atacseq_analysis_subquery}
-                    ) assays_and_analysis
+                        jsonb_object_agg('trial_id', expected_assays.trial_id)
+                        || jsonb_object_agg('excluded_samples', coalesce(excluded_sample_lists.value, '{{}}'::jsonb))
+                        || jsonb_object_agg('expected_assays', coalesce(expected_assays, '[]'::jsonb))
+                        || jsonb_object_agg('file_size_bytes', coalesce(file_sizes.value, 0))
+                        || jsonb_object_agg('clinical_participants', coalesce(clinical_participants.value, 0))
+                        || jsonb_build_object('total_participants', coalesce(total_participants, 0))
+                        || jsonb_build_object('total_samples', coalesce(total_samples, 0))
+                        || coalesce(sample_counts.sample_counts, '{{}}'::jsonb) as result
+                    from ({expected_assays_subquery}) expected_assays
+                    full join (
+                        select
+                            trial_id,
+                            count(distinct cimac_id) as total_samples,
+                            count(distinct left(cimac_id, 7)) as total_participants
+                        from (
+                            {generic_assay_subquery}
+                            union
+                            {nanostring_subquery}
+                            union
+                            {olink_subquery}
+                            union
+                            {elisa_subquery}
+                            union
+                            {wes_subquery}
+                            union
+                            {wes_tumor_assay_subquery}
+                            union
+                            {wes_normal_assay_subquery}
+                        ) assays
+                        group by
+                            trial_id
+                    ) total_counts
+                    on expected_assays.trial_id = total_counts.trial_id
+                    full join (
+                        select
+                            trial_id,
+                            jsonb_object_agg(key, num_sample) as sample_counts
+                        from (
+                            select
+                                trial_id,
+                                key,
+                                count(distinct cimac_id) as num_sample
+                            from (
+                                {generic_assay_subquery}
+                                union all
+                                {nanostring_subquery}
+                                union all
+                                {olink_subquery}
+                                union all
+                                {elisa_subquery}
+                                union all
+                                {wes_subquery}
+                                union all
+                                {wes_tumor_assay_subquery}
+                                union all
+                                {wes_analysis_subquery}
+                                union all
+                                {wes_tumor_only_analysis_subquery}
+                                union all
+                                {rna_level1_analysis_subquery}
+                                union all
+                                {tcr_analysis_subquery}
+                                union all
+                                {cytof_analysis_subquery}
+                                union all
+                                {atacseq_analysis_subquery}
+                            ) assays_and_analysis
+                            group by
+                                trial_id, key
+                        ) q
+                        group by
+                            trial_id
+                    ) sample_counts
+                    on expected_assays.trial_id = sample_counts.trial_id
+                    full join ({excluded_samples_subquery}) excluded_sample_lists
+                    on expected_assays.trial_id = excluded_sample_lists.trial_id
+                    full join ({files_subquery}) file_sizes
+                    on expected_assays.trial_id = file_sizes.trial_id
+                    full join ({clinical_subquery}) clinical_participants
+                    on expected_assays.trial_id = clinical_participants.trial_id
                     group by
-                        trial_id, key
-                ) q
-                group by
-                    trial_id
-            ) sample_counts
-            on expected_assays.trial_id = sample_counts.trial_id
-            full join ({excluded_samples_subquery}) excluded_sample_lists
-            on expected_assays.trial_id = excluded_sample_lists.trial_id
-            full join ({files_subquery}) file_sizes
-            on expected_assays.trial_id = file_sizes.trial_id
-            full join ({clinical_subquery}) clinical_participants
-            on expected_assays.trial_id = clinical_participants.trial_id
-            group by
-                expected_assays.trial_id,
-                total_participants,
-                total_samples,
-                sample_counts.sample_counts
-            ;
-        """
-
-        # Run the query and extract the trial-level summary dictionaries
+                        expected_assays.trial_id,
+                        total_participants,
+                        total_samples,
+                        sample_counts.sample_counts
+                    ;
+                """
+
+        return combined_query
+
+    @staticmethod
+    @with_default_session
+    def get_summaries(session: Session) -> List[dict]:
+        """
+        Return a list of trial summaries, where each summary has structure like:
+        ```python
+            {
+                "trial_id": ...,
+                "expected_assays": ..., # list of assays the trial should have data for
+                "file_size_bytes": ..., # total file size for the trial
+                "clinical_participants": ..., # number of participants with clinical data
+                "total_participants": ..., # number of unique participants with assay data
+                "total_samples": ..., # number of samples with assay data
+                "cytof": ..., # cytof sample count
+                ... # other assays and analysis
+            }
+        ```
+        NOTE: if the metadata model for any existing assays substantially changes,
+        or if new assays are introduced that don't follow the typical structure
+        (batches containing sample-level records), then this method will need to
+        be updated to accommodate those changes.
+
+        Only the assays are used for calculating `"total_participants"` and `"total_samples"`,
+        as all analyses are derived from assay data.
+        Each assay/analysis subquery is expected to return a set with `trial_id`, `key`,
+        and `cimac_id` which are used for both assay-level and overall counting.
+
+        There is a bit of complexity with the way that WES samples are counted:
+            - `"wes"` only counts tumor samples slated for paired wes_analysis
+            - `"wes_tumor_only"` counts all tumor samples NOT slated for paired wes_analysis
+            - `"wes_analysis"` counts tumor samples with paired wes_analysis
+            - `"wes_tumor_only_analysis"` counts (tumor) samples with tumor-only analysis
+        For `"total_[participants/samples]"`, ALL (ie tumor AND normal) WES assay samples are included.
+        """
+        summaries_query = "SELECT result FROM trial_summaries_mv"
+        # Retrieve trial-level summary results from data cached in trial_summaries_mv materialized view.
+        # The source of the SQL query used in trial_summaries_mv is get_summaries_query()
         summaries = [
-            summary for (summary,) in session.execute(combined_query) if summary
+            summary for (summary,) in session.execute(summaries_query) if summary
         ]
 
         # Shortcut to impute 0 values for assays where trials don't yet have data

{nci_cidc_api_modules-1.0.9.dist-info → nci_cidc_api_modules-1.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nci_cidc_api_modules
-Version: 1.0.9
+Version: 1.0.12
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -21,10 +21,11 @@ Requires-Dist: google-cloud-bigquery ==3.18.0
 Requires-Dist: google-api-python-client ==2.64.0
 Requires-Dist: packaging >=20.0.0
 Requires-Dist: pyarrow ==14.0.1
+Requires-Dist: numpy <2,>=1.16.5
 Requires-Dist: pandas <2,>=1
 Requires-Dist: python-dotenv ==0.10.3
-Requires-Dist: requests ==2.31.0
-Requires-Dist: jinja2 ==3.1.3
+Requires-Dist: requests ==2.32.3
+Requires-Dist: jinja2 ==3.1.4
 Requires-Dist: nci-cidc-schemas ==0.26.33
 
 # NCI CIDC API <!-- omit in TOC -->

{nci_cidc_api_modules-1.0.9.dist-info → nci_cidc_api_modules-1.0.12.dist-info}/RECORD

@@ -8,7 +8,7 @@ cidc_api/csms/auth.py,sha256=25Yma2Kz3KLENAPSeBYacFuSZXng-EDgmgInKBsRyP0,3191
 cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
 cidc_api/models/csms_api.py,sha256=Wp4b53vwOqSlOIaoAYGlI1p8ZfXRXmVJ6MLcsvzq0LA,31664
 cidc_api/models/migrations.py,sha256=gp9vtkYbA9FFy2s-7woelAmsvQbJ41LO2_DY-YkFIrQ,11464
-cidc_api/models/models.py,sha256=AYt0rIzaeQ0HHlTSeerbTpYwMJwqt93aadOuFLLEqBA,120820
+cidc_api/models/models.py,sha256=Hjp9sieGdldNbUzneFi-7vRYyo9wwr0D-0m_UbxsDEk,124106
 cidc_api/models/schemas.py,sha256=7tDYtmULuzTt2kg7RorWhte06ffalgpQKrFiDRGcPEQ,2711
 cidc_api/models/files/__init__.py,sha256=8BMTnUSHzUbz0lBeEQY6NvApxDD3GMWMduoVMos2g4Y,213
 cidc_api/models/files/details.py,sha256=eg1u8uZwtxb0m9mFobcTL_mnPBMq1MPZv3NN3KWMGOI,62309
@@ -18,8 +18,8 @@ cidc_api/shared/auth.py,sha256=VMd_3QJE2iG16QxuGzHBV9MzJJItOZNn9gcw0_iUBLI,11647
 cidc_api/shared/emails.py,sha256=5dyuKlpcg1M4P_RrAt0ss2hiCqb-Y7p2XXR1d9uBXg8,4868
 cidc_api/shared/gcloud_client.py,sha256=7dDs0crLMJKdIp4IDSfrZBMB3h-zvWNieB81azoeLO4,33746
 cidc_api/shared/rest_utils.py,sha256=LMfBpvJRjkfQjCzVXuhTTe4Foz4wlvaKg6QntyR-Hkc,6648
-nci_cidc_api_modules-1.0.9.dist-info/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
-nci_cidc_api_modules-1.0.9.dist-info/METADATA,sha256=qgcUc4UDf8wzx1U9aksotUyA1HvGjKfLoBcgSg2ap7w,40474
-nci_cidc_api_modules-1.0.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-nci_cidc_api_modules-1.0.9.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
-nci_cidc_api_modules-1.0.9.dist-info/RECORD,,
+nci_cidc_api_modules-1.0.12.dist-info/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
+nci_cidc_api_modules-1.0.12.dist-info/METADATA,sha256=SPwKY2ReUVP7izlWlwHgVmLTSPxN95TSsFIWSQstems,40508
+nci_cidc_api_modules-1.0.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+nci_cidc_api_modules-1.0.12.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
+nci_cidc_api_modules-1.0.12.dist-info/RECORD,,