idc-index-data 22.1.1__tar.gz → 22.1.2__tar.gz

{idc_index_data-22.1.1 → idc_index_data-22.1.2}/.github/workflows/external-indices.yml

@@ -55,6 +55,6 @@ jobs:
         if: github.event_name == 'release' && github.event.action == 'published'
         uses: ncipollo/release-action@v1
         with:
-          artifacts: "release_artifacts/*.parquet,release_artifacts/*.json"
+          artifacts: "release_artifacts/*.parquet,release_artifacts/*.json,release_artifacts/*.sql"
           allowUpdates: true
           omitBodyDuringUpdate: true

{idc_index_data-22.1.1 → idc_index_data-22.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: idc-index-data
-Version: 22.1.1
+Version: 22.1.2
 Summary: ImagingDataCommons index to query and download data.
 Author-Email: Andrey Fedorov <andrey.fedorov@gmail.com>, Vamsi Thiriveedhi <vthiriveedhi@mgh.harvard.edu>, Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
 License: Copyright 2024 Andrey Fedorov

{idc_index_data-22.1.1 → idc_index_data-22.1.2}/assets/sm_index.sql

@@ -82,10 +82,14 @@ SpecimenPreparationSequence_unnested AS (
 SELECT
   temp_table.SeriesInstanceUID,
   -- Embedding Medium
+  # description:
+  # embedding medium used for the slide preparation
   ARRAY(
     SELECT IF(code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
     FROM UNNEST(embeddingMedium_code_str) AS code
   ) AS embeddingMedium_CodeMeaning,
+  # description:
+  # embedding medium code tuple
   ARRAY(
     SELECT IF(code IS NULL, NULL,
               IF(STRPOS(code, ':') = 0, NULL,
@@ -93,10 +97,14 @@ SELECT
     FROM UNNEST(embeddingMedium_code_str) AS code
   ) AS embeddingMedium_code_designator_value_str,
   -- Tissue Fixative
+  # description:
+  # tissue fixative used for the slide preparation
   ARRAY(
     SELECT IF(code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
     FROM UNNEST(tissueFixative_code_str) AS code
   ) AS tissueFixative_CodeMeaning,
+  # description:
+  # tissue fixative code tuple
   ARRAY(
     SELECT IF(code IS NULL, NULL,
               IF(STRPOS(code, ':') = 0, NULL,
@@ -104,31 +112,56 @@ SELECT
     FROM UNNEST(tissueFixative_code_str) AS code
   ) AS tissueFixative_code_designator_value_str,
   -- Staining using substance
+  # description:
+  # staining substances used for the slide preparation
   ARRAY(
     SELECT IF(code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
     FROM UNNEST(staining_usingSubstance_code_str) AS code
   ) AS staining_usingSubstance_CodeMeaning,
+  # description:
+  # staining using substance code tuple
   ARRAY(
     SELECT IF(code IS NULL, NULL,
               IF(STRPOS(code, ':') = 0, NULL,
                  SUBSTR(code, STRPOS(code, ':') + 1)))
     FROM UNNEST(staining_usingSubstance_code_str) AS code
   ) AS staining_usingSubstance_code_designator_value_str,
-
+  # description:
+  # pixel spacing in mm at the maximum resolution layer, rounded to 2 significant figures
   if(COALESCE(min_spacing_0, fg_min_spacing_0) = 0, 0,
     round(COALESCE(min_spacing_0, fg_min_spacing_0) ,CAST(2 -1-floor(log10(abs(COALESCE(min_spacing_0, fg_min_spacing_0) ))) AS INT64))) AS min_PixelSpacing_2sf,
+  # description:
+  # width of the image at the maximum resolution
   COALESCE(max_TotalPixelMatrixColumns, max_Columns) AS max_TotalPixelMatrixColumns,
+  # description:
+  # height of the image at the maximum resolution
   COALESCE(max_TotalPixelMatrixRows, max_Rows) AS max_TotalPixelMatrixRows,
+  # description:
+  # power of the objective lens of the equipment used to digitize the slide
   SAFE_CAST(ObjectiveLensPower as INT) as ObjectiveLensPower,
+  # description:
+  # anatomic location from where the imaged specimen was collected
   CONCAT(SPLIT(primaryAnatomicStructure_code_str,":")[SAFE_OFFSET(0)],":",SPLIT(primaryAnatomicStructure_code_str,":")[SAFE_OFFSET(1)]) as primaryAnatomicStructure_code_designator_value_str,
+  # description:
+  # code tuple for the anatomic location from where the imaged specimen was collected
   SPLIT(primaryAnatomicStructure_code_str,":")[SAFE_OFFSET(2)] as primaryAnatomicStructure_CodeMeaning,
+  # description:
+  # additional characteristics of the specimen, such as whether it is a tumor or normal tissue (when available)
   CONCAT(SPLIT(primaryAnatomicStructureModifier_code_str,":")[SAFE_OFFSET(0)],":",SPLIT(primaryAnatomicStructureModifier_code_str,":")[SAFE_OFFSET(1)]) as primaryAnatomicStructureModifier_code_designator_value_str,
+  # description:
+  # code tuple for additional characteristics of the specimen, such as whether it is a tumor or normal tissue (when available)
   SPLIT(primaryAnatomicStructureModifier_code_str,":")[SAFE_OFFSET(2)] as primaryAnatomicStructureModifier_CodeMeaning,
-
+  # description:
+  # illumination type used during slide digitization
   CONCAT(SPLIT(illuminationType_code_str,":")[SAFE_OFFSET(0)],":",SPLIT(illuminationType_code_str,":")[SAFE_OFFSET(1)]) as illuminationType_code_designator_value_str,
+  # description:
+  # code tuple for the illumination type used during slide digitization
   SPLIT(illuminationType_code_str,":")[SAFE_OFFSET(2)] as illuminationType_CodeMeaning,
-
+  # description:
+  # admitting diagnosis associated with the specimen imaged on the slide (when available)
   CONCAT(SPLIT(admittingDiagnosis_code_str,":")[SAFE_OFFSET(0)],":",SPLIT(admittingDiagnosis_code_str,":")[SAFE_OFFSET(1)]) as admittingDiagnosis_code_designator_value_str,
+  # description:
+  # code tuple for the admitting diagnosis associated with the specimen imaged on the slide (when available)
   SPLIT(admittingDiagnosis_code_str,":")[SAFE_OFFSET(2)] as admittingDiagnosis_CodeMeaning,
 FROM
   temp_table

{idc_index_data-22.1.1 → idc_index_data-22.1.2}/assets/sm_instance_index.sql

@@ -48,15 +48,23 @@ WITH
   GROUP BY
     SOPInstanceUID )
 SELECT
+  # description:
+  # unique identifier of the instance
   dicom_all.SOPInstanceUID,
+  # description:
+  # unique identifier of the series
   dicom_all.SeriesInstanceUID,
   -- Embedding Medium
+  # description:
+  # embedding medium used for the slide preparation
   ARRAY(
   SELECT
   IF
     (code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
   FROM
     UNNEST(embeddingMedium_code_str) AS code ) AS embeddingMedium_CodeMeaning,
+  # description:
+  # embedding medium code tuple
   ARRAY(
   SELECT
   IF
@@ -66,12 +74,16 @@ SELECT
   FROM
     UNNEST(embeddingMedium_code_str) AS code ) AS embeddingMedium_code_designator_value_str,
   -- Tissue Fixative
+  # description:
+  # tissue fixative used for the slide preparation
   ARRAY(
   SELECT
   IF
     (code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
   FROM
     UNNEST(tissueFixative_code_str) AS code ) AS tissueFixative_CodeMeaning,
+  # description:
+  # tissue fixative code tuple
   ARRAY(
   SELECT
   IF
@@ -81,12 +93,16 @@ SELECT
   FROM
     UNNEST(tissueFixative_code_str) AS code ) AS tissueFixative_code_designator_value_str,
   -- Staining using substance
+  # description:
+  # staining substances used for the slide preparation
   ARRAY(
   SELECT
   IF
     (code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
   FROM
     UNNEST(staining_usingSubstance_code_str) AS code ) AS staining_usingSubstance_CodeMeaning,
+  # description:
+  # staining using substance code tuple
   ARRAY(
   SELECT
   IF
@@ -98,13 +114,27 @@ SELECT
   -- instance-specific image attributes
   -- NB: there is a caveat that I think in general, we expect square pixels, but in htan_wustl and cptac_luad this assumption does not hold,
   -- and in htan_wustl, the difference is rather large (x2) - waiting to hear from David Clunie about this...
+  # description:
+  # pixel spacing in mm, rounded to 2 significant figures
   SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64) AS PixelSpacing_0,
+  # description:
+  # DICOM ImageType attribute
   dicom_all.ImageType,
+  # description:
+  # DICOM TransferSyntaxUID attribute
   dicom_all.TransferSyntaxUID,
+  # description:
+  # size of the instance file in bytes
   dicom_all.instance_size,
+  # description:
+  # number of columns in the image
   dicom_all.TotalPixelMatrixColumns,
+  # description:
+  # number of rows in the image
   dicom_all.TotalPixelMatrixRows,
   -- attributes needed to retrieve the selected instances/files
+  # description:
+  # unique identifier of the instance within the IDC
   dicom_all.crdc_instance_uuid
 FROM
   `bigquery-public-data.idc_v22.dicom_all` AS dicom_all

{idc_index_data-22.1.1 → idc_index_data-22.1.2}/pyproject.toml

@@ -13,7 +13,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "idc-index-data"
-version = "22.1.1"
+version = "22.1.2"
 authors = [
   { name = "Andrey Fedorov", email = "andrey.fedorov@gmail.com" },
   { name = "Vamsi Thiriveedhi", email = "vthiriveedhi@mgh.harvard.edu" },

{idc_index_data-22.1.1 → idc_index_data-22.1.2}/scripts/python/generate-indices.py

@@ -23,10 +23,13 @@ def main():
 
     for file_name in sql_files:
         file_path = assets_dir / file_name
-        index_df, output_basename, schema = manager.execute_sql_query(file_path)
+        index_df, output_basename, schema, sql_query = manager.execute_sql_query(
+            file_path
+        )
         parquet_file_path = output_dir / f"{output_basename}.parquet"
         index_df.to_parquet(parquet_file_path)
         manager.save_schema_to_json(schema, output_basename, output_dir)
+        manager.save_sql_query(sql_query, output_basename, output_dir)
 
     core_indices_dir = scripts_dir.parent / "scripts" / "sql"
 
@@ -34,10 +37,13 @@ def main():
 
     for file_name in sql_files:
         file_path = core_indices_dir / file_name
-        index_df, output_basename, schema = manager.execute_sql_query(file_path)
+        index_df, output_basename, schema, sql_query = manager.execute_sql_query(
+            file_path
+        )
         parquet_file_path = output_dir / f"{output_basename}.parquet"
         index_df.to_parquet(parquet_file_path)
         manager.save_schema_to_json(schema, output_basename, output_dir)
+        manager.save_sql_query(sql_query, output_basename, output_dir)
 
 
 if __name__ == "__main__":

{idc_index_data-22.1.1 → idc_index_data-22.1.2}/scripts/python/idc_index_data_manager.py

@@ -40,7 +40,7 @@ class IDCIndexDataManager:
             index_df["StudyDate"] = index_df["StudyDate"].astype(str)
         output_basename = Path(file_path).name.split(".")[0]
         logger.debug("Executed SQL query from file: %s", file_path)
-        return index_df, output_basename, schema
+        return index_df, output_basename, schema, sql_query
 
     def save_schema_to_json(
         self,
@@ -79,6 +79,31 @@ class IDCIndexDataManager:
             json.dump(schema_dict, f, indent=2)
         logger.debug("Created schema JSON file: %s", json_file_path)
 
+    def save_sql_query(
+        self,
+        sql_query: str,
+        output_basename: str,
+        output_dir: Path | None = None,
+    ) -> None:
+        """
+        Saves the SQL query to a file.
+
+        Args:
+            sql_query: The SQL query string
+            output_basename: The base name for the output file
+            output_dir: Optional directory path for the output file
+        """
+
+        if output_dir:
+            output_dir.mkdir(parents=True, exist_ok=True)
+            query_file_path = output_dir / f"{output_basename}.sql"
+        else:
+            query_file_path = Path(f"{output_basename}.sql")
+
+        with query_file_path.open("w") as f:
+            f.write(sql_query)
+        logger.debug("Created SQL query file: %s", query_file_path)
+
     def generate_index_data_files(
         self,
         generate_compressed_csv: bool = True,
@@ -108,7 +133,9 @@ class IDCIndexDataManager:
         for file_name in Path.iterdir(sql_dir):
             if str(file_name).endswith(".sql"):
                 file_path = Path(sql_dir) / file_name
-                index_df, output_basename, schema = self.execute_sql_query(file_path)
+                index_df, output_basename, schema, sql_query = self.execute_sql_query(
+                    file_path
+                )
                 logger.debug(
                     "Executed and processed SQL queries from file: %s", file_path
                 )
@@ -132,8 +159,10 @@ class IDCIndexDataManager:
                     index_df.to_parquet(parquet_file_path, compression="zstd")
                     logger.debug("Created Parquet file: %s", parquet_file_path)
 
-                    # Save schema to JSON file
-                    self.save_schema_to_json(schema, output_basename, output_dir)
+                # Save schema to JSON file
+                self.save_schema_to_json(schema, output_basename, output_dir)
+                # Save SQL query to file
+                self.save_sql_query(sql_query, output_basename, output_dir)
 
     def retrieve_latest_idc_release_version(self) -> int:
         """

idc_index_data-22.1.2/scripts/sql/collections_index.sql

@@ -0,0 +1,39 @@
+SELECT
+  # description:
+  # name of the collection
+  collection_name,
+  # description:
+  # unique identifier of the collection
+  collection_id,
+  # description:
+  # types of cancer represented in the collection
+  CancerTypes,
+  # description:
+  # locations of tumors represented in the collection
+  TumorLocations,
+  # description:
+  # number of subjects in the collection
+  Subjects,
+  # description:
+  # species represented in the collection
+  Species,
+  # description:
+  # sources of data for the collection
+  Sources,
+  # description:
+  # additional data supporting the collection available in IDC
+  SupportingData,
+  # description:
+  # broader initiative/category under which this collection is being shared
+  Program,
+  # description:
+  # status of the collection (Completed or Ongoing)
+  Status,
+  # description:
+  # timestamp of the last update to the collection
+  Updated,
+  # description:
+  # detailed information about the collection
+  Description
+FROM
+  `bigquery-public-data.idc_v22.original_collections_metadata`

idc_index_data-22.1.2/scripts/sql/idc_index.sql

@@ -0,0 +1,88 @@
+SELECT
+  # collection level attributes
+  # description:
+  # short string with the identifier of the collection the series belongs to
+  ANY_VALUE(collection_id) AS collection_id,
+  # description:
+  # this string is not empty if the specific series is
+  # part of an analysis results collection; analysis results can be added to a
+  #  given collection over time
+  ANY_VALUE(analysis_result_id) AS analysis_result_id,
+  # description:
+  # identifier of the patient within the collection (DICOM attribute)
+  ANY_VALUE(PatientID) AS PatientID,
+  # description:
+  # unique identifier of the DICOM series (DICOM attribute)
+  SeriesInstanceUID,
+  # description:
+  # unique identifier of the DICOM study (DICOM attribute)
+  ANY_VALUE(StudyInstanceUID) AS StudyInstanceUID,
+  # description:
+  # Digital Object Identifier of the dataset that contains the given
+  # series; follow this DOI to learn more about the activity that produced
+  # this series
+  ANY_VALUE(source_DOI) AS source_DOI,
+  # patient level attributes
+  # description:
+  # age of the subject at the time of imaging (DICOM attribute)
+  ANY_VALUE(PatientAge) AS PatientAge,
+  # description:
+  # subject sex (DICOM attribute)
+  ANY_VALUE(PatientSex) AS PatientSex,
+  # study level attributes
+  # description:
+  # date of the study (de-identified) (DICOM attribute)
+  ANY_VALUE(StudyDate) AS StudyDate,
+  # description:
+  # textual description of the study content (DICOM attribute)
+  ANY_VALUE(StudyDescription) AS StudyDescription,
+  # description:
+  # body part imaged (not iniapplicabletialized for SM series) (DICOM attribute)
+  ANY_VALUE(dicom_curated.BodyPartExamined) AS BodyPartExamined,
+  # series level attributes
+  # description:
+  # acquisition modality (DICOM attribute)
+  ANY_VALUE(Modality) AS Modality,
+  # description:
+  # manufacturer of the equipment that produced the series (DICOM attribute)
+  ANY_VALUE(Manufacturer) AS Manufacturer,
+  # description:
+  # model name of the equipment that produced the series (DICOM attribute)
+  ANY_VALUE(ManufacturerModelName) AS ManufacturerModelName,
+  # description:
+  # date of the series (de-identified) (DICOM attribute)
+  ANY_VALUE(SAFE_CAST(SeriesDate AS STRING)) AS SeriesDate,
+  # description:
+  # textual description of the series content (DICOM attribute)
+  ANY_VALUE(SeriesDescription) AS SeriesDescription,
+  # description:
+  # series number (DICOM attribute)
+  ANY_VALUE(SeriesNumber) AS SeriesNumber,
+  # description:
+  # number of instances in the series
+  COUNT(dicom_all.SOPInstanceUID) AS instanceCount,
+  # description:
+  # short name of the license that applies to this series
+  ANY_VALUE(license_short_name) as license_short_name,
+  # download related attributes
+  # description:
+  # name of the AWS S3 bucket that contains the series
+  ANY_VALUE(aws_bucket)  AS aws_bucket,
+  # description:
+  # unique identifier of the series within the IDC
+  ANY_VALUE(crdc_series_uuid) AS crdc_series_uuid,
+  # series_aws_url will be phased out in favor of constructing URL from bucket+UUID
+  # description:
+  # public AWS S3 URL to download the series in bulk (each instance is a separate file)
+  ANY_VALUE(CONCAT(series_aws_url,"*")) AS series_aws_url,
+  # description:
+  # total size of the series in megabytes
+  ROUND(SUM(SAFE_CAST(instance_size AS float64))/1000000, 2) AS series_size_MB,
+FROM
+  `bigquery-public-data.idc_v22.dicom_all` AS dicom_all
+JOIN
+  `bigquery-public-data.idc_v22.dicom_metadata_curated` AS dicom_curated
+ON
+  dicom_all.SOPInstanceUID = dicom_curated.SOPInstanceUID
+GROUP BY
+  SeriesInstanceUID

idc_index_data-22.1.1/scripts/sql/collections_index.sql

@@ -1,15 +0,0 @@
-SELECT
-  collection_name,
-  collection_id,
-  CancerTypes,
-  TumorLocations,
-  Subjects,
-  Species,
-  Sources,
-  SupportingData,
-  Program,
-  Status,
-  Updated,
-  Description
-FROM
-  `bigquery-public-data.idc_v22.original_collections_metadata`

idc_index_data-22.1.1/scripts/sql/idc_index.sql

@@ -1,38 +0,0 @@
-SELECT
-  # collection level attributes
-  ANY_VALUE(collection_id) AS collection_id,
-  ANY_VALUE(analysis_result_id) AS analysis_result_id,
-  ANY_VALUE(PatientID) AS PatientID,
-  SeriesInstanceUID,
-  ANY_VALUE(StudyInstanceUID) AS StudyInstanceUID,
-  ANY_VALUE(source_DOI) AS source_DOI,
-  # patient level attributes
-  ANY_VALUE(PatientAge) AS PatientAge,
-  ANY_VALUE(PatientSex) AS PatientSex,
-  # study level attributes
-  ANY_VALUE(StudyDate) AS StudyDate,
-  ANY_VALUE(StudyDescription) AS StudyDescription,
-  ANY_VALUE(dicom_curated.BodyPartExamined) AS BodyPartExamined,
-  # series level attributes
-  ANY_VALUE(Modality) AS Modality,
-  ANY_VALUE(Manufacturer) AS Manufacturer,
-  ANY_VALUE(ManufacturerModelName) AS ManufacturerModelName,
-  ANY_VALUE(SAFE_CAST(SeriesDate AS STRING)) AS SeriesDate,
-  ANY_VALUE(SeriesDescription) AS SeriesDescription,
-  ANY_VALUE(SeriesNumber) AS SeriesNumber,
-  COUNT(dicom_all.SOPInstanceUID) AS instanceCount,
-  ANY_VALUE(license_short_name) as license_short_name,
-  # download related attributes
-  ANY_VALUE(aws_bucket)  AS aws_bucket,
-  ANY_VALUE(crdc_series_uuid) AS crdc_series_uuid,
-  # series_aws_url will be phased out in favor of constructing URL from bucket+UUID
-  ANY_VALUE(CONCAT(series_aws_url,"*")) AS series_aws_url,
-  ROUND(SUM(SAFE_CAST(instance_size AS float64))/1000000, 2) AS series_size_MB,
-FROM
-  `bigquery-public-data.idc_v22.dicom_all` AS dicom_all
-JOIN
-  `bigquery-public-data.idc_v22.dicom_metadata_curated` AS dicom_curated
-ON
-  dicom_all.SOPInstanceUID = dicom_curated.SOPInstanceUID
-GROUP BY
-  SeriesInstanceUID