idc-index-data 18.0.0__tar.gz → 18.1.0__tar.gz

idc_index_data-18.1.0/.github/workflows/external-indices.yml

@@ -0,0 +1,60 @@
+name: CD-external-indices
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+  release:
+    types:
+      - published
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: 3
+
+jobs:
+  dist:
+    name: CD-external-indices
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install db-dtypes google-cloud-bigquery pandas pyarrow
+
+      - name: Authorize Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: "${{ secrets.SERVICE_ACCOUNT_KEY }}"
+          create_credentials_file: true
+          export_environment_variables: true
+
+      - name: Execute SQL Query and Generate Parquet Files
+        run: |
+          python scripts/python/external-indices.py
+        env:
+          PROJECT_ID: ${{ env.GCP_PROJECT }}
+
+      - name: Create Tagged Release
+        id: create_tagged_release
+        if: github.event_name == 'release' && github.event.action == 'published'
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "*.parquet"
+          allowUpdates: true
+          omitBodyDuringUpdate: true

{idc_index_data-18.0.0 → idc_index_data-18.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: idc-index-data
-Version: 18.0.0
+Version: 18.1.0
 Summary: ImagingDataCommons index to query and download data.
 Author-Email: Andrey Fedorov <andrey.fedorov@gmail.com>, Vamsi Thiriveedhi <vthiriveedhi@mgh.harvard.edu>, Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
 License: Copyright 2024 Andrey Fedorov

idc_index_data-18.1.0/assets/README.md

@@ -0,0 +1,3 @@
+This folder contains SQL scripts that are used to generate tables that are
+attached to the releases as assets. Initially, those will be generated and
+attached manually, but in the future this process may be automated.

idc_index_data-18.1.0/assets/sm_index.sql

@@ -0,0 +1,132 @@
+-- Note that this query can be substituted with a much simpler one below
+-- once this PR is merged and https://github.com/ImagingDataCommons/etl_flow/pull/104
+-- the latter makes it to a public release
+--
+-- SELECT
+--   * EXCEPT(Modality)
+-- FROM
+--   `bigquery-public-data.idc_v18.dicom_metadata_curated_series_level`
+-- WHERE
+--   Modality = "SM"
+
+WITH
+  temp_table AS (
+  SELECT
+    dicom_all.SeriesInstanceUID,
+    ANY_VALUE(Modality) AS Modality,
+    STRING_AGG(DISTINCT(collection_id),",") AS collection_id,
+    ANY_VALUE(OpticalPathSequence[SAFE_OFFSET(0)].ObjectiveLensPower) AS ObjectiveLensPower,
+    MAX(DISTINCT(TotalPixelMatrixColumns)) AS max_TotalPixelMatrixColumns,
+    MAX(DISTINCT(TotalPixelMatrixRows)) AS max_TotalPixelMatrixRows,
+    MAX(DISTINCT(`Columns`)) AS max_Columns,
+    MAX(DISTINCT(`Rows`)) AS max_Rows,
+    MIN(DISTINCT(SAFE_CAST(PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64))) AS min_spacing_0,
+    MIN(SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64)) AS fg_min_spacing_0,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS primaryAnatomicStructure_code_str,
+    ARRAY_AGG(DISTINCT(CONCAT(SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeValue, ":", SpecimenDescriptionSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureSequence[SAFE_OFFSET(0)].PrimaryAnatomicStructureModifierSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS primaryAnatomicStructureModifier_code_str,
+
+    ARRAY_AGG(DISTINCT(CONCAT(OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodingSchemeDesignator,":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeValue, ":", OpticalPathSequence[SAFE_OFFSET(0)].IlluminationTypeCodeSequence[SAFE_OFFSET(0)].CodeMeaning)) IGNORE NULLS)[SAFE_OFFSET(0)] AS illuminationType_code_str,
+  FROM
+    `bigquery-public-data.idc_v18.dicom_all` AS dicom_all
+  GROUP BY
+    SeriesInstanceUID
+  ),
+
+SpecimenPreparationSequence_unnested AS (
+      SELECT
+        SeriesInstanceUID,
+        concept_name_code_sequence.CodeMeaning AS cnc_cm,
+        concept_name_code_sequence.CodingSchemeDesignator AS cnc_csd,
+        concept_name_code_sequence.CodeValue AS cnc_val,
+        concept_code_sequence.CodeMeaning AS ccs_cm,
+        concept_code_sequence.CodingSchemeDesignator AS ccs_csd,
+        concept_code_sequence.CodeValue AS ccs_val,
+      FROM `bigquery-public-data.idc_v18.dicom_all`,
+      UNNEST(SpecimenDescriptionSequence[SAFE_OFFSET(0)].SpecimenPreparationSequence) as preparation_unnest_step1,
+      UNNEST(preparation_unnest_step1.SpecimenPreparationStepContentItemSequence) as preparation_unnest_step2,
+      UNNEST(preparation_unnest_step2.ConceptNameCodeSequence) as concept_name_code_sequence,
+      UNNEST(preparation_unnest_step2.ConceptCodeSequence) as concept_code_sequence
+    ),
+
+    slide_embedding AS (
+    SELECT
+      SeriesInstanceUID,
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val))) as embeddingMedium_code_str
+    FROM SpecimenPreparationSequence_unnested
+    WHERE (cnc_csd = 'SCT' and cnc_val = '430863003') -- CodeMeaning is 'Embedding medium'
+    GROUP BY SeriesInstanceUID
+    ),
+
+    slide_fixative AS (
+    SELECT
+      SeriesInstanceUID,
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) as tissueFixative_code_str
+    FROM SpecimenPreparationSequence_unnested
+    WHERE (cnc_csd = 'SCT' and cnc_val = '430864009') -- CodeMeaning is 'Tissue Fixative'
+    GROUP BY SeriesInstanceUID
+    ),
+
+    slide_staining AS (
+    SELECT
+      SeriesInstanceUID,
+      ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) as staining_usingSubstance_code_str,
+    FROM SpecimenPreparationSequence_unnested
+    WHERE (cnc_csd = 'SCT' and cnc_val = '424361007') -- CodeMeaning is 'Using substance'
+    GROUP BY SeriesInstanceUID
+    )
+
+SELECT
+  temp_table.SeriesInstanceUID,
+  -- Embedding Medium
+  ARRAY(
+    SELECT IF(code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
+    FROM UNNEST(embeddingMedium_code_str) AS code
+  ) AS embeddingMedium_CodeMeaning,
+  ARRAY(
+    SELECT IF(code IS NULL, NULL,
+              IF(STRPOS(code, ':') = 0, NULL,
+                 SUBSTR(code, STRPOS(code, ':') + 1)))
+    FROM UNNEST(embeddingMedium_code_str) AS code
+  ) AS embeddingMedium_code_designator_value_str,
+  -- Tissue Fixative
+  ARRAY(
+    SELECT IF(code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
+    FROM UNNEST(tissueFixative_code_str) AS code
+  ) AS tissueFixative_CodeMeaning,
+  ARRAY(
+    SELECT IF(code IS NULL, NULL,
+              IF(STRPOS(code, ':') = 0, NULL,
+                 SUBSTR(code, STRPOS(code, ':') + 1)))
+    FROM UNNEST(tissueFixative_code_str) AS code
+  ) AS tissueFixative_code_designator_value_str,
+  -- Staining using substance
+  ARRAY(
+    SELECT IF(code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
+    FROM UNNEST(staining_usingSubstance_code_str) AS code
+  ) AS staining_usingSubstance_CodeMeaning,
+  ARRAY(
+    SELECT IF(code IS NULL, NULL,
+              IF(STRPOS(code, ':') = 0, NULL,
+                 SUBSTR(code, STRPOS(code, ':') + 1)))
+    FROM UNNEST(staining_usingSubstance_code_str) AS code
+  ) AS staining_usingSubstance_code_designator_value_str,
+
+  if(COALESCE(min_spacing_0, fg_min_spacing_0) = 0, 0,
+    round(COALESCE(min_spacing_0, fg_min_spacing_0) ,CAST(2 -1-floor(log10(abs(COALESCE(min_spacing_0, fg_min_spacing_0) ))) AS INT64))) AS min_PixelSpacing_2sf,
+  COALESCE(max_TotalPixelMatrixColumns, max_Columns) AS max_TotalPixelMatrixColumns,
+  COALESCE(max_TotalPixelMatrixRows, max_Rows) AS max_TotalPixelMatrixRows,
+  SAFE_CAST(ObjectiveLensPower as INT) as ObjectiveLensPower,
+  CONCAT(SPLIT(primaryAnatomicStructure_code_str,":")[SAFE_OFFSET(0)],":",SPLIT(primaryAnatomicStructure_code_str,":")[SAFE_OFFSET(1)]) as primaryAnatomicStructure_code_designator_value_str,
+  SPLIT(primaryAnatomicStructure_code_str,":")[SAFE_OFFSET(2)] as primaryAnatomicStructure_CodeMeaning,
+  CONCAT(SPLIT(primaryAnatomicStructureModifier_code_str,":")[SAFE_OFFSET(0)],":",SPLIT(primaryAnatomicStructureModifier_code_str,":")[SAFE_OFFSET(1)]) as primaryAnatomicStructureModifier_code_designator_value_str,
+  SPLIT(primaryAnatomicStructureModifier_code_str,":")[SAFE_OFFSET(2)] as primaryAnatomicStructureModifier_CodeMeaning,
+
+  CONCAT(SPLIT(illuminationType_code_str,":")[SAFE_OFFSET(0)],":",SPLIT(illuminationType_code_str,":")[SAFE_OFFSET(1)]) as illuminationType_code_designator_value_str,
+  SPLIT(illuminationType_code_str,":")[SAFE_OFFSET(2)] as illuminationType_CodeMeaning,
+FROM
+  temp_table
+LEFT JOIN slide_embedding on temp_table.SeriesInstanceUID = slide_embedding.SeriesInstanceUID
+LEFT JOIN slide_fixative on temp_table.SeriesInstanceUID = slide_fixative.SeriesInstanceUID
+LEFT JOIN slide_staining on temp_table.SeriesInstanceUID = slide_staining.SeriesInstanceUID
+WHERE
+  Modality = "SM"

idc_index_data-18.1.0/assets/sm_instance_index.sql

@@ -0,0 +1,124 @@
+WITH
+  SpecimenPreparationSequence_unnested AS (
+  SELECT
+    SOPInstanceUID,
+    concept_name_code_sequence.CodeMeaning AS cnc_cm,
+    concept_name_code_sequence.CodingSchemeDesignator AS cnc_csd,
+    concept_name_code_sequence.CodeValue AS cnc_val,
+    concept_code_sequence.CodeMeaning AS ccs_cm,
+    concept_code_sequence.CodingSchemeDesignator AS ccs_csd,
+    concept_code_sequence.CodeValue AS ccs_val,
+  FROM
+    `bigquery-public-data.idc_v18.dicom_all`,
+    UNNEST(SpecimenDescriptionSequence[SAFE_OFFSET(0)].SpecimenPreparationSequence) AS preparation_unnest_step1,
+    UNNEST(preparation_unnest_step1.SpecimenPreparationStepContentItemSequence) AS preparation_unnest_step2,
+    UNNEST(preparation_unnest_step2.ConceptNameCodeSequence) AS concept_name_code_sequence,
+    UNNEST(preparation_unnest_step2.ConceptCodeSequence) AS concept_code_sequence ),
+  slide_embedding AS (
+  SELECT
+    SOPInstanceUID,
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm,":",ccs_csd,":",ccs_val))) AS embeddingMedium_code_str
+  FROM
+    SpecimenPreparationSequence_unnested
+  WHERE
+    (cnc_csd = 'SCT'
+      AND cnc_val = '430863003') -- CodeMeaning is 'Embedding medium'
+  GROUP BY
+    SOPInstanceUID ),
+  slide_fixative AS (
+  SELECT
+    SOPInstanceUID,
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) AS tissueFixative_code_str
+  FROM
+    SpecimenPreparationSequence_unnested
+  WHERE
+    (cnc_csd = 'SCT'
+      AND cnc_val = '430864009') -- CodeMeaning is 'Tissue Fixative'
+  GROUP BY
+    SOPInstanceUID ),
+  slide_staining AS (
+  SELECT
+    SOPInstanceUID,
+    ARRAY_AGG(DISTINCT(CONCAT(ccs_cm, ":", ccs_csd,":",ccs_val))) AS staining_usingSubstance_code_str,
+  FROM
+    SpecimenPreparationSequence_unnested
+  WHERE
+    (cnc_csd = 'SCT'
+      AND cnc_val = '424361007') -- CodeMeaning is 'Using substance'
+  GROUP BY
+    SOPInstanceUID )
+SELECT
+  dicom_all.SOPInstanceUID,
+  dicom_all.SeriesInstanceUID,
+  -- Embedding Medium
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
+  FROM
+    UNNEST(embeddingMedium_code_str) AS code ) AS embeddingMedium_CodeMeaning,
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL,
+    IF
+      (STRPOS(code, ':') = 0, NULL, SUBSTR(code, STRPOS(code, ':') + 1)))
+  FROM
+    UNNEST(embeddingMedium_code_str) AS code ) AS embeddingMedium_code_designator_value_str,
+  -- Tissue Fixative
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
+  FROM
+    UNNEST(tissueFixative_code_str) AS code ) AS tissueFixative_CodeMeaning,
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL,
+    IF
+      (STRPOS(code, ':') = 0, NULL, SUBSTR(code, STRPOS(code, ':') + 1)))
+  FROM
+    UNNEST(tissueFixative_code_str) AS code ) AS tissueFixative_code_designator_value_str,
+  -- Staining using substance
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
+  FROM
+    UNNEST(staining_usingSubstance_code_str) AS code ) AS staining_usingSubstance_CodeMeaning,
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL,
+    IF
+      (STRPOS(code, ':') = 0, NULL, SUBSTR(code, STRPOS(code, ':') + 1)))
+  FROM
+    UNNEST(staining_usingSubstance_code_str) AS code ) AS staining_usingSubstance_code_designator_value_str,
+  -- instance-specific image attributes
+  -- NB: there is a caveat that I think in general, we expect square pixels, but in htan_wustl and cptac_luad this assumption does not hold,
+  -- and in htan_wustl, the difference is rather large (x2) - waiting to hear from David Clunie about this...
+  SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64) AS PixelSpacing_0,
+  dicom_all.ImageType,
+  dicom_all.TransferSyntaxUID,
+  dicom_all.instance_size,
+  -- attributes needed to retrieve the selected instances/files
+  dicom_all.crdc_instance_uuid
+FROM
+  `bigquery-public-data.idc_current.dicom_all` AS dicom_all
+LEFT JOIN
+  slide_embedding
+ON
+  dicom_all.SOPInstanceUID = slide_embedding.SOPInstanceUID
+LEFT JOIN
+  slide_fixative
+ON
+  dicom_all.SOPInstanceUID = slide_fixative.SOPInstanceUID
+LEFT JOIN
+  slide_staining
+ON
+  dicom_all.SOPInstanceUID = slide_staining.SOPInstanceUID
+WHERE
+  dicom_all.Modality="SM"
+ORDER BY
+  SeriesInstanceUID DESC

{idc_index_data-18.0.0 → idc_index_data-18.1.0}/pyproject.toml

@@ -13,7 +13,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "idc-index-data"
-version = "18.0.0"
+version = "18.1.0"
 authors = [
   { name = "Andrey Fedorov", email = "andrey.fedorov@gmail.com" },
   { name = "Vamsi Thiriveedhi", email = "vthiriveedhi@mgh.harvard.edu" },

idc_index_data-18.1.0/scripts/python/external-indices.py

@@ -0,0 +1,26 @@
+# new_script.py
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from idc_index_data_manager import IDCIndexDataManager
+
+
+def main():
+    project_id = os.getenv("PROJECT_ID")
+    manager = IDCIndexDataManager(project_id=project_id)
+    scripts_dir = Path(__file__).resolve().parent.parent
+    assets_dir = scripts_dir.parent / "assets"
+
+    # Collecting all .sql files from sql_dir and assets_dir
+    sql_files = [f for f in os.listdir(assets_dir) if f.endswith(".sql")]
+
+    for file_name in sql_files:
+        file_path = assets_dir / file_name
+        index_df, output_basename = manager.execute_sql_query(file_path)
+        index_df.to_parquet(f"{output_basename}.parquet")
+
+
+if __name__ == "__main__":
+    main()

{idc_index_data-18.0.0 → idc_index_data-18.1.0}/scripts/sql/idc_index.sql

@@ -1,6 +1,7 @@
 SELECT
   # collection level attributes
   ANY_VALUE(collection_id) AS collection_id,
+  ANY_VALUE(analysis_result_id) AS analysis_result_id,
   ANY_VALUE(PatientID) AS PatientID,
   SeriesInstanceUID,
   ANY_VALUE(StudyInstanceUID) AS StudyInstanceUID,