idc-index-data 22.1.4__tar.gz → 23.0.1__tar.gz

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
         args: [--prose-wrap=always]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.14.4"
+    rev: "v0.14.5"
     hooks:
       - id: ruff-check
         args: ["--fix", "--show-fixes"]
@@ -76,12 +76,12 @@ repos:
         exclude: .pre-commit-config.yaml
 
   - repo: https://github.com/henryiii/validate-pyproject-schema-store
-    rev: "2025.11.04"
+    rev: "2025.11.14"
     hooks:
       - id: validate-pyproject
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: "0.34.1"
+    rev: "0.35.0"
     hooks:
       - id: check-dependabot
       - id: check-github-workflows

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: idc-index-data
-Version: 22.1.4
+Version: 23.0.1
 Summary: ImagingDataCommons index to query and download data.
 Author-Email: Andrey Fedorov <andrey.fedorov@gmail.com>, Vamsi Thiriveedhi <vthiriveedhi@mgh.harvard.edu>, Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
 License: Copyright 2024 Andrey Fedorov

idc_index_data-23.0.1/assets/clinical_index.sql

@@ -0,0 +1,30 @@
+# table-description:
+# This table contains metadata about the tabular data, including clinical data, accompanying images that
+# is available in IDC. Think about this table as a dictionary containing information about the columns
+# for all of the tabular data accompanying individual collections in IDC. Each row corresponds to a unique
+# combination of collection, clinical data table that is available for that collection, and a column from that
+# table. Individual tables referenced from this table can be retrieved using idc-index `get_clinical_table()`
+# function.
+SELECT
+  # description:
+  # unique identifier of the collection
+  collection_id,
+  # description:
+  # full name of the table in which the column is stored
+  table_name,
+  # description:
+  # short name of the table in which the column is stored
+  SPLIT(table_name,'.')[SAFE_OFFSET(2)] AS short_table_name,
+  # description:
+  # name of the column in which the value is stored
+  `column`,
+  # description:
+  # human readable name of the column
+  column_label,
+  # description:
+  # values encountered in the column
+  `values`
+FROM
+  `bigquery-public-data.idc_v23_clinical.column_metadata`
+ORDER BY
+  collection_id, table_name

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/assets/sm_index.sql

@@ -9,6 +9,12 @@
 -- WHERE
 --   Modality = "SM"
 
+# table-description:
+# This table contains metadata about the slide microscopy (SM) series available in IDC. Each row
+# corresponds to a DICOM series, and contains attributes specific to SM series, such as the pixel spacing at the maximum
+# resolution layer, the power of the objective lens used to digitize the slide, and the anatomic location
+# from where the imaged specimen was collected. This table can be joined with the main index table using the
+# `SeriesInstanceUID` column.
 WITH
   temp_table AS (
   SELECT
@@ -31,7 +37,7 @@ WITH
 
 
   FROM
-    `bigquery-public-data.idc_v22.dicom_all` AS dicom_all
+    `bigquery-public-data.idc_v23.dicom_all` AS dicom_all
   GROUP BY
     SeriesInstanceUID
   ),
@@ -45,7 +51,7 @@ SpecimenPreparationSequence_unnested AS (
         concept_code_sequence.CodeMeaning AS ccs_cm,
         concept_code_sequence.CodingSchemeDesignator AS ccs_csd,
         concept_code_sequence.CodeValue AS ccs_val,
-      FROM `bigquery-public-data.idc_v22.dicom_all`,
+      FROM `bigquery-public-data.idc_v23.dicom_all`,
       UNNEST(SpecimenDescriptionSequence[SAFE_OFFSET(0)].SpecimenPreparationSequence) as preparation_unnest_step1,
       UNNEST(preparation_unnest_step1.SpecimenPreparationStepContentItemSequence) as preparation_unnest_step2,
       UNNEST(preparation_unnest_step2.ConceptNameCodeSequence) as concept_name_code_sequence,
@@ -80,6 +86,8 @@ SpecimenPreparationSequence_unnested AS (
     )
 
 SELECT
+  # description:
+  # DICOM SeriesInstanceUID identifier of the series
   temp_table.SeriesInstanceUID,
   -- Embedding Medium
   # description:

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/assets/sm_instance_index.sql

@@ -1,3 +1,10 @@
+# table-description:
+# This table contains metadata about the slide microscopy (SM) series available in IDC. Each row
+# corresponds to an instance from a DICOM Slide Microscopy series available from IDC, identified by
+# `SOPInstanceUID`, and contains attributes specific to SM series, such as the pixel spacing at the maximum
+# resolution layer, the power of the objective lens used to digitize the slide, and the anatomic location
+# from where the imaged specimen was collected. This table can be joined with the main index table
+# and/or with `sm_index` using the `SeriesInstanceUID` column.
 WITH
   SpecimenPreparationSequence_unnested AS (
   SELECT
@@ -9,7 +16,7 @@ WITH
     concept_code_sequence.CodingSchemeDesignator AS ccs_csd,
     concept_code_sequence.CodeValue AS ccs_val,
   FROM
-    `bigquery-public-data.idc_v22.dicom_all`,
+    `bigquery-public-data.idc_v23.dicom_all`,
     UNNEST(SpecimenDescriptionSequence[SAFE_OFFSET(0)].SpecimenPreparationSequence) AS preparation_unnest_step1,
     UNNEST(preparation_unnest_step1.SpecimenPreparationStepContentItemSequence) AS preparation_unnest_step2,
     UNNEST(preparation_unnest_step2.ConceptNameCodeSequence) AS concept_name_code_sequence,
@@ -137,7 +144,7 @@ SELECT
   # unique identifier of the instance within the IDC
   dicom_all.crdc_instance_uuid AS crdc_instance_uuid
 FROM
-  `bigquery-public-data.idc_v22.dicom_all` AS dicom_all
+  `bigquery-public-data.idc_v23.dicom_all` AS dicom_all
 LEFT JOIN
   slide_embedding
 ON

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/pyproject.toml

@@ -13,7 +13,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "idc-index-data"
-version = "22.1.4"
+version = "23.0.1"
 authors = [
   { name = "Andrey Fedorov", email = "andrey.fedorov@gmail.com" },
   { name = "Vamsi Thiriveedhi", email = "vthiriveedhi@mgh.harvard.edu" },

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/scripts/python/idc_index_data_manager.py

@@ -22,6 +22,51 @@ class IDCIndexDataManager:
         self.client = bigquery.Client(project=project_id)
         logger.debug("IDCIndexDataManager initialized with project ID: %s", project_id)
 
+    @staticmethod
+    def parse_table_description(sql_query: str) -> str:
+        """
+        Parses the table description from SQL query comments.
+
+        The method looks for comments following the pattern:
+        # table-description:
+        # description text continues here
+        # and can span multiple lines
+
+        Args:
+            sql_query: The SQL query string containing comments
+
+        Returns:
+            The table description as a string
+        """
+        description_lines = []
+        logger.debug("Parsing table description from SQL query comments")
+        logger.debug(sql_query)
+        lines = sql_query.split("\n")
+
+        for i, line in enumerate(lines):
+            stripped = line.strip()
+            if stripped == "# table-description:":
+                # Collect description lines until we hit a non-comment line
+                j = i + 1
+                while j < len(lines):
+                    next_line = lines[j]
+                    next_stripped = next_line.strip()
+                    if next_stripped.startswith("#") and next_stripped != "#":
+                        # Remove the leading # and whitespace
+                        desc_text = next_stripped[1:].strip()
+                        if desc_text:
+                            description_lines.append(desc_text)
+                        j += 1
+                    elif next_stripped.startswith("#"):
+                        # Empty comment line, skip
+                        j += 1
+                    else:
+                        # Non-comment line, stop collecting
+                        break
+                break
+
+        return " ".join(description_lines)
+
     @staticmethod
     def parse_column_descriptions(sql_query: str) -> dict[str, str]:
         """
@@ -232,11 +277,14 @@ class IDCIndexDataManager:
         logger.debug("Parsing column descriptions from SQL query comments")
         logger.debug(sql_query)
         if sql_query is not None:
+            table_description = self.parse_table_description(sql_query)
+            logger.debug("Parsed table description: %s", table_description)
             descriptions = self.parse_column_descriptions(sql_query)
 
             # Convert BigQuery schema to JSON-serializable format
             schema_dict = {
-                "fields": [
+                "table_description": table_description,
+                "columns": [
                     {
                         "name": field.name,
                         "type": field.field_type,
@@ -244,12 +292,12 @@ class IDCIndexDataManager:
                         "description": descriptions.get(field.name, ""),
                     }
                     for field in schema
-                ]
+                ],
             }
         else:
             # If no SQL query provided, save schema without descriptions
             schema_dict = {
-                "fields": [
+                "columns": [
                     {
                         "name": field.name,
                         "type": field.field_type,
@@ -375,7 +423,7 @@ class IDCIndexDataManager:
         SELECT
             MAX(idc_version) AS latest_idc_release_version
         FROM
-            `bigquery-public-data.idc_current.version_metadata`
+            `bigquery-public-data.idc_v23.version_metadata`
         """
         query_job = self.client.query(query)
         result = query_job.result()

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/scripts/sql/analysis_results_index.sql

@@ -1,3 +1,7 @@
+# table-description:
+# This table contains metadata about the analysis results collections available in IDC. Each row corresponds to an
+# analysis results collection, and contains attributes such as the collection name, types of cancer represented,
+# number of subjects, and pointers to the resources to learn more about the content of the collection
 SELECT
   # description:
   # unique identifier of the analysis results collection
@@ -39,4 +43,4 @@ SELECT
   # citation for the analysis results collection that should be used for acknowledgment
   Citation
 FROM
-  `bigquery-public-data.idc_v22.analysis_results_metadata`
+  `bigquery-public-data.idc_v23.analysis_results_metadata`

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/scripts/sql/collections_index.sql

@@ -1,3 +1,7 @@
+# table-description:
+# This table contains metadata about the collections available in IDC. Each row corresponds to a collection,
+# and contains attributes such as the collection name, types of cancer represented, number of subjects,
+# and pointers to the resources to learn more about the content of the collection.
 SELECT
   # description:
   # name of the collection
@@ -36,4 +40,4 @@ SELECT
   # detailed information about the collection
   Description
 FROM
-  `bigquery-public-data.idc_v22.original_collections_metadata`
+  `bigquery-public-data.idc_v23.original_collections_metadata`

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/scripts/sql/idc_index.sql

@@ -1,3 +1,7 @@
+# table-description:
+# This is the main metadata table provided by idc-index. Each row corresponds to a DICOM series, and contains
+# attributes at the collection, patient, study, and series levels. The table also contains download-related
+# attributes, such as the AWS S3 bucket and URL to download the series.
 SELECT
   # collection level attributes
   # description:
@@ -22,7 +26,7 @@ SELECT
   # series; follow this DOI to learn more about the activity that produced
   # this series
   ANY_VALUE(source_DOI) AS source_DOI,
-  # patient level attributes
+  # patient level attributes:
   # description:
   # age of the subject at the time of imaging (DICOM attribute)
   ANY_VALUE(PatientAge) AS PatientAge,
@@ -79,9 +83,9 @@ SELECT
   # total size of the series in megabytes
   ROUND(SUM(SAFE_CAST(instance_size AS float64))/1000000, 2) AS series_size_MB,
 FROM
-  `bigquery-public-data.idc_v22.dicom_all` AS dicom_all
+  `bigquery-public-data.idc_v23.dicom_all` AS dicom_all
 JOIN
-  `bigquery-public-data.idc_v22.dicom_metadata_curated` AS dicom_curated
+  `bigquery-public-data.idc_v23.dicom_metadata_curated` AS dicom_curated
 ON
   dicom_all.SOPInstanceUID = dicom_curated.SOPInstanceUID
 GROUP BY

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/scripts/sql/prior_versions_index.sql

@@ -3,7 +3,7 @@
 --
 -- Step 1: Declare variables
 DECLARE idc_versions ARRAY<INT64>;
-DECLARE latest_idc_version INT64 DEFAULT 22;
+DECLARE latest_idc_version INT64 DEFAULT 23;
 DECLARE union_all_query STRING;
 
 --Step 2

{idc_index_data-22.1.4 → idc_index_data-23.0.1}/tests/test_package.py

@@ -7,7 +7,7 @@ from packaging.version import Version
 
 import idc_index_data as m
 
-EXPECTED_IDC_INDEX_VERSION = 22
+EXPECTED_IDC_INDEX_VERSION = 23
 
 
 def test_version():

idc_index_data-22.1.4/assets/clinical_index.sql

@@ -1,23 +0,0 @@
-SELECT
-  # description:
-  # unique identifier of the collection
-  collection_id,
-  # description:
-  # full name of the table in which the column is stored
-  table_name,
-  # description:
-  # short name of the table in which the column is stored
-  SPLIT(table_name,'.')[SAFE_OFFSET(2)] AS short_table_name,
-  # description:
-  # name of the column in which the value is stored
-  `column`,
-  # description:
-  # human readable name of the column
-  column_label,
-  # description:
-  # values encountered in the column
-  `values`
-FROM
-  `bigquery-public-data.idc_v22_clinical.column_metadata`
-ORDER BY
-  collection_id, table_name