idc-index-data 22.1.3__tar.gz → 22.1.5__tar.gz

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
         args: [--prose-wrap=always]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.14.4"
+    rev: "v0.14.5"
     hooks:
       - id: ruff-check
         args: ["--fix", "--show-fixes"]
@@ -76,12 +76,12 @@ repos:
         exclude: .pre-commit-config.yaml
 
   - repo: https://github.com/henryiii/validate-pyproject-schema-store
-    rev: "2025.11.04"
+    rev: "2025.11.14"
     hooks:
       - id: validate-pyproject
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: "0.34.1"
+    rev: "0.35.0"
     hooks:
       - id: check-dependabot
       - id: check-github-workflows

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: idc-index-data
-Version: 22.1.3
+Version: 22.1.5
 Summary: ImagingDataCommons index to query and download data.
 Author-Email: Andrey Fedorov <andrey.fedorov@gmail.com>, Vamsi Thiriveedhi <vthiriveedhi@mgh.harvard.edu>, Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
 License: Copyright 2024 Andrey Fedorov

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/assets/clinical_index.sql

@@ -1,3 +1,10 @@
+# table-description:
+# This table contains metadata about the tabular data, including clinical data, accompanying images that
+# is available in IDC. Think about this table as a dictionary containing information about the columns
+# for all of the tabular data accompanying individual collections in IDC. Each row corresponds to a unique
+# combination of collection, clinical data table that is available for that collection, and a column from that
+# table. Individual tables referenced from this table can be retrieved using idc-index `get_clinical_table()`
+# function.
 SELECT
   # description:
   # unique identifier of the collection

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/assets/sm_index.sql

@@ -9,6 +9,12 @@
 -- WHERE
 --   Modality = "SM"
 
+# table-description:
+# This table contains metadata about the slide microscopy (SM) series available in IDC. Each row
+# corresponds to a DICOM series, and contains attributes specific to SM series, such as the pixel spacing at the maximum
+# resolution layer, the power of the objective lens used to digitize the slide, and the anatomic location
+# from where the imaged specimen was collected. This table can be joined with the main index table using the
+# `SeriesInstanceUID` column.
 WITH
   temp_table AS (
   SELECT

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/assets/sm_instance_index.sql

@@ -1,3 +1,10 @@
+# table-description:
+# This table contains metadata about the slide microscopy (SM) series available in IDC. Each row
+# corresponds to an instance from a DICOM Slide Microscopy series available from IDC, identified by
+# `SOPInstanceUID`, and contains attributes specific to SM series, such as the pixel spacing at the maximum
+# resolution layer, the power of the objective lens used to digitize the slide, and the anatomic location
+# from where the imaged specimen was collected. This table can be joined with the main index table
+# and/or with `sm_index` using the `SeriesInstanceUID` column.
 WITH
   SpecimenPreparationSequence_unnested AS (
   SELECT
@@ -50,10 +57,10 @@ WITH
 SELECT
   # description:
   # unique identifier of the instance
-  dicom_all.SOPInstanceUID,
+  dicom_all.SOPInstanceUID AS SOPInstanceUID,
   # description:
   # unique identifier of the series
-  dicom_all.SeriesInstanceUID,
+  dicom_all.SeriesInstanceUID AS SeriesInstanceUID,
   -- Embedding Medium
   # description:
   # embedding medium used for the slide preparation
@@ -119,23 +126,23 @@ SELECT
   SAFE_CAST(SharedFunctionalGroupsSequence[SAFE_OFFSET(0)].PixelMeasuresSequence[SAFE_OFFSET(0)]. PixelSpacing[SAFE_OFFSET(0)] AS FLOAT64) AS PixelSpacing_0,
   # description:
   # DICOM ImageType attribute
-  dicom_all.ImageType,
+  dicom_all.ImageType AS ImageType,
   # description:
   # DICOM TransferSyntaxUID attribute
-  dicom_all.TransferSyntaxUID,
+  dicom_all.TransferSyntaxUID AS TransferSyntaxUID,
   # description:
   # size of the instance file in bytes
-  dicom_all.instance_size,
+  dicom_all.instance_size AS instance_size,
   # description:
   # number of columns in the image
-  dicom_all.TotalPixelMatrixColumns,
+  dicom_all.TotalPixelMatrixColumns AS TotalPixelMatrixColumns,
   # description:
   # number of rows in the image
-  dicom_all.TotalPixelMatrixRows,
+  dicom_all.TotalPixelMatrixRows AS TotalPixelMatrixRows,
   -- attributes needed to retrieve the selected instances/files
   # description:
   # unique identifier of the instance within the IDC
-  dicom_all.crdc_instance_uuid
+  dicom_all.crdc_instance_uuid AS crdc_instance_uuid
 FROM
   `bigquery-public-data.idc_v22.dicom_all` AS dicom_all
 LEFT JOIN

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/pyproject.toml

@@ -13,7 +13,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "idc-index-data"
-version = "22.1.3"
+version = "22.1.5"
 authors = [
   { name = "Andrey Fedorov", email = "andrey.fedorov@gmail.com" },
   { name = "Vamsi Thiriveedhi", email = "vthiriveedhi@mgh.harvard.edu" },

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/scripts/python/idc_index_data_manager.py

@@ -22,6 +22,51 @@ class IDCIndexDataManager:
         self.client = bigquery.Client(project=project_id)
         logger.debug("IDCIndexDataManager initialized with project ID: %s", project_id)
 
+    @staticmethod
+    def parse_table_description(sql_query: str) -> str:
+        """
+        Parses the table description from SQL query comments.
+
+        The method looks for comments following the pattern:
+        # table-description:
+        # description text continues here
+        # and can span multiple lines
+
+        Args:
+            sql_query: The SQL query string containing comments
+
+        Returns:
+            The table description as a string
+        """
+        description_lines = []
+        logger.debug("Parsing table description from SQL query comments")
+        logger.debug(sql_query)
+        lines = sql_query.split("\n")
+
+        for i, line in enumerate(lines):
+            stripped = line.strip()
+            if stripped == "# table-description:":
+                # Collect description lines until we hit a non-comment line
+                j = i + 1
+                while j < len(lines):
+                    next_line = lines[j]
+                    next_stripped = next_line.strip()
+                    if next_stripped.startswith("#") and next_stripped != "#":
+                        # Remove the leading # and whitespace
+                        desc_text = next_stripped[1:].strip()
+                        if desc_text:
+                            description_lines.append(desc_text)
+                        j += 1
+                    elif next_stripped.startswith("#"):
+                        # Empty comment line, skip
+                        j += 1
+                    else:
+                        # Non-comment line, stop collecting
+                        break
+                break
+
+        return " ".join(description_lines)
+
     @staticmethod
     def parse_column_descriptions(sql_query: str) -> dict[str, str]:
         """
@@ -130,10 +175,16 @@ class IDCIndexDataManager:
                         logger.debug(
                             "Parsed description for column '%s': %s",
                             column_name,
-                            description[:50] + "..."
-                            if len(description) > 50
-                            else description,
+                            description,
                         )
+                        # throw exception if description is empty
+                        if not description:
+                            raise ValueError(
+                                "Description for column '"
+                                + column_name
+                                + "' is empty, and empty descriptions are not allowed."
+                            )
+
                 else:
                     i += 1
             else:
@@ -226,11 +277,14 @@ class IDCIndexDataManager:
         logger.debug("Parsing column descriptions from SQL query comments")
         logger.debug(sql_query)
         if sql_query is not None:
+            table_description = self.parse_table_description(sql_query)
+            logger.debug("Parsed table description: %s", table_description)
             descriptions = self.parse_column_descriptions(sql_query)
 
             # Convert BigQuery schema to JSON-serializable format
             schema_dict = {
-                "fields": [
+                "table_description": table_description,
+                "columns": [
                     {
                         "name": field.name,
                         "type": field.field_type,
@@ -238,12 +292,12 @@ class IDCIndexDataManager:
                         "description": descriptions.get(field.name, ""),
                     }
                     for field in schema
-                ]
+                ],
             }
         else:
             # If no SQL query provided, save schema without descriptions
             schema_dict = {
-                "fields": [
+                "columns": [
                     {
                         "name": field.name,
                         "type": field.field_type,

idc_index_data-22.1.5/scripts/sql/analysis_results_index.sql

@@ -0,0 +1,46 @@
+# table-description:
+# This table contains metadata about the analysis results collections available in IDC. Each row corresponds to an
+# analysis results collection, and contains attributes such as the collection name, types of cancer represented,
+# number of subjects, and pointers to the resources to learn more about the content of the collection
+SELECT
+  # description:
+  # unique identifier of the analysis results collection
+  ID AS analysis_result_id,
+  # description:
+  # name of the analysis results collection
+  Title AS analysis_result_title,
+  # description:
+  # Digital Object Identifier (DOI) of the analysis results collection
+  source_doi,
+  # description:
+  # URL for the location of additional information about the analysis results collection
+  source_url,
+  # description:
+  # number of subjects analyzed in the analysis results collection
+  Subjects,
+  # description:
+  # collections analyzed in the analysis results collection
+  Collections,
+  # description:
+  # analysis artifacts included in the analysis results collection
+  AnalysisArtifacts,
+  # description:
+  # timestamp of the last update to the analysis results collection
+  Updated,
+  # description:
+  # license URL for the analysis results collection
+  license_url,
+  # description:
+  # license name for the analysis results collection
+  license_long_name,
+  # description:
+  # short name for the license of the analysis results collection
+  license_short_name,
+  # description:
+  # detailed description of the analysis results collection
+  Description,
+  # description:
+  # citation for the analysis results collection that should be used for acknowledgment
+  Citation
+FROM
+  `bigquery-public-data.idc_v22.analysis_results_metadata`

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/scripts/sql/collections_index.sql

@@ -1,3 +1,7 @@
+# table-description:
+# This table contains metadata about the collections available in IDC. Each row corresponds to a collection,
+# and contains attributes such as the collection name, types of cancer represented, number of subjects,
+# and pointers to the resources to learn more about the content of the collection.
 SELECT
   # description:
   # name of the collection

{idc_index_data-22.1.3 → idc_index_data-22.1.5}/tests/test_real_sql_parsing.py

@@ -84,18 +84,6 @@ def test_real_sql_files() -> None:
             else:
                 print(f"✗ Missing expected column: {col}")
 
-    # Test analysis_results_index.sql (should have no descriptions)
-    analysis_sql_path = sql_dir / "analysis_results_index.sql"
-    if analysis_sql_path.exists():
-        with analysis_sql_path.open("r") as f:
-            sql_query = f.read()
-
-        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
-        print("\n=== analysis_results_index.sql ===")
-        print(f"Found {len(descriptions)} column descriptions (expected 0)")
-        assert len(descriptions) == 0, "Expected no descriptions in this file"
-        print("✓ Correctly found no descriptions")
-
 
 if __name__ == "__main__":
     test_real_sql_files()

idc_index_data-22.1.3/scripts/sql/analysis_results_index.sql

@@ -1,16 +0,0 @@
-SELECT
-  ID AS analysis_result_id,
-  Title AS analysis_result_title,
-  source_doi,
-  source_url,
-  Subjects,
-  Collections,
-  AnalysisArtifacts,
-  Updated,
-  license_url,
-  license_long_name,
-  license_short_name,
-  Description,
-  Citation
-FROM
-  `bigquery-public-data.idc_v22.analysis_results_metadata`