idc-index-data 22.1.2__tar.gz → 22.1.3__tar.gz

{idc_index_data-22.1.2 → idc_index_data-22.1.3}/.pre-commit-config.yaml

@@ -55,6 +55,7 @@ repos:
         additional_dependencies:
           - pytest
           - pandas-stubs
+          - google-cloud-bigquery
 
   - repo: https://github.com/codespell-project/codespell
     rev: "v2.4.1"

{idc_index_data-22.1.2 → idc_index_data-22.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: idc-index-data
-Version: 22.1.2
+Version: 22.1.3
 Summary: ImagingDataCommons index to query and download data.
 Author-Email: Andrey Fedorov <andrey.fedorov@gmail.com>, Vamsi Thiriveedhi <vthiriveedhi@mgh.harvard.edu>, Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
 License: Copyright 2024 Andrey Fedorov
@@ -41,6 +41,7 @@ Project-URL: Bug Tracker, https://github.com/ImagingDataCommons/idc-index-data/i
 Project-URL: Discussions, https://discourse.canceridc.dev/
 Project-URL: Changelog, https://github.com/ImagingDataCommons/idc-index-data/releases
 Requires-Python: >=3.10
+Requires-Dist: google-cloud-bigquery
 Provides-Extra: test
 Requires-Dist: pandas; extra == "test"
 Requires-Dist: pyarrow; extra == "test"

idc_index_data-22.1.3/assets/clinical_index.sql

@@ -0,0 +1,23 @@
+SELECT
+  # description:
+  # unique identifier of the collection
+  collection_id,
+  # description:
+  # full name of the table in which the column is stored
+  table_name,
+  # description:
+  # short name of the table in which the column is stored
+  SPLIT(table_name,'.')[SAFE_OFFSET(2)] AS short_table_name,
+  # description:
+  # name of the column in which the value is stored
+  `column`,
+  # description:
+  # human readable name of the column
+  column_label,
+  # description:
+  # values encountered in the column
+  `values`
+FROM
+  `bigquery-public-data.idc_v22_clinical.column_metadata`
+ORDER BY
+  collection_id, table_name

{idc_index_data-22.1.2 → idc_index_data-22.1.3}/pyproject.toml

@@ -13,7 +13,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "idc-index-data"
-version = "22.1.2"
+version = "22.1.3"
 authors = [
   { name = "Andrey Fedorov", email = "andrey.fedorov@gmail.com" },
   { name = "Vamsi Thiriveedhi", email = "vthiriveedhi@mgh.harvard.edu" },
@@ -38,7 +38,9 @@ classifiers = [
   "Topic :: Scientific/Engineering",
   "Typing :: Typed",
 ]
-dependencies = []
+dependencies = [
+  "google-cloud-bigquery"
+]
 
 [project.optional-dependencies]
 test = [
@@ -102,7 +104,7 @@ report.exclude_also = [
 
 [tool.mypy]
 files = ["src", "tests"]
-python_version = "3.8"
+python_version = "3.10"
 warn_unused_configs = true
 strict = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -158,7 +160,7 @@ isort.required-imports = ["from __future__ import annotations"]
 
 
 [tool.pylint]
-py-version = "3.8"
+py-version = "3.10"
 ignore-paths = [".*/_version.py"]
 reports.output-format = "colorized"
 similarities.ignore-imports = "yes"

idc_index_data-22.1.3/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+filterwarnings = ignore::FutureWarning:google.api_core

{idc_index_data-22.1.2 → idc_index_data-22.1.3}/scripts/python/generate-indices.py

@@ -28,7 +28,7 @@ def main():
         )
         parquet_file_path = output_dir / f"{output_basename}.parquet"
         index_df.to_parquet(parquet_file_path)
-        manager.save_schema_to_json(schema, output_basename, output_dir)
+        manager.save_schema_to_json(schema, output_basename, sql_query, output_dir)
         manager.save_sql_query(sql_query, output_basename, output_dir)
 
     core_indices_dir = scripts_dir.parent / "scripts" / "sql"
@@ -42,7 +42,11 @@ def main():
         )
         parquet_file_path = output_dir / f"{output_basename}.parquet"
         index_df.to_parquet(parquet_file_path)
-        manager.save_schema_to_json(schema, output_basename, output_dir)
+        if output_basename == "prior_versions_index":
+            # For prior_versions_index, save schema without descriptions
+            manager.save_schema_to_json(schema, output_basename, None, output_dir)
+        else:
+            manager.save_schema_to_json(schema, output_basename, sql_query, output_dir)
         manager.save_sql_query(sql_query, output_basename, output_dir)
 
 

idc_index_data-22.1.3/scripts/python/idc_index_data_manager.py

@@ -0,0 +1,424 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from pathlib import Path
+
+import pandas as pd
+from google.cloud import bigquery
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
+class IDCIndexDataManager:
+    def __init__(self, project_id: str):
+        """
+        Initializes the IDCIndexDataManager using the Google Cloud Platform project ID.
+        """
+        self.project_id = project_id
+        self.client = bigquery.Client(project=project_id)
+        logger.debug("IDCIndexDataManager initialized with project ID: %s", project_id)
+
+    @staticmethod
+    def parse_column_descriptions(sql_query: str) -> dict[str, str]:
+        """
+        Parses column descriptions from SQL query comments.
+
+        The method looks for comments following the pattern:
+        # description:
+        # description text continues here
+        # and can span multiple lines
+        column_name or expression AS column_name,
+
+        Args:
+            sql_query: The SQL query string containing comments
+
+        Returns:
+            Dictionary mapping column names to their descriptions
+        """
+        descriptions: dict[str, str] = {}
+        logger.debug("Parsing column descriptions from SQL query comments")
+        logger.debug(sql_query)
+        lines = sql_query.split("\n")
+
+        i = 0
+        while i < len(lines):
+            line = lines[i]
+            stripped = line.strip()
+
+            # Check if this line starts a description comment
+            if stripped == "# description:":
+                # Collect description lines until we hit a non-comment line
+                description_lines = []
+                i += 1
+
+                while i < len(lines):
+                    next_line = lines[i]
+                    next_stripped = next_line.strip()
+
+                    # If it's a description comment line (starts with #)
+                    if next_stripped.startswith("#") and next_stripped != "#":
+                        # Remove the leading # and whitespace
+                        desc_text = next_stripped[1:].strip()
+                        if desc_text:
+                            description_lines.append(desc_text)
+                        i += 1
+                    elif next_stripped.startswith("#"):
+                        # Empty comment line, skip
+                        i += 1
+                    else:
+                        # Non-comment line - this should contain the column definition
+                        break
+
+                # Now parse the column definition
+                if i < len(lines) and description_lines:
+                    # Join the description lines
+                    description = " ".join(description_lines)
+
+                    # Find the column name by parsing the SELECT clause
+                    # We need to handle multi-line column definitions with nested structures
+                    column_def = ""
+                    paren_depth = (
+                        0  # Track parentheses depth to handle nested SELECT/FROM
+                    )
+
+                    while i < len(lines):
+                        current_line = lines[i]
+                        current_stripped = current_line.strip()
+
+                        # Count parentheses to track nesting depth
+                        paren_depth += current_line.count("(") - current_line.count(")")
+
+                        # Only check for top-level SQL keywords when not inside nested structures
+                        if paren_depth == 0 and any(
+                            current_stripped.upper().startswith(keyword)
+                            for keyword in [
+                                "FROM",
+                                "WHERE",
+                                "GROUP BY",
+                                "ORDER BY",
+                                "JOIN",
+                                "LEFT",
+                                "RIGHT",
+                                "INNER",
+                                "OUTER",
+                            ]
+                        ):
+                            # Don't include this line in column_def
+                            # Don't increment i here - let outer loop handle it
+                            break
+
+                        column_def += " " + current_stripped
+                        i += 1
+
+                        # Check if we've found a complete column definition
+                        # (has a comma at depth 0)
+                        if paren_depth == 0 and "," in current_line:
+                            break
+
+                        # Safety check: if we've gone too deep, break
+                        if paren_depth < 0:
+                            break
+
+                    # Extract column name from the definition
+                    column_name = IDCIndexDataManager._extract_column_name(column_def)
+                    if column_name:
+                        descriptions[column_name] = description
+                        logger.debug(
+                            "Parsed description for column '%s': %s",
+                            column_name,
+                            description[:50] + "..."
+                            if len(description) > 50
+                            else description,
+                        )
+                else:
+                    i += 1
+            else:
+                i += 1
+
+        return descriptions
+
+    @staticmethod
+    def _extract_column_name(column_def: str) -> str | None:
+        """
+        Extracts the column name from a column definition.
+
+        Handles various formats:
+        - column_name,
+        - expression AS column_name,
+        - ANY_VALUE(column) AS column_name,
+        - Complex expressions with nested parentheses
+
+        Args:
+            column_def: The column definition string
+
+        Returns:
+            The column name or None if not found
+        """
+        # Remove trailing comma and whitespace
+        column_def = column_def.strip().rstrip(",").strip()
+
+        # Look for the last AS clause (to handle nested AS in CAST expressions)
+        # Use a regex that finds the rightmost AS followed by a word
+        as_matches = list(re.finditer(r"\bAS\b\s+(\w+)", column_def, re.IGNORECASE))
+        if as_matches:
+            # Return the last match (rightmost AS clause)
+            return as_matches[-1].group(1)
+
+        # If no AS clause, try to get the column name
+        # Remove function calls and get the last word before comma
+        # Handle cases like: column_name, or just column_name
+        parts = column_def.split()
+        if parts:
+            # Get the last word that looks like an identifier
+            for original_part in reversed(parts):
+                # Remove trailing punctuation
+                part = original_part.rstrip(",").strip()
+                # Check if it's a valid identifier (word characters only)
+                if re.match(r"^\w+$", part):
+                    return part
+
+        return None
+
+    def execute_sql_query(
+        self, file_path: str
+    ) -> tuple[pd.DataFrame, str, list[bigquery.SchemaField], str]:
+        """
+        Executes the SQL query in the specified file.
+
+        Returns:
+            Tuple[pd.DataFrame, str, List[bigquery.SchemaField], str]: A tuple containing
+            the DataFrame with query results, the output basename, the BigQuery schema, and
+            the SQL query string.
+        """
+        with Path(file_path).open("r") as file:
+            sql_query = file.read()
+        query_job_result = self.client.query(sql_query).result()
+        schema = query_job_result.schema  # Get schema from BigQuery QueryJob
+        index_df = query_job_result.to_dataframe()
+        if "StudyDate" in index_df.columns:
+            index_df["StudyDate"] = index_df["StudyDate"].astype(str)
+        output_basename = Path(file_path).name.split(".")[0]
+        logger.debug("Executed SQL query from file: %s", file_path)
+        return index_df, output_basename, schema, sql_query
+
+    def save_schema_to_json(
+        self,
+        schema: list[bigquery.SchemaField],
+        output_basename: str,
+        sql_query: str | None = None,
+        output_dir: Path | None = None,
+    ) -> None:
+        """
+        Saves the BigQuery schema to a JSON file, including column descriptions
+        parsed from SQL query comments.
+
+        Args:
+            schema: List of BigQuery SchemaField objects from the query result
+            output_basename: The base name for the output file
+            sql_query: The SQL query string to parse for column descriptions
+            output_dir: Optional directory path for the output file
+        """
+        # Parse column descriptions from SQL comments
+        logger.debug("Parsing column descriptions from SQL query comments")
+        logger.debug(sql_query)
+        if sql_query is not None:
+            descriptions = self.parse_column_descriptions(sql_query)
+
+            # Convert BigQuery schema to JSON-serializable format
+            schema_dict = {
+                "fields": [
+                    {
+                        "name": field.name,
+                        "type": field.field_type,
+                        "mode": field.mode,
+                        "description": descriptions.get(field.name, ""),
+                    }
+                    for field in schema
+                ]
+            }
+        else:
+            # If no SQL query provided, save schema without descriptions
+            schema_dict = {
+                "fields": [
+                    {
+                        "name": field.name,
+                        "type": field.field_type,
+                        "mode": field.mode,
+                        "description": "",
+                    }
+                    for field in schema
+                ]
+            }
+
+        # Save to JSON file
+        if output_dir:
+            output_dir.mkdir(parents=True, exist_ok=True)
+            json_file_path = output_dir / f"{output_basename}.json"
+        else:
+            json_file_path = Path(f"{output_basename}.json")
+
+        with json_file_path.open("w") as f:
+            json.dump(schema_dict, f, indent=2)
+        logger.debug("Created schema JSON file: %s", json_file_path)
+
+    def save_sql_query(
+        self,
+        sql_query: str,
+        output_basename: str,
+        output_dir: Path | None = None,
+    ) -> None:
+        """
+        Saves the SQL query to a file.
+
+        Args:
+            sql_query: The SQL query string
+            output_basename: The base name for the output file
+            output_dir: Optional directory path for the output file
+        """
+
+        if output_dir:
+            output_dir.mkdir(parents=True, exist_ok=True)
+            query_file_path = output_dir / f"{output_basename}.sql"
+        else:
+            query_file_path = Path(f"{output_basename}.sql")
+
+        with query_file_path.open("w") as f:
+            f.write(sql_query)
+        logger.debug("Created SQL query file: %s", query_file_path)
+
+    def generate_index_data_files(
+        self,
+        generate_compressed_csv: bool = True,
+        generate_parquet: bool = False,
+        output_dir: Path | None = None,
+    ) -> None:
+        """
+        Generates index-data files locally by executing queries against
+        the Google Cloud Platform IDC project tables.
+
+        This method iterates over SQL files in the 'scripts/sql' directory,
+        executing each query using :func:`execute_sql_query` and generating a DataFrame,
+        'index_df'. The DataFrame is then saved as compressed CSV and/or Parquet file.
+
+        Args:
+            generate_compressed_csv: Whether to generate compressed CSV files
+            generate_parquet: Whether to generate Parquet files
+            output_dir: Optional directory path for the output files
+        """
+
+        scripts_dir = Path(__file__).parent.parent
+        sql_dir = scripts_dir / "sql"
+
+        if output_dir:
+            output_dir.mkdir(parents=True, exist_ok=True)
+
+        for file_name in Path.iterdir(sql_dir):
+            if str(file_name).endswith(".sql"):
+                file_path = Path(sql_dir) / file_name
+                index_df, output_basename, schema, sql_query = self.execute_sql_query(
+                    str(file_path)
+                )
+                logger.debug(
+                    "Executed and processed SQL queries from file: %s", file_path
+                )
+                if generate_compressed_csv:
+                    csv_file_path = (
+                        output_dir / f"{output_basename}.csv.zip"
+                        if output_dir
+                        else Path(f"{output_basename}.csv.zip")
+                    )
+                    index_df.to_csv(
+                        csv_file_path, compression={"method": "zip"}, escapechar="\\"
+                    )
+                    logger.debug("Created CSV zip file: %s", csv_file_path)
+
+                if generate_parquet:
+                    parquet_file_path = (
+                        output_dir / f"{output_basename}.parquet"
+                        if output_dir
+                        else Path(f"{output_basename}.parquet")
+                    )
+                    index_df.to_parquet(parquet_file_path, compression="zstd")
+                    logger.debug("Created Parquet file: %s", parquet_file_path)
+
+                # Save schema to JSON file
+                # Skip parsing descriptions for prior_versions_index as it has dynamic SQL
+                if output_basename != "prior_versions_index":
+                    self.save_schema_to_json(
+                        schema, output_basename, sql_query, output_dir
+                    )
+                else:
+                    # For prior_versions_index, save schema without descriptions
+                    self.save_schema_to_json(schema, output_basename, None, output_dir)
+                # Save SQL query to file
+                self.save_sql_query(sql_query, output_basename, output_dir)
+
+    def retrieve_latest_idc_release_version(self) -> int:
+        """
+        Retrieves the latest IDC release version.
+
+        This function executes a SQL query on the `version_metadata` table in the
+        `idc_current` dataset of the BigQuery client. It retrieves the maximum
+        `idc_version` and returns it as an integer.
+        """
+        query = """
+        SELECT
+            MAX(idc_version) AS latest_idc_release_version
+        FROM
+            `bigquery-public-data.idc_current.version_metadata`
+        """
+        query_job = self.client.query(query)
+        result = query_job.result()
+        return int(next(result).latest_idc_release_version)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--project",
+        default=os.environ.get("GCP_PROJECT", None),
+        help="Google Cloud Platform Project ID (default from GCP_PROJECT env. variable)",
+    )
+    parser.add_argument(
+        "--generate-csv-archive",
+        action="store_true",
+        help="Generate idc_index.csv.zip file",
+    )
+    parser.add_argument(
+        "--generate-parquet",
+        action="store_true",
+        help="Generate idc_index.parquet file",
+    )
+    parser.add_argument(
+        "--retrieve-latest-idc-release-version",
+        action="store_true",
+        help="Retrieve and display the latest IDC release version",
+    )
+
+    args = parser.parse_args()
+
+    if not args.project:
+        parser.error(
+            "Set GCP_PROJECT environment variable or specify --project argument"
+        )
+
+    if any([args.generate_csv_archive, args.generate_parquet]):
+        IDCIndexDataManager(args.project).generate_index_data_files(
+            generate_compressed_csv=args.generate_csv_archive,
+            generate_parquet=args.generate_parquet,
+        )
+    elif args.retrieve_latest_idc_release_version:
+        logging.basicConfig(level=logging.ERROR, force=True)
+        logger.setLevel(logging.ERROR)
+        version = IDCIndexDataManager(
+            args.project
+        ).retrieve_latest_idc_release_version()
+        print(f"{version}")  # noqa: T201
+    else:
+        parser.print_help()

idc_index_data-22.1.3/tests/test_column_description_parser.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+from scripts.python.idc_index_data_manager import IDCIndexDataManager
+
+
+class TestColumnDescriptionParser:
+    """Tests for parsing column descriptions from SQL comments."""
+
+    def test_simple_column_description(self):
+        """Test parsing a simple column with description."""
+        sql_query = """
+SELECT
+  # description:
+  # name of the collection
+  collection_name,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert "collection_name" in descriptions
+        assert descriptions["collection_name"] == "name of the collection"
+
+    def test_multiline_description(self):
+        """Test parsing a column with multi-line description."""
+        sql_query = """
+SELECT
+  # description:
+  # this string is not empty if the specific series is
+  # part of an analysis results collection; analysis results can be added to a
+  # given collection over time
+  ANY_VALUE(analysis_result_id) AS analysis_result_id,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert "analysis_result_id" in descriptions
+        expected = (
+            "this string is not empty if the specific series is "
+            "part of an analysis results collection; analysis results can be added to a "
+            "given collection over time"
+        )
+        assert descriptions["analysis_result_id"] == expected
+
+    def test_column_with_as_clause(self):
+        """Test parsing a column with AS clause."""
+        sql_query = """
+SELECT
+  # description:
+  # unique identifier of the DICOM series
+  SeriesInstanceUID AS series_id,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert "series_id" in descriptions
+        assert descriptions["series_id"] == "unique identifier of the DICOM series"
+
+    def test_column_with_function(self):
+        """Test parsing a column with function call."""
+        sql_query = """
+SELECT
+  # description:
+  # age of the subject at the time of imaging
+  ANY_VALUE(PatientAge) AS PatientAge,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert "PatientAge" in descriptions
+        assert descriptions["PatientAge"] == "age of the subject at the time of imaging"
+
+    def test_multiple_columns(self):
+        """Test parsing multiple columns with descriptions."""
+        sql_query = """
+SELECT
+  # description:
+  # name of the collection
+  collection_name,
+  # description:
+  # unique identifier of the collection
+  collection_id,
+  # description:
+  # types of cancer represented in the collection
+  CancerTypes,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert len(descriptions) == 3
+        assert descriptions["collection_name"] == "name of the collection"
+        assert descriptions["collection_id"] == "unique identifier of the collection"
+        assert (
+            descriptions["CancerTypes"]
+            == "types of cancer represented in the collection"
+        )
+
+    def test_column_without_description(self):
+        """Test that columns without descriptions are not in the result."""
+        sql_query = """
+SELECT
+  # description:
+  # name of the collection
+  collection_name,
+  collection_id,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert "collection_name" in descriptions
+        assert "collection_id" not in descriptions
+
+    def test_extract_column_name_simple(self):
+        """Test extracting column name from simple definition."""
+        assert (
+            IDCIndexDataManager._extract_column_name("collection_name,")
+            == "collection_name"
+        )
+        assert (
+            IDCIndexDataManager._extract_column_name("collection_name")
+            == "collection_name"
+        )
+
+    def test_extract_column_name_with_as(self):
+        """Test extracting column name with AS clause."""
+        assert (
+            IDCIndexDataManager._extract_column_name(
+                "ANY_VALUE(collection_id) AS collection_id,"
+            )
+            == "collection_id"
+        )
+        assert IDCIndexDataManager._extract_column_name("column AS alias,") == "alias"
+
+    def test_extract_column_name_complex(self):
+        """Test extracting column name from complex expressions."""
+        assert (
+            IDCIndexDataManager._extract_column_name(
+                "ROUND(SUM(SAFE_CAST(instance_size AS float64))/1000000, 2) AS series_size_MB,"
+            )
+            == "series_size_MB"
+        )
+        assert (
+            IDCIndexDataManager._extract_column_name("COUNT(SOPInstanceUID) AS count,")
+            == "count"
+        )
+
+    def test_complex_multiline_select(self):
+        """Test parsing a complex multi-line SELECT statement."""
+        sql_query = """
+SELECT
+  # description:
+  # total size of the series in megabytes
+  ROUND(SUM(SAFE_CAST(instance_size AS float64))/1000000, 2) AS series_size_MB,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert "series_size_MB" in descriptions
+        assert descriptions["series_size_MB"] == "total size of the series in megabytes"
+
+    def test_no_descriptions(self):
+        """Test SQL query with no descriptions."""
+        sql_query = """
+SELECT
+  collection_name,
+  collection_id,
+  CancerTypes
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert len(descriptions) == 0
+
+    def test_empty_description_lines(self):
+        """Test handling of empty comment lines in descriptions."""
+        sql_query = """
+SELECT
+  # description:
+  # name of the collection
+  #
+  # additional info
+  collection_name,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert "collection_name" in descriptions
+        # Empty comment lines should be skipped
+        assert (
+            descriptions["collection_name"] == "name of the collection additional info"
+        )
+
+    def test_nested_array_select_with_if(self):
+        """Test parsing complex nested ARRAY/SELECT/IF statements."""
+        sql_query = """
+SELECT
+  # description:
+  # embedding medium used for the slide preparation
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL, SPLIT(code, ':')[SAFE_OFFSET(0)])
+  FROM
+    UNNEST(embeddingMedium_code_str) AS code ) AS embeddingMedium_CodeMeaning,
+  # description:
+  # embedding medium code tuple
+  ARRAY(
+  SELECT
+  IF
+    (code IS NULL, NULL,
+    IF
+      (STRPOS(code, ':') = 0, NULL, SUBSTR(code, STRPOS(code, ':') + 1)))
+  FROM
+    UNNEST(embeddingMedium_code_str) AS code ) AS embeddingMedium_code_designator_value_str,
+FROM table
+"""
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        assert len(descriptions) == 2
+        assert "embeddingMedium_CodeMeaning" in descriptions
+        assert (
+            descriptions["embeddingMedium_CodeMeaning"]
+            == "embedding medium used for the slide preparation"
+        )
+        assert "embeddingMedium_code_designator_value_str" in descriptions
+        assert (
+            descriptions["embeddingMedium_code_designator_value_str"]
+            == "embedding medium code tuple"
+        )

idc_index_data-22.1.3/tests/test_real_sql_parsing.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from scripts.python.idc_index_data_manager import IDCIndexDataManager
+
+"""Test script to verify column description parsing with real SQL files."""
+
+
+def test_real_sql_files() -> None:
+    """Test parsing descriptions from actual SQL files in the repository."""
+    scripts_dir = Path(__file__).parent.parent / "scripts"
+    sql_dir = scripts_dir / "sql"
+
+    # Test collections_index.sql
+    collections_sql_path = sql_dir / "collections_index.sql"
+    if collections_sql_path.exists():
+        with collections_sql_path.open("r") as f:
+            sql_query = f.read()
+
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        print("\n=== collections_index.sql ===")
+        print(f"Found {len(descriptions)} column descriptions:")
+        for col_name, desc in descriptions.items():
+            print(f"  {col_name}: {desc[:60]}...")
+
+        # Validate expected columns
+        expected_columns = [
+            "collection_name",
+            "collection_id",
+            "CancerTypes",
+            "TumorLocations",
+            "Subjects",
+            "Species",
+            "Sources",
+            "SupportingData",
+            "Program",
+            "Status",
+            "Updated",
+            "Description",
+        ]
+        for col in expected_columns:
+            assert col in descriptions, (
+                f"Expected column '{col}' not found in descriptions"
+            )
+        print("✓ All expected columns found")
+
+    # Test idc_index.sql
+    idc_index_sql_path = sql_dir / "idc_index.sql"
+    if idc_index_sql_path.exists():
+        with idc_index_sql_path.open("r") as f:
+            sql_query = f.read()
+
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        print("\n=== idc_index.sql ===")
+        print(f"Found {len(descriptions)} column descriptions:")
+
+        # Show first 10 descriptions
+        for i, (col_name, desc) in enumerate(descriptions.items()):
+            if i < 10:
+                print(f"  {col_name}: {desc[:60]}...")
+            else:
+                break
+
+        if len(descriptions) > 10:
+            print(f"  ... and {len(descriptions) - 10} more")
+
+        # Validate some expected columns
+        expected_columns = [
+            "collection_id",
+            "analysis_result_id",
+            "PatientID",
+            "SeriesInstanceUID",
+            "StudyInstanceUID",
+            "source_DOI",
+            "PatientAge",
+            "PatientSex",
+            "StudyDate",
+            "series_size_MB",
+        ]
+        for col in expected_columns:
+            if col in descriptions:
+                print(f"✓ Found expected column: {col}")
+            else:
+                print(f"✗ Missing expected column: {col}")
+
+    # Test analysis_results_index.sql (should have no descriptions)
+    analysis_sql_path = sql_dir / "analysis_results_index.sql"
+    if analysis_sql_path.exists():
+        with analysis_sql_path.open("r") as f:
+            sql_query = f.read()
+
+        descriptions = IDCIndexDataManager.parse_column_descriptions(sql_query)
+        print("\n=== analysis_results_index.sql ===")
+        print(f"Found {len(descriptions)} column descriptions (expected 0)")
+        assert len(descriptions) == 0, "Expected no descriptions in this file"
+        print("✓ Correctly found no descriptions")
+
+
+if __name__ == "__main__":
+    test_real_sql_files()

idc_index_data-22.1.2/assets/clinical_index.sql

@@ -1,11 +0,0 @@
-SELECT
-  collection_id,
-  table_name,
-  SPLIT(table_name,'.')[SAFE_OFFSET(2)] AS short_table_name,
-  `column`,
-  column_label,
-  `values`
-FROM
-  `bigquery-public-data.idc_v22_clinical.column_metadata`
-ORDER BY
-  collection_id, table_name

idc_index_data-22.1.2/scripts/python/idc_index_data_manager.py

@@ -1,231 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import os
-from pathlib import Path
-
-import pandas as pd
-from google.cloud import bigquery
-
-logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger(__name__)
-
-
-class IDCIndexDataManager:
-    def __init__(self, project_id: str):
-        """
-        Initializes the IDCIndexDataManager using the Google Cloud Platform project ID.
-        """
-        self.project_id = project_id
-        self.client = bigquery.Client(project=project_id)
-        logger.debug("IDCIndexDataManager initialized with project ID: %s", project_id)
-
-    def execute_sql_query(
-        self, file_path: str
-    ) -> tuple[pd.DataFrame, str, list[bigquery.SchemaField]]:
-        """
-        Executes the SQL query in the specified file.
-
-        Returns:
-            Tuple[pd.DataFrame, str, List[bigquery.SchemaField]]: A tuple containing
-            the DataFrame with query results, the output basename, and the BigQuery schema.
-        """
-        with Path(file_path).open("r") as file:
-            sql_query = file.read()
-        query_job_result = self.client.query(sql_query).result()
-        schema = query_job_result.schema  # Get schema from BigQuery QueryJob
-        index_df = query_job_result.to_dataframe()
-        if "StudyDate" in index_df.columns:
-            index_df["StudyDate"] = index_df["StudyDate"].astype(str)
-        output_basename = Path(file_path).name.split(".")[0]
-        logger.debug("Executed SQL query from file: %s", file_path)
-        return index_df, output_basename, schema, sql_query
-
-    def save_schema_to_json(
-        self,
-        schema: list[bigquery.SchemaField],
-        output_basename: str,
-        output_dir: Path | None = None,
-    ) -> None:
-        """
-        Saves the BigQuery schema to a JSON file.
-
-        Args:
-            schema: List of BigQuery SchemaField objects from the query result
-            output_basename: The base name for the output file
-            output_dir: Optional directory path for the output file
-        """
-        # Convert BigQuery schema to JSON-serializable format
-        schema_dict = {
-            "fields": [
-                {
-                    "name": field.name,
-                    "type": field.field_type,
-                    "mode": field.mode,
-                }
-                for field in schema
-            ]
-        }
-
-        # Save to JSON file
-        if output_dir:
-            output_dir.mkdir(parents=True, exist_ok=True)
-            json_file_path = output_dir / f"{output_basename}.json"
-        else:
-            json_file_path = Path(f"{output_basename}.json")
-
-        with json_file_path.open("w") as f:
-            json.dump(schema_dict, f, indent=2)
-        logger.debug("Created schema JSON file: %s", json_file_path)
-
-    def save_sql_query(
-        self,
-        sql_query: str,
-        output_basename: str,
-        output_dir: Path | None = None,
-    ) -> None:
-        """
-        Saves the SQL query to a file.
-
-        Args:
-            sql_query: The SQL query string
-            output_basename: The base name for the output file
-            output_dir: Optional directory path for the output file
-        """
-
-        if output_dir:
-            output_dir.mkdir(parents=True, exist_ok=True)
-            query_file_path = output_dir / f"{output_basename}.sql"
-        else:
-            query_file_path = Path(f"{output_basename}.sql")
-
-        with query_file_path.open("w") as f:
-            f.write(sql_query)
-        logger.debug("Created SQL query file: %s", query_file_path)
-
-    def generate_index_data_files(
-        self,
-        generate_compressed_csv: bool = True,
-        generate_parquet: bool = False,
-        output_dir: Path | None = None,
-    ) -> None:
-        """
-        Generates index-data files locally by executing queries against
-        the Google Cloud Platform IDC project tables.
-
-        This method iterates over SQL files in the 'scripts/sql' directory,
-        executing each query using :func:`execute_sql_query` and generating a DataFrame,
-        'index_df'. The DataFrame is then saved as compressed CSV and/or Parquet file.
-
-        Args:
-            generate_compressed_csv: Whether to generate compressed CSV files
-            generate_parquet: Whether to generate Parquet files
-            output_dir: Optional directory path for the output files
-        """
-
-        scripts_dir = Path(__file__).parent.parent
-        sql_dir = scripts_dir / "sql"
-
-        if output_dir:
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-        for file_name in Path.iterdir(sql_dir):
-            if str(file_name).endswith(".sql"):
-                file_path = Path(sql_dir) / file_name
-                index_df, output_basename, schema, sql_query = self.execute_sql_query(
-                    file_path
-                )
-                logger.debug(
-                    "Executed and processed SQL queries from file: %s", file_path
-                )
-                if generate_compressed_csv:
-                    csv_file_path = (
-                        output_dir / f"{output_basename}.csv.zip"
-                        if output_dir
-                        else Path(f"{output_basename}.csv.zip")
-                    )
-                    index_df.to_csv(
-                        csv_file_path, compression={"method": "zip"}, escapechar="\\"
-                    )
-                    logger.debug("Created CSV zip file: %s", csv_file_path)
-
-                if generate_parquet:
-                    parquet_file_path = (
-                        output_dir / f"{output_basename}.parquet"
-                        if output_dir
-                        else Path(f"{output_basename}.parquet")
-                    )
-                    index_df.to_parquet(parquet_file_path, compression="zstd")
-                    logger.debug("Created Parquet file: %s", parquet_file_path)
-
-                # Save schema to JSON file
-                self.save_schema_to_json(schema, output_basename, output_dir)
-                # Save SQL query to file
-                self.save_sql_query(sql_query, output_basename, output_dir)
-
-    def retrieve_latest_idc_release_version(self) -> int:
-        """
-        Retrieves the latest IDC release version.
-
-        This function executes a SQL query on the `version_metadata` table in the
-        `idc_current` dataset of the BigQuery client. It retrieves the maximum
-        `idc_version` and returns it as an integer.
-        """
-        query = """
-        SELECT
-            MAX(idc_version) AS latest_idc_release_version
-        FROM
-            `bigquery-public-data.idc_current.version_metadata`
-        """
-        query_job = self.client.query(query)
-        result = query_job.result()
-        return int(next(result).latest_idc_release_version)
-
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--project",
-        default=os.environ.get("GCP_PROJECT", None),
-        help="Google Cloud Platform Project ID (default from GCP_PROJECT env. variable)",
-    )
-    parser.add_argument(
-        "--generate-csv-archive",
-        action="store_true",
-        help="Generate idc_index.csv.zip file",
-    )
-    parser.add_argument(
-        "--generate-parquet",
-        action="store_true",
-        help="Generate idc_index.parquet file",
-    )
-    parser.add_argument(
-        "--retrieve-latest-idc-release-version",
-        action="store_true",
-        help="Retrieve and display the latest IDC release version",
-    )
-
-    args = parser.parse_args()
-
-    if not args.project:
-        parser.error(
-            "Set GCP_PROJECT environment variable or specify --project argument"
-        )
-
-    if any([args.generate_csv_archive, args.generate_parquet]):
-        IDCIndexDataManager(args.project).generate_index_data_files(
-            generate_compressed_csv=args.generate_csv_archive,
-            generate_parquet=args.generate_parquet,
-        )
-    elif args.retrieve_latest_idc_release_version:
-        logging.basicConfig(level=logging.ERROR, force=True)
-        logger.setLevel(logging.ERROR)
-        version = IDCIndexDataManager(
-            args.project
-        ).retrieve_latest_idc_release_version()
-        print(f"{version}")  # noqa: T201
-    else:
-        parser.print_help()