celltype-cli 0.1.0__py3-none-any.whl

ct/api/app.py

@@ -0,0 +1,211 @@
+"""
+FastAPI application for the ct Data Query API.
+
+Serves filtered queries against large datasets (PerturbAtlas, ChEMBL, etc.)
+using DuckDB for efficient Parquet/CSV.gz querying.
+
+Run with: uvicorn ct.api.app:app --host 0.0.0.0 --port 8000
+"""
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+
+from ct.api.config import discover_datasets, validate_schema, DATASET_REGISTRY, DEFAULT_QUERY_LIMIT
+from ct.api.engine import QueryEngine
+
+app = FastAPI(
+    title="ct Data API",
+    description="Query large drug discovery datasets via DuckDB",
+    version="0.1.0",
+)
+
+engine = QueryEngine()
+
+
+class QueryRequest(BaseModel):
+    dataset: str = Field(..., description="Dataset name (e.g. 'perturbatlas', 'chembl')")
+    gene: str | None = Field(None, description="Filter by gene symbol")
+    compound: str | None = Field(None, description="Filter by compound name/ID")
+    filters: dict | None = Field(None, description="Additional column filters")
+    columns: list[str] | None = Field(None, description="Columns to return")
+    limit: int = Field(DEFAULT_QUERY_LIMIT, ge=1, le=10000, description="Max rows")
+    order_by: str | None = Field(None, description="Column to sort by")
+
+
+class QueryResponse(BaseModel):
+    dataset: str
+    total_rows: int
+    data: list[dict]
+
+
+@app.get("/health")
+def health():
+    """Health check — also reports available datasets."""
+    available = discover_datasets()
+    return {
+        "status": "ok",
+        "datasets": list(available.keys()),
+        "n_datasets": len(available),
+    }
+
+
+@app.get("/health/schema")
+def schema_health():
+    """Deep health check — validates file schemas match expectations."""
+    results = validate_schema(engine)
+    has_invalid = any(r["status"] == "invalid" for r in results.values())
+
+    status_code = 503 if has_invalid else 200
+    from fastapi.responses import JSONResponse
+    return JSONResponse(
+        status_code=status_code,
+        content={
+            "status": "unhealthy" if has_invalid else "ok",
+            "datasets": results,
+        },
+    )
+
+
+@app.get("/datasets")
+def list_datasets():
+    """List available datasets with metadata."""
+    available = discover_datasets()
+    result = []
+    for name, info in available.items():
+        registry_entry = DATASET_REGISTRY.get(name, {})
+        result.append({
+            "name": name,
+            "description": info["description"],
+            "format": info["format"],
+            "n_files": info["n_files"],
+            "total_size_mb": info["total_size_mb"],
+            "filterable_columns": info.get("filterable", []),
+            "columns": registry_entry.get("columns", []),
+            "required_columns": registry_entry.get("required_columns", []),
+        })
+    return result
+
+
+@app.post("/query", response_model=QueryResponse)
+def query_dataset(req: QueryRequest):
+    """Run a filtered query against a dataset."""
+    if req.dataset not in DATASET_REGISTRY:
+        raise HTTPException(status_code=404, detail=f"Unknown dataset: {req.dataset}")
+
+    ds_config = DATASET_REGISTRY[req.dataset]
+
+    # Build filters
+    filters = dict(req.filters) if req.filters else {}
+    if req.gene:
+        filters["gene"] = req.gene
+    if req.compound:
+        # Try common column names for compound
+        for col in ["compound", "molecule_chembl_id", "pert_name", "compound_name"]:
+            if col in ds_config.get("filterable", []):
+                filters[col] = req.compound
+                break
+        else:
+            filters["compound"] = req.compound
+
+    try:
+        data = engine.query_parquet(
+            file_pattern=ds_config["path_pattern"],
+            filters=filters if filters else None,
+            columns=req.columns,
+            limit=req.limit,
+            order_by=req.order_by,
+        )
+        return QueryResponse(
+            dataset=req.dataset,
+            total_rows=len(data),
+            data=data,
+        )
+    except FileNotFoundError:
+        raise HTTPException(
+            status_code=503,
+            detail=f"Dataset '{req.dataset}' files not found on disk",
+        )
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@app.get("/datasets/{name}/gene/{gene}")
+def gene_summary(name: str, gene: str):
+    """Get a gene-level summary from a dataset."""
+    if name not in DATASET_REGISTRY:
+        raise HTTPException(status_code=404, detail=f"Unknown dataset: {name}")
+
+    ds_config = DATASET_REGISTRY[name]
+
+    try:
+        data = engine.query_parquet(
+            file_pattern=ds_config["path_pattern"],
+            filters={"gene": gene},
+            limit=1000,
+        )
+
+        if not data:
+            raise HTTPException(status_code=404, detail=f"Gene {gene} not found in {name}")
+
+        # Compute summary stats
+        import statistics
+        effects = [r.get("log2FoldChange", r.get("lfc", 0)) for r in data if r.get("log2FoldChange") or r.get("lfc")]
+        mean_effect = statistics.mean(effects) if effects else 0
+        n_perturbations = len(data)
+
+        return {
+            "gene": gene,
+            "dataset": name,
+            "n_perturbations": n_perturbations,
+            "mean_effect": round(mean_effect, 4),
+            "n_significant": sum(
+                1 for r in data
+                if (r.get("padj") or r.get("pvalue", 1)) < 0.05
+            ),
+            "sample_data": data[:10],
+        }
+    except FileNotFoundError:
+        raise HTTPException(status_code=503, detail=f"Dataset '{name}' files not found")
+
+
+@app.get("/datasets/{name}/compound/{compound}")
+def compound_summary(name: str, compound: str):
+    """Get a compound-level summary from a dataset."""
+    if name not in DATASET_REGISTRY:
+        raise HTTPException(status_code=404, detail=f"Unknown dataset: {name}")
+
+    ds_config = DATASET_REGISTRY[name]
+
+    # Try common compound column names — use filterable from dataset config
+    filters = {}
+    filterable = ds_config.get("filterable", [])
+    for col in ["compound", "molecule_chembl_id", "pert_name", "compound_name"]:
+        if col in filterable:
+            filters[col] = compound
+            break
+    else:
+        filters["compound"] = compound
+
+    try:
+        data = engine.query_parquet(
+            file_pattern=ds_config["path_pattern"],
+            filters=filters,
+            limit=1000,
+        )
+
+        if not data:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Compound {compound} not found in {name}",
+            )
+
+        return {
+            "compound": compound,
+            "dataset": name,
+            "n_records": len(data),
+            "sample_data": data[:10],
+        }
+    except FileNotFoundError:
+        raise HTTPException(status_code=503, detail=f"Dataset '{name}' files not found")
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))

ct/api/config.py

@@ -0,0 +1,120 @@
+"""
+Configuration for the ct Data Query API.
+
+Maps dataset names to file paths and schemas. The data root is configurable
+via CT_DATA_ROOT environment variable or defaults to /data (for Docker).
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+DATA_ROOT = Path(os.environ.get("CT_DATA_ROOT", "/data"))
+
+# Dataset registry: name → config
+DATASET_REGISTRY = {
+    "perturbatlas": {
+        "description": "PerturbAtlas: differential expression from 2,066 perturbation experiments",
+        "path_pattern": "perturbatlas/Homo sapiens/*/degs.csv.gz",
+        "format": "csv.gz",
+        "columns": ["column0", "perturb_id", "gene", "baseMean", "log2FoldChange", "lfcSE", "stat", "pvalue", "padj"],
+        "required_columns": ["gene", "log2FoldChange", "pvalue", "padj"],
+        "filterable": ["gene", "perturb_id"],
+    },
+    "chembl": {
+        "description": "ChEMBL v36: bioactivity data for drug-like compounds",
+        "path_pattern": "chembl/chembl/36/*.parquet",
+        "format": "parquet",
+        "columns": [],
+        "required_columns": ["molecule_chembl_id", "target_chembl_id"],
+        "filterable": ["molecule_chembl_id", "target_chembl_id", "pchembl_value"],
+    },
+}
+
+# Maximum rows returned per query
+MAX_QUERY_ROWS = 10000
+DEFAULT_QUERY_LIMIT = 100
+
+
+def discover_datasets() -> dict:
+    """Discover which datasets are actually available on disk."""
+    available = {}
+    for name, config in DATASET_REGISTRY.items():
+        pattern = config["path_pattern"]
+        # Check if any files match the pattern
+        parts = pattern.split("*")
+        base_dir = DATA_ROOT / parts[0].rstrip("/")
+        if base_dir.exists():
+            # Glob for matching files
+            files = list(DATA_ROOT.glob(pattern))
+            if files:
+                available[name] = {
+                    **config,
+                    "n_files": len(files),
+                    "total_size_mb": round(sum(f.stat().st_size for f in files) / 1e6, 1),
+                }
+    return available
+
+
+def validate_schema(engine=None) -> dict[str, dict]:
+    """Validate that on-disk files match expected schemas.
+
+    Samples one file per dataset, reads column names via DuckDB,
+    and compares against DATASET_REGISTRY declarations.
+    Returns {dataset_name: {status, expected, actual, missing, extra}}.
+    """
+    if engine is None:
+        from ct.api.engine import QueryEngine
+        engine = QueryEngine()
+
+    results = {}
+    for name, config in DATASET_REGISTRY.items():
+        pattern = config["path_pattern"]
+        files = list(DATA_ROOT.glob(pattern))
+
+        if not files:
+            results[name] = {
+                "status": "unavailable",
+                "message": "No files found on disk",
+            }
+            continue
+
+        try:
+            actual_cols = engine.sample_columns(pattern)
+        except Exception as e:
+            results[name] = {
+                "status": "error",
+                "message": f"Failed to read columns: {e}",
+            }
+            continue
+
+        expected = set(config.get("columns", []))
+        required = set(config.get("required_columns", []))
+        actual = set(actual_cols)
+
+        missing_required = required - actual
+        missing_declared = expected - actual if expected else set()
+        extra = actual - expected if expected else set()
+
+        if missing_required:
+            status = "invalid"
+        elif missing_declared:
+            status = "warning"
+        else:
+            status = "valid"
+
+        results[name] = {
+            "status": status,
+            "actual_columns": sorted(actual),
+            "expected_columns": sorted(expected) if expected else None,
+            "required_columns": sorted(required),
+            "missing_required": sorted(missing_required) if missing_required else [],
+            "missing_declared": sorted(missing_declared) if missing_declared else [],
+            "extra_columns": sorted(extra) if extra else [],
+        }
+
+    return results

ct/api/engine.py

@@ -0,0 +1,124 @@
+"""
+DuckDB query engine for the ct Data API.
+
+Queries Parquet and CSV.gz files directly without loading into memory.
+Supports predicate pushdown for efficient filtering of large datasets.
+"""
+
+from pathlib import Path
+
+import duckdb
+
+from ct.api.config import DATA_ROOT, MAX_QUERY_ROWS, DEFAULT_QUERY_LIMIT
+
+
+class QueryEngine:
+    """DuckDB-based query engine for tabular data files."""
+
+    def __init__(self, data_root: Path = None):
+        self.data_root = data_root or DATA_ROOT
+        self.conn = duckdb.connect(":memory:")
+
+    def query_parquet(self, file_pattern: str, filters: dict = None,
+                      columns: list = None, limit: int = None,
+                      order_by: str = None) -> list[dict]:
+        """Query Parquet or CSV.gz files with optional filtering.
+
+        Args:
+            file_pattern: Glob pattern relative to data_root (e.g. "chembl/36/*.parquet")
+            filters: Dict of column_name → value for WHERE clauses
+            columns: List of columns to SELECT (default: all)
+            limit: Max rows to return
+            order_by: Column to ORDER BY
+
+        Returns:
+            List of dicts, one per row.
+        """
+        full_pattern = str(self.data_root / file_pattern)
+        limit = min(limit or DEFAULT_QUERY_LIMIT, MAX_QUERY_ROWS)
+
+        # Build SELECT clause
+        select_cols = ", ".join(columns) if columns else "*"
+
+        # Determine read function based on file extension
+        if ".parquet" in file_pattern:
+            source = f"read_parquet('{full_pattern}', union_by_name=true)"
+        elif ".csv.gz" in file_pattern or ".csv" in file_pattern:
+            source = f"read_csv_auto('{full_pattern}', union_by_name=true, ignore_errors=true)"
+        else:
+            source = f"read_parquet('{full_pattern}', union_by_name=true)"
+
+        # Build WHERE clause
+        where_parts = []
+        params = []
+        if filters:
+            for col, val in filters.items():
+                if isinstance(val, list):
+                    placeholders = ", ".join(["?" for _ in val])
+                    where_parts.append(f'"{col}" IN ({placeholders})')
+                    params.extend(val)
+                elif isinstance(val, str) and "%" in val:
+                    where_parts.append(f'"{col}" LIKE ?')
+                    params.append(val)
+                else:
+                    where_parts.append(f'"{col}" = ?')
+                    params.append(val)
+
+        where_clause = f" WHERE {' AND '.join(where_parts)}" if where_parts else ""
+        order_clause = f' ORDER BY "{order_by}"' if order_by else ""
+        limit_clause = f" LIMIT {limit}"
+
+        query = f"SELECT {select_cols} FROM {source}{where_clause}{order_clause}{limit_clause}"
+
+        try:
+            result = self.conn.execute(query, params).fetchdf()
+            return result.to_dict("records")
+        except duckdb.IOException as e:
+            raise FileNotFoundError(f"Data files not found: {full_pattern}") from e
+        except (duckdb.CatalogException, duckdb.BinderException) as e:
+            raise ValueError(f"Query error (bad column?): {e}") from e
+
+    def sample_columns(self, file_pattern: str) -> list[str]:
+        """Read column names from first matching file without loading data."""
+        full_pattern = str(self.data_root / file_pattern)
+
+        if ".parquet" in file_pattern:
+            source = f"read_parquet('{full_pattern}', union_by_name=true)"
+        elif ".csv.gz" in file_pattern or ".csv" in file_pattern:
+            source = f"read_csv_auto('{full_pattern}', union_by_name=true, ignore_errors=true)"
+        else:
+            source = f"read_parquet('{full_pattern}', union_by_name=true)"
+
+        query = f"SELECT * FROM {source} LIMIT 0"
+        result = self.conn.execute(query)
+        return [desc[0] for desc in result.description]
+
+    def count(self, file_pattern: str, filters: dict = None) -> int:
+        """Count rows matching filters."""
+        full_pattern = str(self.data_root / file_pattern)
+
+        if ".parquet" in file_pattern:
+            source = f"read_parquet('{full_pattern}', union_by_name=true)"
+        else:
+            source = f"read_csv_auto('{full_pattern}', union_by_name=true)"
+
+        where_parts = []
+        params = []
+        if filters:
+            for col, val in filters.items():
+                if isinstance(val, list):
+                    placeholders = ", ".join(["?" for _ in val])
+                    where_parts.append(f'"{col}" IN ({placeholders})')
+                    params.extend(val)
+                else:
+                    where_parts.append(f'"{col}" = ?')
+                    params.append(val)
+
+        where_clause = f" WHERE {' AND '.join(where_parts)}" if where_parts else ""
+        query = f"SELECT COUNT(*) as cnt FROM {source}{where_clause}"
+
+        try:
+            result = self.conn.execute(query, params).fetchone()
+            return result[0] if result else 0
+        except duckdb.IOException:
+            return 0