celltype-cli 0.1.0__py3-none-any.whl

ct/data/downloader.py

@@ -0,0 +1,238 @@
+"""
+Dataset downloader for ct.
+
+Manages downloading and caching of common drug discovery datasets.
+Supports automatic downloads for open-access datasets and guided
+instructions for datasets requiring portal authentication.
+"""
+
+import gzip
+import hashlib
+import shutil
+from pathlib import Path
+
+import httpx
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, BarColumn, DownloadColumn, TransferSpeedColumn
+from rich.table import Table
+
+from ct.agent.config import Config
+
+console = Console()
+
+# Download timeout in seconds (large files like CRISPRGeneEffect ~700MB need more time)
+DOWNLOAD_TIMEOUT = 600
+
+DATASETS = {
+    "depmap": {
+        "description": "DepMap CRISPR gene dependencies, mutations, cell line metadata (24Q4)",
+        "files": {
+            "CRISPRGeneEffect.csv": "https://ndownloader.figshare.com/files/51064667",
+            "Model.csv": "https://ndownloader.figshare.com/files/51065297",
+            "OmicsSomaticMutationsMatrixDamaging.csv": "https://ndownloader.figshare.com/files/51065747",
+        },
+        "source": "https://plus.figshare.com/articles/dataset/DepMap_24Q4_Public/27993248",
+        "auto_download": True,
+        "size_hint": "~580MB",
+    },
+    "prism": {
+        "description": "PRISM cell viability screening data",
+        "files": {
+            "prism_LFC_COLLAPSED.csv": None,
+        },
+        "source": "https://depmap.org/repurposing/",
+        "auto_download": False,
+        "note": "PRISM data requires manual download from https://depmap.org/repurposing/ or symlink from existing data.",
+        "size_hint": "~600MB",
+    },
+    "l1000": {
+        "description": "L1000 landmark gene expression signatures (978 genes)",
+        "files": {
+            "l1000_landmark_only.parquet": None,
+        },
+        "source": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE92742",
+        "auto_download": False,
+        "note": "Run: python scripts/prepare_l1000.py to download from GEO and extract landmark subset.",
+        "size_hint": "~200MB",
+    },
+    "msigdb": {
+        "description": "MSigDB gene set collections (Hallmark, KEGG, Reactome, GO)",
+        "files": {
+            "h.all.v2024.1.Hs.json": "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2024.1.Hs/h.all.v2024.1.Hs.json",
+            "c2.cp.kegg_legacy.v2024.1.Hs.json": "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2024.1.Hs/c2.cp.kegg_legacy.v2024.1.Hs.json",
+            "c2.cp.reactome.v2024.1.Hs.json": "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2024.1.Hs/c2.cp.reactome.v2024.1.Hs.json",
+            "c5.go.bp.v2024.1.Hs.json": "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2024.1.Hs/c5.go.bp.v2024.1.Hs.json",
+        },
+        "source": "https://www.gsea-msigdb.org/gsea/msigdb/",
+        "auto_download": True,
+    },
+    "string": {
+        "description": "STRING protein-protein interaction network (human)",
+        "files": {
+            "9606.protein.links.v12.0.txt.gz": "https://stringdb-downloads.org/download/protein.links.v12.0/9606.protein.links.v12.0.txt.gz",
+        },
+        "source": "https://string-db.org/",
+        "auto_download": True,
+    },
+    "alphafold": {
+        "description": "AlphaFold predicted protein structures (downloaded on demand per-protein)",
+        "files": {},
+        "source": "https://alphafold.ebi.ac.uk/",
+        "auto_download": False,
+        "note": "Structures are fetched on-demand by structure.alphafold_fetch tool.",
+    },
+}
+
+
+def _download_file(url: str, dest: Path, desc: str = None) -> bool:
+    """Download a file with progress bar. Returns True on success."""
+    desc = desc or dest.name
+    try:
+        with httpx.stream("GET", url, timeout=DOWNLOAD_TIMEOUT, follow_redirects=True) as resp:
+            if resp.status_code != 200:
+                console.print(f"  [red]HTTP {resp.status_code} for {url}[/red]")
+                return False
+
+            total = int(resp.headers.get("content-length", 0))
+
+            with Progress(
+                SpinnerColumn(),
+                "[progress.description]{task.description}",
+                BarColumn(),
+                DownloadColumn(),
+                TransferSpeedColumn(),
+                console=console,
+            ) as progress:
+                task = progress.add_task(f"  {desc}", total=total or None)
+                with open(dest, "wb") as f:
+                    for chunk in resp.iter_bytes(chunk_size=8192):
+                        f.write(chunk)
+                        progress.advance(task, len(chunk))
+
+        return True
+    except httpx.HTTPError as e:
+        console.print(f"  [red]Download failed: {e}[/red]")
+        if dest.exists():
+            dest.unlink()
+        return False
+
+
+def download_dataset(name: str, output: Path = None):
+    """Download a dataset."""
+    if name == "--all" or name == "all":
+        download_all(output)
+        return
+
+    if name not in DATASETS:
+        console.print(f"[red]Unknown dataset: {name}[/red]")
+        console.print(f"Available: {', '.join(DATASETS.keys())}")
+        return
+
+    ds = DATASETS[name]
+    cfg = Config.load()
+    dest = output or Path(cfg.get("data.base")) / name
+    dest.mkdir(parents=True, exist_ok=True)
+
+    console.print(f"\n[cyan]{name}:[/cyan] {ds['description']}")
+    if ds.get("size_hint"):
+        console.print(f"  Size: {ds['size_hint']}")
+    console.print(f"  Destination: {dest}")
+
+    if not ds.get("auto_download"):
+        # Manual download required
+        if "note" in ds:
+            console.print(f"  [yellow]{ds['note']}[/yellow]")
+        console.print(f"  Source: {ds['source']}")
+        console.print(f"  Files needed:")
+        for fname in ds["files"]:
+            fpath = dest / fname
+            status = "[green]found[/green]" if fpath.exists() else "[red]missing[/red]"
+            console.print(f"    {fname} — {status}")
+        console.print(f"\n  Download from {ds['source']} and place in {dest}/")
+        console.print(f"  Then run: [cyan]ct config set data.{name} {dest}[/cyan]")
+        return
+
+    # Automatic download
+    downloaded = 0
+    skipped = 0
+    failed = 0
+
+    for fname, url in ds["files"].items():
+        fpath = dest / fname
+        if fpath.exists():
+            console.print(f"  [dim]{fname} — already exists, skipping[/dim]")
+            skipped += 1
+            continue
+
+        if url is None:
+            console.print(f"  [yellow]{fname} — no download URL, skip[/yellow]")
+            failed += 1
+            continue
+
+        if _download_file(url, fpath, fname):
+            downloaded += 1
+        else:
+            failed += 1
+
+    # Summary
+    total = len(ds["files"])
+    console.print(f"\n  [green]{downloaded} downloaded[/green], {skipped} skipped, ", end="")
+    if failed:
+        console.print(f"[red]{failed} failed[/red]")
+    else:
+        console.print(f"0 failed")
+
+    # Auto-configure data path after successful download
+    if downloaded > 0 or skipped > 0:
+        cfg.set(f"data.{name}", str(dest))
+        cfg.save()
+        console.print(f"  [green]Auto-configured data.{name} = {dest}[/green]")
+
+
+def download_all(output: Path = None):
+    """Download all auto-downloadable datasets."""
+    auto_datasets = [name for name, ds in DATASETS.items() if ds.get("auto_download")]
+    console.print(f"[cyan]Downloading {len(auto_datasets)} datasets: {', '.join(auto_datasets)}[/cyan]")
+    for name in auto_datasets:
+        download_dataset(name, output=output)
+
+
+def dataset_status() -> Table:
+    """Check which datasets are available locally."""
+    cfg = Config.load()
+    base = Path(cfg.get("data.base"))
+
+    table = Table(title="Dataset Status")
+    table.add_column("Dataset", style="cyan")
+    table.add_column("Status")
+    table.add_column("Files", style="dim")
+    table.add_column("Auto-DL")
+
+    for name, ds in DATASETS.items():
+        # Check custom config path first, then default location
+        custom_path = cfg.get(f"data.{name}")
+        path = Path(custom_path) if custom_path else base / name
+
+        expected = set(ds["files"].keys())
+        found = set()
+        if path.exists():
+            existing = {f.name for f in path.iterdir() if f.is_file()}
+            found = expected & existing
+
+        if not expected:
+            status = "[dim]on-demand[/dim]"
+            files_str = "-"
+        elif found == expected:
+            status = "[green]complete[/green]"
+            files_str = f"{len(found)}/{len(expected)}"
+        elif found:
+            status = "[yellow]partial[/yellow]"
+            files_str = f"{len(found)}/{len(expected)}"
+        else:
+            status = "[red]missing[/red]"
+            files_str = f"0/{len(expected)}"
+
+        auto = "[green]yes[/green]" if ds.get("auto_download") else "[dim]manual[/dim]"
+        table.add_row(name, status, files_str, auto)
+
+    return table

ct/data/loaders.py

@@ -0,0 +1,252 @@
+"""
+Data loaders for common drug discovery datasets.
+
+Each loader checks configured paths, then the ct-data sister project,
+and supports both CSV and Parquet formats.
+Data paths are configured via ct config.
+"""
+
+import pandas as pd
+from pathlib import Path
+from functools import lru_cache
+
+from ct.agent.config import Config
+
+
+# Search order for data files (first match wins)
+_DATA_SEARCH_PATHS = [
+    Path.home() / "Projects" / "CellType" / "ct-data",         # Primary: ct-data repo
+    Path.home() / "Projects" / "CellType" / "crews-glue-discovery",  # Legacy fallback
+]
+
+
+def _data_path(key: str) -> Path:
+    """Get configured data path."""
+    cfg = Config.load()
+    path = cfg.get(f"data.{key}")
+    if path:
+        return Path(path)
+    base = cfg.get("data.base")
+    if base:
+        return Path(base) / key
+    return Path.home() / ".ct" / "data" / key
+
+
+def _find_file(name: str, subdirs: list[str] = None) -> Path | None:
+    """Search for a file across configured paths and common locations."""
+    subdirs = subdirs or [""]
+    stem = Path(name).stem
+
+    # 1. Check configured data.base
+    cfg = Config.load()
+    base = cfg.get("data.base")
+    search_dirs = []
+    if base:
+        search_dirs.append(Path(base))
+
+    # 2. Check known data locations
+    search_dirs.extend(_DATA_SEARCH_PATHS)
+
+    # 3. Check ~/.ct/data
+    search_dirs.append(Path.home() / ".ct" / "data")
+
+    for base_dir in search_dirs:
+        for sub in subdirs:
+            d = base_dir / sub if sub else base_dir
+            if not d.exists():
+                continue
+            # Exact match
+            candidate = d / name
+            if candidate.exists():
+                return candidate
+            # Try parquet variant
+            parquet = d / f"{stem}.parquet"
+            if parquet.exists():
+                return parquet
+
+    return None
+
+
+def _resolve_path(p: Path, filenames: list[str]) -> Path | None:
+    """If p is a directory, search for one of filenames inside it. If p is a file, return it."""
+    if p.is_file():
+        return p
+    if p.is_dir():
+        for name in filenames:
+            candidate = p / name
+            if candidate.exists():
+                return candidate
+    return None
+
+
+def _read_tabular(path: Path, **kwargs) -> pd.DataFrame:
+    """Read a CSV or Parquet file based on extension."""
+    if path.suffix == ".parquet":
+        return pd.read_parquet(path, **{k: v for k, v in kwargs.items() if k != "index_col"})
+    return pd.read_csv(path, **kwargs)
+
+
+@lru_cache(maxsize=1)
+def load_crispr() -> pd.DataFrame:
+    """Load DepMap CRISPR gene effect data."""
+    # Try configured path first
+    path = _find_file("CRISPRGeneEffect.csv", subdirs=["", "depmap"])
+    if path is None:
+        raise FileNotFoundError(
+            "DepMap CRISPR data not found. "
+            "Run: ct data pull depmap\n"
+            "Or set: ct config set data.base /path/to/data"
+        )
+    df = _read_tabular(path, index_col=0)
+    # Clean column names: "TP53 (7157)" → "TP53"
+    df.columns = [c.split(' (')[0] for c in df.columns]
+    return df
+
+
+@lru_cache(maxsize=1)
+def load_model_metadata() -> pd.DataFrame:
+    """Load DepMap cell line metadata."""
+    path = _find_file("Model.csv", subdirs=["", "depmap"])
+    if path is None:
+        raise FileNotFoundError(
+            "Model metadata not found. "
+            "Run: ct data pull depmap"
+        )
+    return _read_tabular(path)
+
+
+@lru_cache(maxsize=1)
+def load_proteomics() -> pd.DataFrame:
+    """Load proteomics LFC matrix."""
+    _prot_files = ["proteomics_log2fc_matrix.parquet", "proteomics_log2fc_matrix.csv", "merged_proteomics.csv"]
+    # Check configured path
+    cfg = Config.load()
+    explicit = cfg.get("data.proteomics")
+    if explicit:
+        p = _resolve_path(Path(explicit), _prot_files)
+        if p:
+            return _read_tabular(p, index_col=0)
+
+    # Search common locations
+    for name in ["merged_proteomics.csv", "proteomics_log2fc_matrix.parquet",
+                 "proteomics_log2fc_matrix.csv"]:
+        path = _find_file(name)
+        if path:
+            return _read_tabular(path, index_col=0)
+
+    raise FileNotFoundError(
+        "Proteomics data not found. "
+        "Set: ct config set data.proteomics /path/to/file"
+    )
+
+
+@lru_cache(maxsize=1)
+def load_l1000() -> pd.DataFrame:
+    """Load L1000 landmark gene expression data.
+
+    Prefers the compound-named profiles parquet (19,811 compounds × 978 genes)
+    built from the Broad LINCS GSE92742 Level 5 GCTX. Falls back to legacy
+    formats (YU-indexed parquet/CSV).
+    """
+    cfg = Config.load()
+
+    # 1. Prefer compound-named profiles (from LINCS GCTX, 19,811 compounds)
+    lincs_path = Path("/mnt2/bronze/lincs/l1000_compound_profiles.parquet")
+    if lincs_path.exists():
+        return _read_tabular(lincs_path, index_col=0)
+
+    # 2. Check data.base for compound profiles
+    base = cfg.get("data.base")
+    if base:
+        candidate = Path(base) / "lincs" / "l1000_compound_profiles.parquet"
+        if candidate.exists():
+            return _read_tabular(candidate, index_col=0)
+
+    # 3. Check explicit config (may point to legacy YU-indexed data)
+    explicit = cfg.get("data.l1000")
+    if explicit:
+        _l1000_files = [
+            "l1000_compound_profiles.parquet",
+            "l1000_landmark_only.parquet",
+            "L1000_landmark_LFC.csv",
+            "l1000_expression_matrix.parquet",
+            "l1000_landmark_only.csv",
+        ]
+        p = _resolve_path(Path(explicit), _l1000_files)
+        if p:
+            return _read_tabular(p, index_col=0)
+
+    # 4. Fall back to legacy formats
+    for name in ["l1000_compound_profiles.parquet", "L1000_landmark_LFC.csv",
+                 "l1000_landmark_only.parquet", "l1000_expression_matrix.parquet",
+                 "l1000_landmark_only.csv"]:
+        path = _find_file(name, subdirs=["", "lincs", "l1000"])
+        if path:
+            return _read_tabular(path, index_col=0)
+
+    raise FileNotFoundError(
+        "L1000 data not found. "
+        "Set: ct config set data.l1000 /path/to/file"
+    )
+
+
+@lru_cache(maxsize=1)
+def load_prism() -> pd.DataFrame:
+    """Load PRISM cell viability data."""
+    _prism_files = ["prism_LFC_COLLAPSED.csv", "prism_LFC_COLLAPSED.parquet"]
+    cfg = Config.load()
+    explicit = cfg.get("data.prism")
+    if explicit:
+        p = _resolve_path(Path(explicit), _prism_files)
+        if p:
+            return _read_tabular(p)
+
+    for name in ["prism_LFC_COLLAPSED.csv", "prism_LFC_COLLAPSED.parquet"]:
+        path = _find_file(name)
+        if path:
+            return _read_tabular(path)
+
+    raise FileNotFoundError(
+        "PRISM data not found. "
+        "Run: ct data pull prism\n"
+        "Or set: ct config set data.prism /path/to/file"
+    )
+
+
+def load_mutations() -> pd.DataFrame:
+    """Load DepMap somatic mutation data."""
+    path = _find_file("OmicsSomaticMutationsMatrixDamaging.csv", subdirs=["", "depmap"])
+    if path is None:
+        raise FileNotFoundError(
+            "Mutation data not found. "
+            "Run: ct data pull depmap"
+        )
+
+    df = _read_tabular(path)
+    meta_cols = ['Unnamed: 0', 'SequencingID', 'ModelConditionID',
+                 'IsDefaultEntryForModel', 'IsDefaultEntryForMC']
+
+    if 'IsDefaultEntryForModel' in df.columns:
+        df = df[df['IsDefaultEntryForModel'] == 'Yes']
+
+    if 'ModelID' in df.columns:
+        df = df.set_index('ModelID')
+    elif 'Unnamed: 0' in df.columns and df['Unnamed: 0'].astype(str).str.startswith('ACH-').any():
+        df = df.set_index('Unnamed: 0')
+        df.index.name = 'ModelID'
+    df = df.drop(columns=[c for c in meta_cols if c in df.columns], errors='ignore')
+    df.columns = [c.split(' (')[0] for c in df.columns]
+    return df
+
+
+def load_msigdb(collection: str = "h") -> dict:
+    """Load MSigDB gene sets."""
+    import json
+    # Try both naming patterns: "h.all.v2024.1.Hs.json" and "c2.cp.kegg_legacy.v2024.1.Hs.json"
+    for pattern in [f"{collection}.all.v2024.1.Hs.json", f"{collection}.v2024.1.Hs.json"]:
+        path = _find_file(pattern, subdirs=["", "msigdb"])
+        if path and path.exists():
+            with open(path) as f:
+                return json.load(f)
+
+    raise FileNotFoundError("MSigDB data not found. Run: ct data pull msigdb")

ct/kb/__init__.py

@@ -0,0 +1,5 @@
+"""Knowledge substrate package for ct."""
+
+from ct.kb.substrate import KnowledgeSubstrate
+
+__all__ = ["KnowledgeSubstrate"]

ct/kb/benchmarks.py

@@ -0,0 +1,147 @@
+"""
+Domain benchmark harness and release gating.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+import json
+from pathlib import Path
+from typing import Any
+
+try:
+    from ct.agent.quality import evaluate_synthesis_quality
+except ImportError:
+    evaluate_synthesis_quality = None
+
+
+@dataclass
+class BenchmarkCase:
+    name: str
+    domain: str
+    synthesis: str
+    completed_step_ids: list[int]
+    expect_pass: bool = True
+
+
+@dataclass
+class BenchmarkResult:
+    name: str
+    domain: str
+    passed: bool
+    expected_pass: bool
+    issues: list[str]
+
+
+class BenchmarkSuite:
+    """Runs deterministic benchmark cases for synthesis quality gates."""
+
+    def __init__(self, cases: list[BenchmarkCase]):
+        self.cases = cases
+
+    @classmethod
+    def load(cls, path: Path | None = None) -> "BenchmarkSuite":
+        source = path or (Path.cwd() / "configs" / "pharma_benchmarks.json")
+        if source.exists():
+            data = json.loads(source.read_text(encoding="utf-8"))
+            raw_cases = data.get("cases", [])
+            cases = [BenchmarkCase(**item) for item in raw_cases]
+            if cases:
+                return cls(cases)
+        return cls(default_cases())
+
+    def run(self) -> dict[str, Any]:
+        results: list[BenchmarkResult] = []
+        for case in self.cases:
+            quality = evaluate_synthesis_quality(
+                case.synthesis,
+                completed_step_ids=set(case.completed_step_ids),
+                require_key_evidence=True,
+                min_next_steps=2,
+                max_next_steps=3,
+            )
+            passed = quality.ok
+            results.append(
+                BenchmarkResult(
+                    name=case.name,
+                    domain=case.domain,
+                    passed=passed,
+                    expected_pass=case.expect_pass,
+                    issues=quality.issues,
+                )
+            )
+
+        expected_correct = 0
+        for result in results:
+            if result.passed == result.expected_pass:
+                expected_correct += 1
+        total = len(results)
+        pass_rate = expected_correct / max(total, 1)
+        domain_scores = self._domain_scores(results)
+        return {
+            "total_cases": total,
+            "expected_behavior_matches": expected_correct,
+            "pass_rate": round(pass_rate, 4),
+            "domain_scores": domain_scores,
+            "results": [asdict(r) for r in results],
+        }
+
+    @staticmethod
+    def _domain_scores(results: list[BenchmarkResult]) -> dict[str, float]:
+        buckets: dict[str, list[bool]] = {}
+        for result in results:
+            buckets.setdefault(result.domain, []).append(result.passed == result.expected_pass)
+        scores = {}
+        for domain, vals in buckets.items():
+            scores[domain] = round(sum(1 for v in vals if v) / max(len(vals), 1), 4)
+        return scores
+
+    @staticmethod
+    def gate(summary: dict[str, Any], *, min_pass_rate: float = 0.9) -> dict[str, Any]:
+        actual = float(summary.get("pass_rate", 0.0))
+        ok = actual >= min_pass_rate
+        return {
+            "ok": ok,
+            "min_pass_rate": min_pass_rate,
+            "actual_pass_rate": actual,
+            "message": (
+                f"Benchmark gate passed ({actual:.2%} >= {min_pass_rate:.2%})"
+                if ok
+                else f"Benchmark gate failed ({actual:.2%} < {min_pass_rate:.2%})"
+            ),
+        }
+
+
+def default_cases() -> list[BenchmarkCase]:
+    """Fallback deterministic cases when benchmark file is absent."""
+    return [
+        BenchmarkCase(
+            name="target_validation_grounded",
+            domain="target_validation",
+            synthesis=(
+                "## Answer\nSignal supports target progression.\n\n"
+                "## Key Evidence\n- Genetic support observed [step:1]\n"
+                "- Expression concordance supports mechanism [step:2]\n\n"
+                "## Confidence & Caveats\n- Moderate confidence.\n\n"
+                "## Suggested Next Steps\n"
+                "1. Run genomics.coloc for the top locus.\n"
+                "2. Run target.disease_association for indication prioritization.\n"
+            ),
+            completed_step_ids=[1, 2],
+            expect_pass=True,
+        ),
+        BenchmarkCase(
+            name="moa_missing_citation",
+            domain="moa_inference",
+            synthesis=(
+                "## Answer\nMechanism inferred.\n\n"
+                "## Key Evidence\n- Strong MOA pattern without citation\n\n"
+                "## Confidence & Caveats\n- Preliminary.\n\n"
+                "## Suggested Next Steps\n"
+                "1. Run expression.pathway_enrichment for the top signature.\n"
+                "2. Run repurposing.cmap_query on the differential profile.\n"
+            ),
+            completed_step_ids=[1, 2],
+            expect_pass=False,
+        ),
+    ]