omicsync 0.1.0__py3-none-any.whl

omicsync/loaders/tcga.py

@@ -0,0 +1,251 @@
+"""TCGA data loaders for omicsync."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence, Union
+
+import pandas as pd
+
+from omicsync.core.dataset import OmicsDataset
+from omicsync.core.modality import make_modality, OmicsModality
+from omicsync.core.sample_index import SampleIndex
+from omicsync.utils.barcode import truncate_to_participant
+from omicsync.utils.logging import get_logger
+
+logger = get_logger("loaders.tcga")
+
+# Expected filename patterns for each modality within a TCGA data directory.
+# Pattern: {cancer_type}_{modality}.{ext}
+_MODALITY_FILE_PATTERNS = {
+    "rna": ["{cancer_type}_rna.tsv", "{cancer_type}_rna.csv",
+            "{cancer_type}_rna_fpkm.tsv", "{cancer_type}_htseq_counts.tsv"],
+    "mutations": ["{cancer_type}_mutations.tsv", "{cancer_type}_mutations.maf",
+                  "{cancer_type}_somatic.maf"],
+    "methylation": ["{cancer_type}_methylation.tsv", "{cancer_type}_methylation.csv"],
+    "cnv": ["{cancer_type}_cnv.tsv", "{cancer_type}_cnv.csv",
+            "{cancer_type}_gistic2.tsv"],
+    "protein": ["{cancer_type}_protein.tsv", "{cancer_type}_protein.csv",
+                "{cancer_type}_rppa.tsv"],
+}
+
+# GDC data portal file type identifiers used in the manifest helper
+_GDC_DATA_TYPES = {
+    "rna": "Gene Expression Quantification",
+    "mutations": "Masked Somatic Mutation",
+    "methylation": "Methylation Beta Value",
+    "cnv": "Copy Number Segment",
+    "protein": "Protein Expression Quantification",
+}
+
+
+def _find_modality_file(
+    data_dir: Path,
+    cancer_type: str,
+    modality: str,
+) -> Optional[Path]:
+    """Search *data_dir* for a file matching known naming conventions."""
+    patterns = _MODALITY_FILE_PATTERNS.get(modality, [])
+    for pattern in patterns:
+        candidate = data_dir / pattern.format(cancer_type=cancer_type.upper())
+        if candidate.exists():
+            return candidate
+        candidate = data_dir / pattern.format(cancer_type=cancer_type.lower())
+        if candidate.exists():
+            return candidate
+    # Fallback: look for any file containing the modality name
+    for f in data_dir.iterdir():
+        if modality in f.name.lower() and f.suffix in (".tsv", ".csv", ".maf"):
+            return f
+    return None
+
+
+def _load_maf_mutations(path: Path) -> pd.DataFrame:
+    """Parse a MAF file into a binary genes × samples mutation matrix."""
+    df = pd.read_csv(path, sep="\t", comment="#", low_memory=False)
+
+    required = {"Hugo_Symbol", "Tumor_Sample_Barcode"}
+    missing = required - set(df.columns)
+    if missing:
+        raise ValueError(
+            f"MAF file missing required columns: {missing}. "
+            f"Found: {df.columns.tolist()[:15]}."
+        )
+
+    df["sample_id"] = df["Tumor_Sample_Barcode"].apply(
+        lambda x: truncate_to_participant(x) if isinstance(x, str) and x.startswith("TCGA") else x
+    )
+    mat = (
+        df.groupby(["sample_id", "Hugo_Symbol"])
+        .size()
+        .unstack(fill_value=0)
+        .clip(upper=1)
+    )
+    return mat.astype(float)
+
+
+def _load_generic_matrix(path: Path) -> pd.DataFrame:
+    """Load a TSV/CSV as a sample-indexed matrix.
+
+    Tries to detect whether samples are rows or columns.
+    """
+    sep = "\t" if path.suffix in (".tsv", ".maf") else ","
+    df = pd.read_csv(path, sep=sep, index_col=0, low_memory=False)
+
+    # Heuristic: if more than half of values in first column look numeric,
+    # samples are rows; otherwise transpose.
+    numeric_frac = pd.to_numeric(df.iloc[:, 0], errors="coerce").notna().mean()
+    if numeric_frac < 0.5:
+        df = df.T
+
+    df = df.apply(pd.to_numeric, errors="coerce")
+    return df
+
+
+def load_tcga_files(
+    data_dir: Union[str, Path],
+    cancer_type: str,
+    modalities: Sequence[str],
+) -> OmicsDataset:
+    """Load TCGA data from local files into an :class:`~omicsync.core.dataset.OmicsDataset`.
+
+    Parameters
+    ----------
+    data_dir:
+        Path to directory containing TCGA data files.  Expected naming:
+        ``{cancer_type}_{modality}.tsv`` or ``{cancer_type}_{modality}.csv``.
+        For mutations, ``.maf`` files are also supported.
+    cancer_type:
+        TCGA cancer type abbreviation, e.g. ``"BRCA"``.
+    modalities:
+        List of modality names to load, e.g.
+        ``["rna", "mutations", "methylation"]``.
+
+    Returns
+    -------
+    OmicsDataset
+
+    Raises
+    ------
+    FileNotFoundError
+        If *data_dir* does not exist.
+    ValueError
+        If no file is found for a requested modality.
+    """
+    data_dir = Path(data_dir)
+    if not data_dir.exists():
+        raise FileNotFoundError(f"Data directory not found: {data_dir}")
+
+    loaded: Dict[str, OmicsModality] = {}
+
+    for modality in modalities:
+        path = _find_modality_file(data_dir, cancer_type, modality)
+        if path is None:
+            raise ValueError(
+                f"No file found for modality {modality!r} in {data_dir}. "
+                f"Expected patterns like: "
+                + str(_MODALITY_FILE_PATTERNS.get(modality, []))
+            )
+
+        logger.info("load_tcga_files: loading %r from %s.", modality, path.name)
+
+        if modality == "mutations" and path.suffix == ".maf":
+            df = _load_maf_mutations(path)
+        else:
+            df = _load_generic_matrix(path)
+
+        # Harmonise TCGA barcodes if detected
+        if all(str(idx).startswith("TCGA") for idx in df.index[:5]):
+            df.index = pd.Index(
+                [truncate_to_participant(str(i)) for i in df.index],
+                name=df.index.name,
+            )
+            df = df[~df.index.duplicated(keep="first")]
+            logger.info(
+                "load_tcga_files: truncated %r sample IDs to participant level.",
+                modality,
+            )
+
+        loaded[modality] = make_modality(df, modality_type=modality, source="tcga")
+
+    dataset = OmicsDataset(loaded, study_id=f"TCGA-{cancer_type.upper()}")
+
+    # Print coverage report
+    coverage = dataset.sample_coverage
+    logger.info(
+        "TCGA %s coverage: %d total samples, %d complete cases.",
+        cancer_type.upper(),
+        len(coverage),
+        dataset.n_complete_cases,
+    )
+    return dataset
+
+
+def download_tcga_manifest(
+    cancer_type: str,
+    modalities: Sequence[str],
+    output_dir: Union[str, Path],
+) -> None:
+    """Print GDC data portal instructions for downloading TCGA data.
+
+    This function does NOT download anything itself — TCGA data via the GDC
+    API requires authentication tokens.  It prints the required ``curl``
+    commands and GDC portal URLs so you can retrieve the data manually.
+
+    Parameters
+    ----------
+    cancer_type:
+        TCGA cancer type abbreviation, e.g. ``"BRCA"``.
+    modalities:
+        Modalities to retrieve, e.g. ``["rna", "mutations"]``.
+    output_dir:
+        Directory where you plan to save the files (used in printed commands).
+    """
+    ct = cancer_type.upper()
+    output_dir = Path(output_dir)
+
+    instructions = [
+        f"",
+        f"TCGA {ct} data download instructions",
+        f"{'=' * 50}",
+        f"",
+        f"To download TCGA data you need a GDC account and an API token.",
+        f"",
+        f"1. Create/log into your account at: https://portal.gdc.cancer.gov/",
+        f"2. Download your API token from:",
+        f"   Profile → Download Token  (valid 30 days)",
+        f"3. Save the token to a file, e.g. ~/gdc-token.txt",
+        f"",
+        f"Data types needed for {ct}:",
+        f"",
+    ]
+
+    for modality in modalities:
+        data_type = _GDC_DATA_TYPES.get(modality, modality)
+        filename = f"{ct}_{modality}.tsv"
+        instructions += [
+            f"  [{modality.upper()}]",
+            f"  GDC data type : {data_type}",
+            f"  Portal filter : https://portal.gdc.cancer.gov/repository"
+            f"?filters=%7B%22op%22%3A%22and%22%2C%22content%22%3A%5B%7B%22op%22%3A%22in%22%2C%22content%22%3A%7B%22field%22%3A%22cases.project.project_id%22%2C%22value%22%3A%5B%22TCGA-{ct}%22%5D%7D%7D%5D%7D"
+            f"&facetTab=files&searchTableTab=files",
+            f"  Save to       : {output_dir / filename}",
+            f"",
+        ]
+
+    instructions += [
+        f"Alternative: use the GDC client tool",
+        f"  pip install gdc-client",
+        f"  gdc-client download -t ~/gdc-token.txt -d {output_dir} <manifest-file>",
+        f"",
+        f"Alternative: use TCGAbiolinks (R):",
+        f"  library(TCGAbiolinks)",
+        f"  query <- GDCquery(project = 'TCGA-{ct}', ...)",
+        f"  GDCdownload(query)",
+        f"",
+    ]
+
+    for line in instructions:
+        logger.info(line)
+        print(line)

omicsync/normalisation/__init__.py

@@ -0,0 +1,5 @@
+"""Normalisation methods for each omics modality."""
+
+from omicsync.normalisation import cnv, methylation, mutations, protein, rna
+
+__all__ = ["rna", "methylation", "cnv", "mutations", "protein"]

omicsync/normalisation/cnv.py

@@ -0,0 +1,97 @@
+"""Copy number variation normalisation utilities."""
+
+from __future__ import annotations
+
+from typing import Tuple
+
+import numpy as np
+import pandas as pd
+
+from omicsync.utils.logging import get_logger
+
+logger = get_logger("normalisation.cnv")
+
+
+def centre_diploid(df: pd.DataFrame, diploid: float = 2.0) -> pd.DataFrame:
+    """Subtract the diploid baseline from all values.
+
+    Parameters
+    ----------
+    df:
+        CNV matrix (samples × genes). Values are typically absolute copy
+        number estimates (centred around 2 for diploid).
+    diploid:
+        The baseline copy number to subtract (default 2.0).
+
+    Returns
+    -------
+    pandas.DataFrame
+        Copy number deviation from diploid.
+    """
+    result = df.values.astype(float) - diploid
+    logger.info("centre_diploid: subtracted diploid=%.1f from %s.", diploid, df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)
+
+
+def log2_ratio(df: pd.DataFrame, pseudo: float = 1.0) -> pd.DataFrame:
+    """Compute log2 copy-number ratio relative to diploid.
+
+    Assumes input is already centred (deviation from diploid = 0).
+    Adds *pseudo* before log2 to handle zero deviations.
+
+    Parameters
+    ----------
+    df:
+        CNV deviation matrix (output of :func:`centre_diploid`).
+    pseudo:
+        Pseudocount added before log2 transform (default 1.0).
+
+    Returns
+    -------
+    pandas.DataFrame
+        log2 ratio matrix.
+    """
+    data = df.values.astype(float)
+    shifted = data + pseudo
+    with np.errstate(divide="ignore", invalid="ignore"):
+        result = np.where(shifted > 0, np.log2(shifted), np.nan)
+    logger.info("log2_ratio: applied log2 to %s.", df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)
+
+
+def discretise(
+    df: pd.DataFrame,
+    thresholds: Tuple[float, float, float, float] = (-1.0, -0.3, 0.3, 1.0),
+) -> pd.DataFrame:
+    """Discretise copy-number values into -2/-1/0/1/2 states.
+
+    Parameters
+    ----------
+    df:
+        CNV matrix (log2 ratios or deviations from diploid).
+    thresholds:
+        Four boundary values ``(deep_del, del, amp, high_amp)`` that define
+        the five copy-number states:
+
+        * < deep_del  → -2 (deep deletion)
+        * < del       → -1 (deletion)
+        * <= amp      → 0  (diploid)
+        * <= high_amp → 1  (gain)
+        * > high_amp  → 2  (amplification)
+
+    Returns
+    -------
+    pandas.DataFrame
+        Integer copy-number state matrix.
+    """
+    if len(thresholds) != 4:
+        raise ValueError("thresholds must have exactly 4 values.")
+    t1, t2, t3, t4 = thresholds
+    data = df.values.astype(float)
+    result = np.zeros_like(data, dtype=float)
+    result[data < t1] = -2.0
+    result[(data >= t1) & (data < t2)] = -1.0
+    result[(data > t3) & (data <= t4)] = 1.0
+    result[data > t4] = 2.0
+    logger.info("discretise: discretised CNV matrix %s.", df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)

omicsync/normalisation/methylation.py

@@ -0,0 +1,131 @@
+"""Methylation normalisation utilities."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from omicsync.utils.logging import get_logger
+
+logger = get_logger("normalisation.methylation")
+
+
+def beta_to_m(df: pd.DataFrame) -> pd.DataFrame:
+    """Convert beta values to M-values: log2(beta / (1 - beta)).
+
+    Parameters
+    ----------
+    df:
+        Beta value matrix (samples × CpG sites). Values must be in (0, 1).
+
+    Returns
+    -------
+    pandas.DataFrame
+        M-value matrix.
+
+    Raises
+    ------
+    ValueError
+        If any value is outside [0, 1].
+    """
+    data = df.values.astype(float)
+    finite = data[np.isfinite(data)]
+    if len(finite) > 0 and (finite.min() < 0 or finite.max() > 1):
+        raise ValueError(
+            f"beta_to_m: beta values must be in [0, 1]. "
+            f"Got min={finite.min():.4f}, max={finite.max():.4f}. "
+            "Clip first with clip_beta()."
+        )
+    eps = 1e-6
+    clipped = np.clip(data, eps, 1 - eps)
+    result = np.log2(clipped / (1 - clipped))
+    logger.info("beta_to_m: converted beta → M-values for %s.", df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)
+
+
+def m_to_beta(df: pd.DataFrame) -> pd.DataFrame:
+    """Convert M-values back to beta values: 2^M / (2^M + 1).
+
+    Parameters
+    ----------
+    df:
+        M-value matrix (samples × CpG sites).
+
+    Returns
+    -------
+    pandas.DataFrame
+        Beta value matrix in (0, 1).
+    """
+    data = df.values.astype(float)
+    exp = np.power(2.0, data)
+    result = exp / (exp + 1.0)
+    logger.info("m_to_beta: converted M-values → beta for %s.", df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)
+
+
+def clip_beta(
+    df: pd.DataFrame,
+    low: float = 0.001,
+    high: float = 0.999,
+) -> pd.DataFrame:
+    """Clip beta values to avoid extreme values near 0 and 1.
+
+    Parameters
+    ----------
+    df:
+        Beta value matrix.
+    low:
+        Lower clip bound (default 0.001).
+    high:
+        Upper clip bound (default 0.999).
+
+    Returns
+    -------
+    pandas.DataFrame
+        Clipped beta matrix.
+    """
+    result = df.values.astype(float).clip(low, high)
+    logger.info("clip_beta: clipped to [%.4f, %.4f] for %s.", low, high, df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)
+
+
+def detect_and_normalise(df: pd.DataFrame) -> pd.DataFrame:
+    """Auto-detect M-values vs beta values and normalise to clipped beta.
+
+    Heuristic: if any finite value is outside [0, 1], treat as M-values
+    and convert to beta.  Otherwise clip beta to [0.001, 0.999].
+
+    Parameters
+    ----------
+    df:
+        Methylation matrix (samples × CpG sites).
+
+    Returns
+    -------
+    pandas.DataFrame
+        Beta values clipped to [0.001, 0.999].
+    """
+    data = df.values.astype(float)
+    finite = data[np.isfinite(data)]
+    if len(finite) == 0:
+        logger.warning("detect_and_normalise: no finite values; skipping.")
+        return df
+
+    vmin, vmax = finite.min(), finite.max()
+
+    if vmin < -0.01 or vmax > 1.01:
+        logger.info(
+            "detect_and_normalise (methylation): detected M-values "
+            "(min=%.4f, max=%.4f); converting to beta.",
+            vmin, vmax,
+        )
+        result = m_to_beta(df)
+    else:
+        logger.info(
+            "detect_and_normalise (methylation): detected beta values "
+            "(min=%.4f, max=%.4f); clipping.",
+            vmin, vmax,
+        )
+        result = df
+
+    return clip_beta(result)

omicsync/normalisation/mutations.py

@@ -0,0 +1,123 @@
+"""Mutation matrix processing utilities."""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+import numpy as np
+import pandas as pd
+
+from omicsync.utils.logging import get_logger
+
+logger = get_logger("normalisation.mutations")
+
+# Standard Sequence Ontology consequence terms
+CONSEQUENCE_TERMS = frozenset({
+    "missense_variant",
+    "stop_gained",
+    "stop_lost",
+    "frameshift_variant",
+    "splice_acceptor_variant",
+    "splice_donor_variant",
+    "start_lost",
+    "inframe_insertion",
+    "inframe_deletion",
+    "synonymous_variant",
+    "3_prime_UTR_variant",
+    "5_prime_UTR_variant",
+    "intron_variant",
+    "upstream_gene_variant",
+    "downstream_gene_variant",
+    "non_coding_transcript_variant",
+})
+
+
+def binarise(df: pd.DataFrame, threshold: float = 0) -> pd.DataFrame:
+    """Binarise a mutation matrix: any value above *threshold* becomes 1.
+
+    Parameters
+    ----------
+    df:
+        Mutation matrix (samples × genes). May contain counts or continuous
+        variant scores.
+    threshold:
+        Values strictly above this threshold are set to 1; others to 0.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Binary mutation matrix with dtype float64.
+    """
+    result = (df.values.astype(float) > threshold).astype(float)
+    logger.info(
+        "binarise: threshold=%.2f applied to %s; "
+        "%.1f%% mutated entries.",
+        threshold,
+        df.shape,
+        100.0 * result.mean(),
+    )
+    return pd.DataFrame(result, index=df.index, columns=df.columns)
+
+
+def filter_by_consequence(
+    df: pd.DataFrame,
+    consequences: Sequence[str],
+    consequence_map: dict | None = None,
+) -> pd.DataFrame:
+    """Keep only genes that have at least one sample with a specified consequence.
+
+    This function operates on a pre-binarised mutation matrix.  If a
+    ``consequence_map`` is provided (mapping gene → consequence), genes whose
+    mapped consequence is not in *consequences* are zeroed out.
+
+    Parameters
+    ----------
+    df:
+        Mutation matrix (samples × genes).
+    consequences:
+        Consequence types to retain, e.g. ``["missense_variant", "stop_gained"]``.
+    consequence_map:
+        Optional dict mapping gene ID to its predominant consequence.  If
+        ``None``, this function simply returns *df* unchanged with a warning.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Filtered mutation matrix.
+    """
+    if consequence_map is None:
+        logger.warning(
+            "filter_by_consequence: no consequence_map provided; returning input unchanged."
+        )
+        return df
+
+    keep = [gene for gene in df.columns if consequence_map.get(gene) in consequences]
+    n_before = df.shape[1]
+    result = df[keep].copy()
+    logger.info(
+        "filter_by_consequence: kept %d/%d genes matching %s.",
+        len(keep),
+        n_before,
+        list(consequences),
+    )
+    return result
+
+
+def compute_tmb(df: pd.DataFrame) -> pd.Series:
+    """Compute tumour mutation burden (total mutations per sample).
+
+    Parameters
+    ----------
+    df:
+        Binary mutation matrix (samples × genes).
+
+    Returns
+    -------
+    pandas.Series
+        Mutation count per sample, indexed by sample ID.
+    """
+    tmb = df.sum(axis=1).rename("tmb")
+    logger.info(
+        "compute_tmb: TMB computed for %d samples; mean=%.2f.", len(tmb), tmb.mean()
+    )
+    return tmb

omicsync/normalisation/protein.py

@@ -0,0 +1,54 @@
+"""Protein abundance normalisation utilities."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from omicsync.utils.logging import get_logger
+
+logger = get_logger("normalisation.protein")
+
+
+def z_score(df: pd.DataFrame) -> pd.DataFrame:
+    """Z-score normalise protein abundance per feature (column).
+
+    Constant columns (zero standard deviation) are set to 0.
+
+    Parameters
+    ----------
+    df:
+        Protein abundance matrix (samples × proteins).
+
+    Returns
+    -------
+    pandas.DataFrame
+        Z-scored matrix.
+    """
+    data = df.values.astype(float)
+    mu = np.nanmean(data, axis=0, keepdims=True)
+    sd = np.nanstd(data, axis=0, keepdims=True)
+    sd = np.where(sd == 0, 1.0, sd)
+    result = (data - mu) / sd
+    logger.info("z_score (protein): applied to %s.", df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)
+
+
+def median_centring(df: pd.DataFrame) -> pd.DataFrame:
+    """Centre each protein on its median across samples.
+
+    Parameters
+    ----------
+    df:
+        Protein abundance matrix (samples × proteins).
+
+    Returns
+    -------
+    pandas.DataFrame
+        Median-centred matrix.
+    """
+    data = df.values.astype(float)
+    medians = np.nanmedian(data, axis=0, keepdims=True)
+    result = data - medians
+    logger.info("median_centring: applied to %s.", df.shape)
+    return pd.DataFrame(result, index=df.index, columns=df.columns)