cytocommunity2 0.1.1__py3-none-any.whl

cytocommunity2/downstream/communication/cca.py

@@ -0,0 +1,332 @@
+"""Canonical correlation analysis between cellular neighborhoods."""
+
+from itertools import combinations
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.cross_decomposition import CCA
+
+from ...config import CytoCommunityConfig
+from ...paths import build_paths
+from .utils import condition_sort_key
+
+
+def pair_rng(base_seed, condition, cn_a, cn_b):
+    """Derive a reproducible independent random stream for a CN pair."""
+    seed = int(base_seed) % (2**32)
+    for character in f"{condition}|{cn_a}|{cn_b}":
+        seed = (seed * 31 + ord(character)) % (2**32)
+    return np.random.default_rng(seed)
+
+
+def clean_cca_data(x, y, minimum_common_samples, sd_epsilon):
+    """Align two matrices and remove incomplete or near-constant variables."""
+    if x is None or y is None or x.shape[1] == 0 or y.shape[1] == 0:
+        return None
+    common = x.index.intersection(y.index)
+    if len(common) < minimum_common_samples:
+        return None
+    x = x.loc[common]
+    y = y.loc[common]
+    x = x.loc[:, x.notna().sum(axis=0) > 0]
+    y = y.loc[:, y.notna().sum(axis=0) > 0]
+    x = x.loc[:, x.std(axis=0) > sd_epsilon]
+    y = y.loc[:, y.std(axis=0) > sd_epsilon]
+    if x.shape[1] == 0 or y.shape[1] == 0:
+        return None
+    complete = x.notna().all(axis=1) & y.notna().all(axis=1)
+    x, y = x.loc[complete], y.loc[complete]
+    if x.shape[0] < minimum_common_samples or x.shape[1] == 0 or y.shape[1] == 0:
+        return None
+    return {
+        "X": x.values,
+        "Y": y.values,
+        "X_ct": x.columns.tolist(),
+        "Y_ct": y.columns.tolist(),
+    }
+
+
+def build_score_list(frame, cn_list):
+    """Build one sample-by-cell-type score matrix per CN."""
+    matrices = {}
+    for cn in cn_list:
+        subset = frame[frame["CN"].astype(str) == str(cn)]
+        if subset.empty:
+            continue
+        matrix = subset.pivot_table(
+            index="Sample", columns="CellType", values="Score", aggfunc="mean"
+        )
+        if not matrix.empty:
+            matrices[str(cn)] = matrix.astype(float)
+    return matrices
+
+
+def run_cca_for_scores(
+    score_list, cn_list, communication_params, random_seed, condition=None
+):
+    """Run CCA and permutation tests on in-memory enrichment matrices."""
+    result_rows = []
+    cache = {}
+    for cn_a, cn_b in combinations(map(str, cn_list), 2):
+        if cn_a not in score_list or cn_b not in score_list:
+            continue
+        cleaned = clean_cca_data(
+            score_list[cn_a],
+            score_list[cn_b],
+            communication_params["cca_minimum_common_samples"],
+            communication_params["cca_sd_epsilon"],
+        )
+        if cleaned is None:
+            continue
+        x_matrix, y_matrix = cleaned["X"], cleaned["Y"]
+        n_samples = x_matrix.shape[0]
+        max_components = min(x_matrix.shape[1], y_matrix.shape[1], n_samples - 2)
+        if max_components < 1:
+            continue
+
+        model = CCA(
+            n_components=max_components,
+            scale=True,
+            max_iter=communication_params["cca_max_iterations"],
+        )
+        model.fit(x_matrix, y_matrix)
+        x_scores, y_scores = model.transform(x_matrix, y_matrix)
+        actual_components = min(
+            communication_params["cca_canonical_components"], x_scores.shape[1]
+        )
+        correlations = [
+            np.corrcoef(x_scores[:, index], y_scores[:, index])[0, 1]
+            for index in range(actual_components)
+        ]
+
+        p_values = []
+        invalid_counts = []
+        rng = pair_rng(random_seed, condition, cn_a, cn_b)
+        for component, observed in enumerate(correlations):
+            extreme = 0
+            valid_count = 0
+            invalid_count = 0
+            for _ in range(communication_params["cca_n_bootstrap"]):
+                shuffled_y = rng.permutation(y_matrix)
+                permutation_model = CCA(
+                    n_components=min(
+                        communication_params["cca_canonical_components"],
+                        max_components,
+                    ),
+                    scale=True,
+                    max_iter=communication_params["cca_max_iterations"],
+                )
+                try:
+                    permutation_model.fit(x_matrix, shuffled_y)
+                    permuted_x, permuted_y = permutation_model.transform(
+                        x_matrix, shuffled_y
+                    )
+                except (ValueError, np.linalg.LinAlgError, FloatingPointError):
+                    invalid_count += 1
+                    continue
+                if permuted_x.shape[1] <= component:
+                    continue
+                permuted_rho = abs(
+                    np.corrcoef(
+                        permuted_x[:, component], permuted_y[:, component]
+                    )[0, 1]
+                )
+                valid_count += 1
+                if permuted_rho >= abs(observed) - 1e-12:
+                    extreme += 1
+            p_values.append(
+                (extreme + 1) / (valid_count + 1) if valid_count else np.nan
+            )
+            invalid_counts.append(invalid_count)
+
+        row = {"CN_A": cn_a, "CN_B": cn_b, "n_samples": n_samples}
+        if condition is not None:
+            row["Condition"] = str(condition)
+        for component in range(communication_params["cca_canonical_components"]):
+            row[f"rho{component + 1}"] = (
+                correlations[component] if component < len(correlations) else None
+            )
+            row[f"pval{component + 1}"] = (
+                p_values[component] if component < len(p_values) else None
+            )
+            row[f"invalid_perm{component + 1}"] = (
+                invalid_counts[component] if component < len(invalid_counts) else None
+            )
+        result_rows.append(row)
+        key = "__".join(
+            filter(
+                None,
+                [str(condition) if condition is not None else "", cn_a, cn_b],
+            )
+        )
+        cache[key] = {
+            "X_scores": x_scores,
+            "Y_scores": y_scores,
+            "X_ct": cleaned["X_ct"],
+            "Y_ct": cleaned["Y_ct"],
+            "X_mat": x_matrix,
+            "Y_mat": y_matrix,
+        }
+
+    columns = ["CN_A", "CN_B", "n_samples"]
+    if condition is not None:
+        columns.append("Condition")
+    for component in range(communication_params["cca_canonical_components"]):
+        columns.extend(
+            [f"rho{component + 1}", f"pval{component + 1}", f"invalid_perm{component + 1}"]
+        )
+    results = pd.DataFrame(result_rows, columns=columns)
+    if not results.empty:
+        results = results.sort_values(
+            "rho1", key=lambda values: values.abs(), ascending=False, na_position="last"
+        )
+    return {"results": results, "cache": cache}
+
+
+def cca_coordinate_tables(
+    results, cache, celltype_order, canonical_components, condition=None
+):
+    """Convert fitted CCA scores into per-pair cell-type loading tables."""
+    tables = {}
+    if results is None or results.empty:
+        return tables
+    for _, row in results.iterrows():
+        cn_a, cn_b = str(row["CN_A"]), str(row["CN_B"])
+        key = "__".join(
+            filter(None, [str(condition) if condition is not None else "", cn_a, cn_b])
+        )
+        if key not in cache:
+            continue
+        entry = cache[key]
+        all_celltypes = [
+            cell_type
+            for cell_type in celltype_order
+            if cell_type in set(entry["X_ct"]) | set(entry["Y_ct"])
+        ]
+        data = {"CellType": all_celltypes}
+        maximum = min(canonical_components, entry["X_scores"].shape[1])
+        for component in range(maximum):
+            x_score = entry["X_scores"][:, component]
+            y_score = entry["Y_scores"][:, component]
+            x_loadings = {
+                cell_type: np.corrcoef(entry["X_mat"][:, index], x_score)[0, 1]
+                if len(x_score) > 1
+                else np.nan
+                for index, cell_type in enumerate(entry["X_ct"])
+            }
+            y_loadings = {
+                cell_type: np.corrcoef(entry["Y_mat"][:, index], y_score)[0, 1]
+                if len(y_score) > 1
+                else np.nan
+                for index, cell_type in enumerate(entry["Y_ct"])
+            }
+            data[f"CN_A_coord_can{component + 1}"] = [
+                x_loadings.get(cell_type, np.nan) for cell_type in all_celltypes
+            ]
+            data[f"CN_B_coord_can{component + 1}"] = [
+                y_loadings.get(cell_type, np.nan) for cell_type in all_celltypes
+            ]
+            data[f"rho{component + 1}"] = [row.get(f"rho{component + 1}", np.nan)] * len(all_celltypes)
+            data[f"pval{component + 1}"] = [row.get(f"pval{component + 1}", np.nan)] * len(all_celltypes)
+        tables[(cn_a, cn_b)] = pd.DataFrame(data)
+    return tables
+
+
+def write_cca_coordinates(
+    results, cache, output_dir, celltype_order, canonical_components, condition=None
+):
+    """Write coordinate tables using the original filenames."""
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    tables = cca_coordinate_tables(
+        results, cache, celltype_order, canonical_components, condition
+    )
+    prefix = f"{condition}_" if condition is not None else ""
+    for (cn_a, cn_b), table in tables.items():
+        table.to_csv(
+            output_dir / f"CCA_coordinates_{prefix}CN{cn_a}_vs_CN{cn_b}.csv",
+            index=False,
+        )
+    return tables
+
+
+def _normalize_scores(frame):
+    frame = frame.copy()
+    for column in ["Sample", "Condition", "CN", "CellType"]:
+        frame[column] = frame[column].astype(str)
+    frame["Score"] = pd.to_numeric(frame["Score"], errors="coerce")
+    return frame
+
+
+def run_cca_analysis(config, *, input_file=None, output_dir=None):
+    """Run pooled and condition-specific CCA using configured paths."""
+    if not isinstance(config, CytoCommunityConfig):
+        config = CytoCommunityConfig.from_json(config)
+    paths = build_paths(config)
+    params = config.downstream["communication"]
+    output_dir = Path(output_dir or paths.communication_config)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    frame = _normalize_scores(
+        pd.read_csv(input_file or output_dir / "EnrichScoreMatrix_long.csv")
+    )
+    conditions = sorted(frame["Condition"].unique(), key=condition_sort_key)
+    cn_list = sorted(frame["CN"].unique(), key=condition_sort_key)
+    pooled = run_cca_for_scores(
+        build_score_list(frame, cn_list), cn_list, params, config.random_seed
+    )
+    pooled["results"].to_csv(output_dir / "CCA_config.csv", index=False)
+    write_cca_coordinates(
+        pooled["results"],
+        pooled["cache"],
+        output_dir,
+        config.celltype_order,
+        params["cca_canonical_components"],
+    )
+    condition_results = []
+    for condition in conditions:
+        subset = frame[frame["Condition"] == condition]
+        result = run_cca_for_scores(
+            build_score_list(subset, cn_list),
+            cn_list,
+            params,
+            config.random_seed,
+            condition,
+        )
+        result["results"].to_csv(output_dir / f"CCA_config_{condition}.csv", index=False)
+        write_cca_coordinates(
+            result["results"],
+            result["cache"],
+            output_dir,
+            config.celltype_order,
+            params["cca_canonical_components"],
+            condition,
+        )
+        condition_results.append(result["results"])
+    by_condition = (
+        pd.concat(condition_results, ignore_index=True)
+        if condition_results
+        else pd.DataFrame(columns=["Condition", *pooled["results"].columns])
+    )
+    if not by_condition.empty:
+        by_condition["Condition"] = pd.Categorical(
+            by_condition["Condition"], categories=conditions, ordered=True
+        )
+        by_condition = by_condition.sort_values(
+            ["Condition", "rho1"],
+            key=lambda values: abs(values) if values.name == "rho1" else values,
+            ascending=[True, False],
+        )
+    by_condition.to_csv(output_dir / "CCA_config_by_condition.csv", index=False)
+    return {"pooled": pooled["results"], "by_condition": by_condition}
+
+
+__all__ = [
+    "build_score_list",
+    "cca_coordinate_tables",
+    "clean_cca_data",
+    "pair_rng",
+    "run_cca_analysis",
+    "run_cca_for_scores",
+    "write_cca_coordinates",
+]

cytocommunity2/downstream/communication/utils.py

@@ -0,0 +1,63 @@
+"""Shared, side-effect-free communication-analysis utilities."""
+
+import itertools
+
+import numpy as np
+import pandas as pd
+from scipy.stats import spearmanr
+
+
+def condition_label(condition):
+    condition = str(condition)
+    return condition if condition == "rest" else f"label{condition}"
+
+
+def condition_pair_label(condition_a, condition_b):
+    return f"{condition_label(condition_a)}_vs_{condition_label(condition_b)}"
+
+
+def condition_sort_key(value):
+    value = str(value)
+    try:
+        return (0, int(value), value)
+    except ValueError:
+        return (1, value)
+
+
+def available_conditions(values):
+    present = pd.Series(values).dropna().astype(str).str.strip()
+    present = present[present.ne("")].unique().tolist()
+    return sorted(present, key=condition_sort_key)
+
+
+def condition_pairs(values):
+    return list(itertools.combinations(available_conditions(values), 2))
+
+
+def stable_seed(base_seed, *parts):
+    return int(base_seed) + sum(ord(character) for character in "_".join(map(str, parts)))
+
+
+def safe_spearman(x, y, min_samples):
+    x = np.asarray(x, dtype=float)
+    y = np.asarray(y, dtype=float)
+    valid = (~np.isnan(x)) & (~np.isnan(y))
+    x = x[valid]
+    y = y[valid]
+    if len(x) < min_samples or len(np.unique(x)) < 2 or len(np.unique(y)) < 2:
+        return np.nan, np.nan
+    rho, p_value = spearmanr(x, y)
+    if np.isnan(rho):
+        return np.nan, np.nan
+    return float(rho), float(p_value) if np.isfinite(p_value) else np.nan
+
+
+__all__ = [
+    "available_conditions",
+    "condition_label",
+    "condition_pair_label",
+    "condition_pairs",
+    "condition_sort_key",
+    "safe_spearman",
+    "stable_seed",
+]

cytocommunity2/downstream/communication/within.py

@@ -0,0 +1,266 @@
+"""Within-CN cell-type communication statistics."""
+
+import itertools
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from scipy.stats import spearmanr
+
+from ...config import CytoCommunityConfig
+from ...paths import build_paths
+from .utils import (
+    available_conditions,
+    condition_pair_label,
+    safe_spearman,
+    stable_seed,
+)
+
+
+def compute_pooled_within_cn(scores_long, num_cn, min_samples):
+    """Compute pooled within-CN Spearman correlations."""
+    rows = []
+    for cn in range(1, num_cn + 1):
+        subset = scores_long[scores_long["CN"].astype(str) == str(cn)]
+        matrix = subset.pivot(index="Sample", columns="CellType", values="Score")
+        valid_columns = [
+            cell_type
+            for cell_type in matrix.columns
+            if matrix[cell_type].notna().sum() >= min_samples
+            and matrix[cell_type].dropna().std() > 0
+        ]
+        matrix = matrix[valid_columns]
+        if matrix.shape[1] < 2:
+            continue
+        correlation, p_value = spearmanr(matrix.values, axis=0, nan_policy="omit")
+        if np.isscalar(correlation):
+            correlation = np.array([[1.0, correlation], [correlation, 1.0]])
+            p_value = np.array([[0.0, p_value], [p_value, 0.0]])
+        for left, right in itertools.combinations(range(matrix.shape[1]), 2):
+            cell_type_a = matrix.columns[left]
+            cell_type_b = matrix.columns[right]
+            rows.append(
+                {
+                    "CN": str(cn),
+                    "CellType1": cell_type_a,
+                    "CellType2": cell_type_b,
+                    "rho": correlation[left, right],
+                    "p": p_value[left, right],
+                    "n": len(matrix[[cell_type_a, cell_type_b]].dropna()),
+                }
+            )
+    return pd.DataFrame(rows)
+
+
+def prepare_wide_tables(scores_long, cn_order):
+    tables = {}
+    for cn in cn_order:
+        subset = scores_long[scores_long["CN"].astype(str) == str(cn)]
+        if subset.empty:
+            continue
+        tables[str(cn)] = (
+            subset.pivot_table(
+                index=["Sample", "Condition"],
+                columns="CellType",
+                values="Score",
+                aggfunc="mean",
+            ).reset_index()
+        )
+    return tables
+
+
+def permutation_pvalue(
+    x,
+    y,
+    labels,
+    condition_a,
+    condition_b,
+    observed_delta,
+    n_permutations,
+    min_samples,
+    max_attempt_multiplier,
+    seed,
+):
+    """Two-sided label permutation test for a difference in Spearman rho."""
+    x = np.asarray(x, dtype=float)
+    y = np.asarray(y, dtype=float)
+    labels = np.asarray(labels)
+    valid = (~np.isnan(x)) & (~np.isnan(y)) & np.isin(
+        labels, [condition_a, condition_b]
+    )
+    x, y, labels = x[valid], y[valid], labels[valid]
+    indices_a = np.where(labels == condition_a)[0]
+    indices_b = np.where(labels == condition_b)[0]
+    if len(indices_a) < min_samples or len(indices_b) < min_samples:
+        return np.nan, 0
+
+    rng = np.random.default_rng(seed)
+    all_indices = np.arange(len(labels))
+    extreme = 0
+    valid_count = 0
+    for _ in range(n_permutations * max_attempt_multiplier):
+        permutation = rng.permutation(all_indices)
+        permuted_a = permutation[: len(indices_a)]
+        permuted_b = permutation[len(indices_a) :]
+        rho_a, _ = safe_spearman(x[permuted_a], y[permuted_a], min_samples)
+        rho_b, _ = safe_spearman(x[permuted_b], y[permuted_b], min_samples)
+        if np.isnan(rho_a) or np.isnan(rho_b):
+            continue
+        valid_count += 1
+        if abs(rho_b - rho_a) >= abs(observed_delta) - 1e-12:
+            extreme += 1
+        if valid_count >= n_permutations:
+            break
+    if valid_count == 0:
+        return np.nan, 0
+    return (extreme + 1.0) / (valid_count + 1.0), valid_count
+
+
+def build_condition_comparisons(values, comparison_mode):
+    labels = available_conditions(values)
+    if len(labels) < 2:
+        return []
+    if comparison_mode == "pairwise":
+        return [
+            {
+                "cond_a": condition_a,
+                "cond_b": condition_b,
+                "labels_a": [condition_a],
+                "labels_b": [condition_b],
+            }
+            for condition_a, condition_b in itertools.combinations(labels, 2)
+        ]
+    if comparison_mode == "one_vs_rest":
+        return [
+            {
+                "cond_a": target,
+                "cond_b": "rest",
+                "labels_a": [target],
+                "labels_b": [label for label in labels if label != target],
+            }
+            for target in labels
+        ]
+    raise ValueError(f"Unknown comparison mode: {comparison_mode}")
+
+
+def compute_condition_comparison_stats(
+    wide_dict,
+    cond_a,
+    cond_b,
+    labels_a,
+    labels_b,
+    cn_order,
+    celltype_order,
+    min_samples,
+    n_permutations,
+    max_attempt_multiplier,
+    base_seed,
+):
+    """Compute one pairwise or one-vs-rest within-CN comparison."""
+    rows = []
+    for cn in cn_order:
+        wide = wide_dict.get(str(cn))
+        if wide is None:
+            continue
+        wide = wide[wide["Condition"].astype(str).isin(labels_a + labels_b)].copy()
+        if wide.empty:
+            continue
+        wide["ComparisonGroup"] = np.where(
+            wide["Condition"].astype(str).isin(labels_a), cond_a, cond_b
+        )
+        celltypes = [cell_type for cell_type in celltype_order if cell_type in wide]
+        for cell_type_a, cell_type_b in itertools.combinations(celltypes, 2):
+            paired = wide[["Sample", "ComparisonGroup", cell_type_a, cell_type_b]].dropna()
+            group_a = paired[paired["ComparisonGroup"] == cond_a]
+            group_b = paired[paired["ComparisonGroup"] == cond_b]
+            if len(group_a) < min_samples or len(group_b) < min_samples:
+                continue
+            rho_a, p_a = safe_spearman(
+                group_a[cell_type_a], group_a[cell_type_b], min_samples
+            )
+            rho_b, p_b = safe_spearman(
+                group_b[cell_type_a], group_b[cell_type_b], min_samples
+            )
+            if np.isnan(rho_a) or np.isnan(rho_b):
+                continue
+            delta = rho_b - rho_a
+            permutation_p, _ = permutation_pvalue(
+                paired[cell_type_a],
+                paired[cell_type_b],
+                paired["ComparisonGroup"],
+                cond_a,
+                cond_b,
+                delta,
+                n_permutations,
+                min_samples,
+                max_attempt_multiplier,
+                stable_seed(base_seed, cond_a, cond_b, cn, cell_type_a, cell_type_b),
+            )
+            rows.append(
+                {
+                    "CN": str(cn),
+                    "CellType1": cell_type_a,
+                    "CellType2": cell_type_b,
+                    "rho_A": rho_a,
+                    "p_A": p_a,
+                    "rho_B": rho_b,
+                    "p_B": p_b,
+                    "perm_p": permutation_p,
+                }
+            )
+    if not rows:
+        return pd.DataFrame()
+    return pd.DataFrame(rows).sort_values(
+        ["CN", "CellType1", "CellType2"]
+    ).reset_index(drop=True)
+
+
+def run_within_cn_analysis(config, *, input_file=None, output_dir=None):
+    """Run and save pooled plus condition-specific within-CN analyses."""
+    if not isinstance(config, CytoCommunityConfig):
+        config = CytoCommunityConfig.from_json(config)
+    paths = build_paths(config)
+    params = config.downstream["communication"]
+    output_dir = Path(output_dir or paths.communication_config)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    input_file = Path(input_file or output_dir / "EnrichScoreMatrix_long.csv")
+    scores = pd.read_csv(input_file, dtype={"Condition": str, "CN": str})
+    num_cn = int(paths.selected_cn_num.read_text(encoding="utf-8"))
+    cn_order = [str(index) for index in range(1, num_cn + 1)]
+    wide = prepare_wide_tables(scores, cn_order)
+    outputs = {
+        "pooled": compute_pooled_within_cn(scores, num_cn, params["minimum_samples"])
+    }
+    outputs["pooled"].to_csv(output_dir / "SpearmanWithinCNs.csv", index=False)
+    for comparison in build_condition_comparisons(
+        scores["Condition"], config.downstream["ttest_mode"]
+    ):
+        result = compute_condition_comparison_stats(
+            wide_dict=wide,
+            cond_a=comparison["cond_a"],
+            cond_b=comparison["cond_b"],
+            labels_a=comparison["labels_a"],
+            labels_b=comparison["labels_b"],
+            cn_order=cn_order,
+            celltype_order=config.celltype_order,
+            min_samples=params["minimum_samples"],
+            n_permutations=params["n_permutations"],
+            max_attempt_multiplier=params["permutation_max_attempt_multiplier"],
+            base_seed=config.random_seed,
+        )
+        if result.empty:
+            continue
+        label = condition_pair_label(comparison["cond_a"], comparison["cond_b"])
+        result.to_csv(output_dir / f"CommunicationPair_{label}.csv", index=False)
+        outputs[label] = result
+    return outputs
+
+
+__all__ = [
+    "build_condition_comparisons",
+    "compute_condition_comparison_stats",
+    "compute_pooled_within_cn",
+    "permutation_pvalue",
+    "prepare_wide_tables",
+    "run_within_cn_analysis",
+]