autoencodix 0.2.0__py3-none-any.whl

autoencodix/utils/adata_converter.py

@@ -0,0 +1,111 @@
+from typing import Literal, Dict, Optional
+from autoencodix.data._multimodal_dataset import MultiModalDataset
+from autoencodix.data._numeric_dataset import NumericDataset
+from autoencodix.data._datasetcontainer import DatasetContainer
+import anndata as ad
+import pandas as pd
+import torch
+from scipy.sparse import issparse
+
+
+class AnnDataConverter:
+    """Utility class for converting datasets into AnnData or multimodal AnnData dictionaries."""
+
+    @staticmethod
+    def _numeric_ds_to_adata(ds: NumericDataset) -> Dict[str, ad.AnnData]:
+        """Convert a NumericDataset to an AnnData object.
+
+        Args:
+            ds: The numeric dataset to convert.
+
+        Returns:
+            An AnnData object containing the dataset's data, features, and metadata.
+        """
+
+        metadata = ds.metadata.copy()
+        if isinstance(metadata, dict):
+            if len(metadata) != 1:
+                raise NotImplementedError(
+                    "Unpaired metadata conversion not implemented yet. Expected single modality."
+                )
+            first_key = next(iter(metadata.keys()))
+            metadata = metadata[first_key]
+        if not isinstance(metadata, pd.DataFrame):
+            raise ValueError(f"metadata needs to be pd.DataFrame, got {type(metadata)}")
+        metadata.index = metadata.index.astype(str)
+
+        var = pd.DataFrame(index=pd.Index(ds.feature_ids, dtype=str))
+        # check if ds.data issparse
+        if issparse(ds.data):
+            x = torch.tensor(ds.data.toarray())
+        else:
+            x = ds.data
+        return {
+            "global": ad.AnnData(
+                X=x.clone().detach().cpu().numpy(),
+                var=var,
+                obs=metadata,
+            )
+        }
+
+    @staticmethod
+    def _parse_multimodal(mds: MultiModalDataset) -> Dict[str, ad.AnnData]:
+        """Convert a MultiModalDataset into a dictionary of AnnData objects.
+
+        Args:
+            mds: The multimodal dataset to convert.
+
+        Returns:
+            A dictionary mapping modality names to AnnData objects.
+
+        Raises:
+            NotImplementedError: If any modality is not a NumericDataset.
+        """
+        result_dict: Dict[str, ad.AnnData] = {}
+        for mod_name, dataset in mds.datasets.items():
+            if not isinstance(dataset, NumericDataset):
+                raise NotImplementedError(
+                    f"Feature Importance is only implemented for NumericDataset, got type: {type(dataset)}"
+                )
+            result_dict[mod_name] = AnnDataConverter._numeric_ds_to_adata(dataset)  # type: ignore
+        return result_dict
+
+    @staticmethod
+    def dataset_to_adata(
+        datasetcontainer: DatasetContainer,
+        split: Literal["train", "valid", "test"] = "train",
+    ) -> Optional[Dict[str, ad.AnnData]]:
+        """Convert a DatasetContainer split to an AnnData or multimodal AnnData dictionary.
+
+        Args:
+            datasetcontainer: Container holding train/valid/test datasets.
+            split: The dataset split to convert. Defaults to "train".
+
+        Returns:
+            A single AnnData object (for NumericDataset) or a dictionary of AnnData objects (for MultiModalDataset).
+
+        Raises:
+            ValueError: If the specified split does not exist in the DatasetContainer.
+            NotImplementedError: If the dataset type is not supported.
+        """
+        if not hasattr(datasetcontainer, split):
+            raise ValueError(
+                f"Split: {split} not present in DatasetContainer: {datasetcontainer}"
+            )
+
+        ds = datasetcontainer[split]
+
+        if isinstance(ds, MultiModalDataset):
+            return AnnDataConverter._parse_multimodal(ds)
+        elif isinstance(ds, NumericDataset):
+            return AnnDataConverter._numeric_ds_to_adata(ds)
+        elif ds is None:
+            import warnings
+
+            warnings.warn(f"No dataset found for split: {split}, returning None")
+            return None
+
+        else:
+            raise NotImplementedError(
+                f"Conversion not implemented for type: {type(ds)}"
+            )

autoencodix/utils/example_data.py

@@ -0,0 +1,398 @@
+import numpy as np
+import pandas as pd
+from typing import Tuple
+import torch
+from sklearn.datasets import make_blobs  # type: ignore
+from sklearn.model_selection import train_test_split  # type: ignore
+from autoencodix.data._datasetcontainer import DatasetContainer
+from autoencodix.data._numeric_dataset import NumericDataset
+from autoencodix.configs.default_config import DefaultConfig
+from autoencodix.data.datapackage import DataPackage
+from autoencodix.configs.default_config import DataCase
+import mudata
+import anndata
+
+config = DefaultConfig()
+
+
+def generate_example_data(
+    n_samples=1000, n_features=30, n_clusters=5, random_seed=config.global_seed
+):
+    """
+    Generate synthetic data for autoencoder testing and examples.
+
+    Args:
+        n_samples: Number of samples to generate, default=1000
+        n_features: Number of features for each sample, default=30
+        n_clusters: Number of clusters to generate, default=5
+        random_seed:: Random seed for reproducibility
+
+    Returns:
+        DatasetContainer: Container with train, validation and test datasets
+    """
+    np.random.seed(random_seed)
+    torch.manual_seed(random_seed)
+
+    X, cluster_labels = make_blobs(
+        n_samples=n_samples,
+        n_features=n_features,
+        centers=n_clusters,
+        cluster_std=1.8,
+        random_state=random_seed,
+    )
+
+    # Convert integer cluster labels to string labels like "Cluster_1"
+    str_cluster_labels = np.array([f"Cluster_{i + 1}" for i in cluster_labels])
+    # Create additional metadata features that correlate with the clusters
+    # Convert integer cluster labels to string labels like "Cluster_1"
+    str_cluster_labels = np.array([f"Cluster_{i + 1}" for i in cluster_labels])
+
+    metadata_df = pd.DataFrame(
+        {
+            "cluster": str_cluster_labels,
+            "age": np.random.normal(30, 10, n_samples)
+            + cluster_labels * 5,  # Age correlates with cluster
+            "size": np.random.uniform(0, 10, n_samples)
+            + cluster_labels * 2,  # Size correlates with cluster
+            "density": np.random.exponential(1, n_samples)
+            * (cluster_labels + 1)
+            / 3,  # Density correlates with cluster
+            "category": np.random.choice(
+                ["A", "B", "C", "D", "E"], n_samples, p=[0.2, 0.2, 0.2, 0.2, 0.2]
+            ),  # Random category
+        }
+    )
+
+    # Add some noise features to metadata that don't correlate with clusters
+    metadata_df["random_feature"] = np.random.normal(0, 1, n_samples)
+
+    ids = [f"sample_{i}" for i in range(n_samples)]
+    metadata_df["sample_id"] = ids
+    metadata_df.index = ids  # Set sample_id as index
+    data_tensor = torch.tensor(X, dtype=torch.float32)
+
+    # Get split ratios from DefaultConfig
+    config = DefaultConfig()
+    valid_ratio = config.valid_ratio
+    test_ratio = config.test_ratio
+
+    # Calculate test_size for first split (everything except train)
+    first_test_size = valid_ratio + test_ratio
+
+    # Calculate the proportion of valid in the remaining data
+    # If valid_ratio + test_ratio = 0.3 and valid_ratio = 0.1, then valid should be 1/3 of the remaining data
+    valid_proportion = valid_ratio / first_test_size if first_test_size > 0 else 0
+
+    train_idx, temp_idx = train_test_split(
+        np.arange(n_samples),
+        test_size=first_test_size,
+        random_state=random_seed,
+        stratify=cluster_labels,  # Ensure balanced classes in each split
+    )
+    val_idx, test_idx = train_test_split(
+        temp_idx,
+        test_size=(1 - valid_proportion),  # Remaining proportion goes to test
+        random_state=random_seed,
+        stratify=cluster_labels[temp_idx],  # Ensure balanced classes in each split
+    )
+
+    # Create split indicators as numpy arrays
+    train_split = np.zeros(n_samples, dtype=bool)
+    train_split[train_idx] = True
+
+    val_split = np.zeros(n_samples, dtype=bool)
+    val_split[val_idx] = True
+
+    test_split = np.zeros(n_samples, dtype=bool)
+    test_split[test_idx] = True
+
+    train_dataset = NumericDataset(
+        data=data_tensor[train_idx],
+        config=DefaultConfig(),
+        sample_ids=[ids[i] for i in train_idx],
+        metadata=metadata_df.iloc[train_idx],
+        split_indices=train_split,
+        feature_ids=[f"feature_{i}" for i in range(n_features)],
+    )
+
+    val_dataset = NumericDataset(
+        data=data_tensor[val_idx],
+        config=DefaultConfig(),
+        sample_ids=[ids[i] for i in val_idx],
+        metadata=metadata_df.iloc[val_idx],
+        split_indices=val_split,
+        feature_ids=[f"feature_{i}" for i in range(n_features)],
+    )
+
+    test_dataset = NumericDataset(
+        data=data_tensor[test_idx],
+        config=DefaultConfig(),
+        sample_ids=[ids[i] for i in test_idx],
+        metadata=metadata_df.iloc[test_idx],
+        split_indices=test_split,
+        feature_ids=[f"feature_{i}" for i in range(n_features)],
+    )
+
+    return DatasetContainer(train=train_dataset, valid=val_dataset, test=test_dataset)
+
+
+def generate_multi_bulk_example(
+    n_samples=500,
+    n_features_modality1=100,
+    n_features_modality2=80,
+    random_seed=config.global_seed,
+) -> Tuple[DataPackage, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+    """Generate example data for MULTI_BULK case.
+
+    Args:
+        n_samples: Number of samples to generate
+        n_features_modality1: Number of features for first modality
+        n_features_modality2: Number of features for second modality
+        random_seed: Random seed for reproducibility
+
+    Returns:
+        DataPackage with multi_bulk data and raw_data
+    """
+    np.random.seed(random_seed)
+
+    latent_dim = 10
+    latent_representation = np.random.normal(0, 1, (n_samples, latent_dim))
+
+    weights_mod1 = np.random.normal(0, 1, (latent_dim, n_features_modality1))
+    weights_mod2 = np.random.normal(0, 1, (latent_dim, n_features_modality2))
+
+    # Generate observed data with noise
+    data_mod1 = np.dot(latent_representation, weights_mod1) + np.random.normal(
+        0, 0.5, (n_samples, n_features_modality1)
+    )
+    data_mod2 = np.dot(latent_representation, weights_mod2) + np.random.normal(
+        0, 0.5, (n_samples, n_features_modality2)
+    )
+
+    mod1_features = [f"gene_{i}" for i in range(n_features_modality1)]
+    mod2_features = [f"protein_{i}" for i in range(n_features_modality2)]
+
+    sample_ids = [f"sample_{i}" for i in range(n_samples)]
+
+    # Create DataFrames
+    df_mod1 = pd.DataFrame(data_mod1, columns=mod1_features, index=sample_ids)
+    df_mod2 = pd.DataFrame(data_mod2, columns=mod2_features, index=sample_ids)
+
+    # Create annotation DataFrame with sample information
+    # Adding some metadata that correlates with the latent structure
+    group_assignments = np.argmax(latent_representation[:, :3], axis=1)
+    conditions = ["condition_" + str(g) for g in group_assignments]
+    batch_effects = np.random.choice(["batch_1", "batch_2", "batch_3"], size=n_samples)
+
+    annotation_df = pd.DataFrame(
+        {
+            "condition": conditions,
+            "batch": batch_effects,
+            "quality_score": np.random.uniform(0.6, 1.0, n_samples),
+        },
+        index=sample_ids,
+    )
+
+    data_package = DataPackage()
+    data_package.multi_bulk = {"transcriptomics": df_mod1, "proteomics": df_mod2}
+    data_package.annotation = {
+        "transcriptomics": annotation_df.copy(),
+        "proteomics": annotation_df.copy(),
+    }
+
+    return data_package, df_mod1, df_mod2, annotation_df
+
+
+def generate_default_bulk_bulk_example(
+    n_samples=500, n_features=200, random_seed=config.global_seed
+):
+    """NOT IMPLEMENTED YET"""
+    np.random.seed(random_seed)
+    raise NotImplementedError("BULK_TO_BULK example generation not yet implemented")
+
+
+def generate_default_sc_sc_example(
+    n_cells=1000, n_features=500, random_seed=config.global_seed
+):
+    """NOT IMPLEMENTED YET"""
+    np.random.seed(random_seed)
+
+    raise NotImplementedError(
+        "SINGLE_CELL_TO_SINGLE_CELL example generation not yet implemented"
+    )
+
+
+def generate_default_img_bulk_example(n_samples=100, random_seed=config.global_seed):
+    """NOT IMPLEMENTED YET"""
+    np.random.seed(random_seed)
+
+    # Not implemented yet
+    raise NotImplementedError("IMG_TO_BULK example generation not yet implemented")
+
+
+def generate_default_sc_img_example(n_samples=100, random_seed=config.global_seed):
+    """NOT IMPLEMENTED YET"""
+    np.random.seed(random_seed)
+
+    raise NotImplementedError(
+        "SINGLE_CELL_TO_IMG example generation not yet implemented"
+    )
+
+
+def generate_default_img_img_example(n_samples=100, random_seed=config.global_seed):
+    """NOT IMPLEMENTED YET"""
+    np.random.seed(random_seed)
+
+    raise NotImplementedError("IMG_TO_IMG example generation not yet implemented")
+
+
+def generate_raw_datapackage(data_case: DataCase, **kwargs):
+    """Generate a raw DataPackage for the specified data case.
+
+    Args:
+        data_case: The type of data to generate
+        **kwargs: Additional arguments to pass to the specific generator function
+
+    Returns:
+        Raw data package ready for preprocessing
+    """
+    generator_map = {
+        DataCase.MULTI_BULK: generate_multi_bulk_example,
+        DataCase.MULTI_SINGLE_CELL: generate_multi_sc_example,
+        DataCase.BULK_TO_BULK: generate_default_bulk_bulk_example,
+        DataCase.SINGLE_CELL_TO_SINGLE_CELL: generate_default_sc_sc_example,
+        DataCase.IMG_TO_BULK: generate_default_img_bulk_example,
+        DataCase.SINGLE_CELL_TO_IMG: generate_default_sc_img_example,
+        DataCase.IMG_TO_IMG: generate_default_img_img_example,
+    }
+
+    if data_case not in generator_map:
+        raise ValueError(f"Unknown data case: {data_case}")
+
+    generator_function = generator_map[data_case]
+    return generator_function(**kwargs)
+
+
+def generate_multi_sc_example(
+    n_cells=1000, n_genes=500, n_proteins=200, random_seed=config.global_seed
+):
+    """Generate example data for MULTI_SINGLE_CELL case.
+
+    Args:
+        n_cells: Number of cells to generate
+        n_genes: Number of genes for RNA modality
+        n_proteins: Number of proteins for protein modality
+        random_seed: Random seed for reproducibility
+
+    Returns:
+         DataPackage with multi_sc data
+    """
+    np.random.seed(random_seed)
+
+    cell_ids = [f"cell_{i}" for i in range(n_cells)]
+
+    gene_names = [f"gene_{i}" for i in range(n_genes)]
+    protein_names = [f"protein_{i}" for i in range(n_proteins)]
+
+    n_cell_types = 5
+    cell_type_probabilities = np.random.dirichlet(np.ones(n_cell_types), size=1)[0]
+    cell_types = np.random.choice(
+        np.arange(n_cell_types), size=n_cells, p=cell_type_probabilities
+    )
+
+    cell_metadata = pd.DataFrame(
+        {
+            "cell_type": [f"type_{t}" for t in cell_types],
+            "batch": np.random.choice(["batch1", "batch2", "batch3"], size=n_cells),
+            "donor": np.random.choice(
+                ["donor1", "donor2", "donor3", "donor4"], size=n_cells
+            ),
+            "cell_cycle": np.random.choice(["G1", "S", "G2M"], size=n_cells),
+        },
+        index=cell_ids,
+    )
+
+    # Generate RNA data (sparse count matrix with negative binomial distribution)
+    gene_programs = np.zeros((n_cell_types, n_genes))
+    for i in range(n_cell_types):
+        high_expr_genes = np.random.choice(
+            n_genes, size=int(n_genes * 0.2), replace=False
+        )
+        gene_programs[i, high_expr_genes] = np.random.gamma(
+            5, 2, size=len(high_expr_genes)
+        )
+
+    lambda_values = np.zeros((n_cells, n_genes))
+    for i in range(n_cells):
+        lambda_values[i] = gene_programs[cell_types[i]] + np.random.gamma(
+            0.5, 0.5, n_genes
+        )
+
+    seq_depth = np.random.lognormal(mean=8, sigma=0.5, size=n_cells)
+    lambda_values = lambda_values * seq_depth[:, np.newaxis]
+
+    rna_counts = np.random.poisson(lambda_values)
+    dropout_prob = 0.3
+    dropout_mask = np.random.binomial(1, 1 - dropout_prob, size=rna_counts.shape)
+    rna_counts = rna_counts * dropout_mask
+
+    # Generate protein data
+    protein_programs = np.zeros((n_cell_types, n_proteins))
+    for i in range(n_cell_types):
+        high_expr_proteins = np.random.choice(
+            n_proteins, size=int(n_proteins * 0.3), replace=False
+        )
+        protein_programs[i, high_expr_proteins] = np.random.gamma(
+            3, 1, size=len(high_expr_proteins)
+        )
+
+    protein_levels = np.zeros((n_cells, n_proteins))
+    for i in range(n_cells):
+        protein_levels[i] = protein_programs[cell_types[i]] + np.random.gamma(
+            0.2, 0.3, n_proteins
+        )
+
+    batch_effect = np.zeros(n_cells)
+    batch_map = {"batch1": 0.8, "batch2": 1.0, "batch3": 1.2}
+    for i in range(n_cells):
+        batch_effect[i] = batch_map[cell_metadata.iloc[i]["batch"]]
+
+    protein_levels = protein_levels * batch_effect[:, np.newaxis]
+    protein_levels = protein_levels + np.random.normal(
+        0, 0.1, size=protein_levels.shape
+    )
+    protein_levels = np.maximum(0, protein_levels)
+
+    var_rna = pd.DataFrame(index=gene_names)
+    var_protein = pd.DataFrame(index=protein_names)
+
+    from scipy import sparse
+
+    # Convert to scipy sparse matrices to avoid copy issues
+    rna_counts_sparse = sparse.csr_matrix(rna_counts)
+    protein_levels_sparse = sparse.csr_matrix(protein_levels)
+
+    rna_adata = anndata.AnnData(X=rna_counts_sparse, obs=cell_metadata, var=var_rna)
+
+    protein_adata = anndata.AnnData(
+        X=protein_levels_sparse, obs=cell_metadata, var=var_protein
+    )
+
+    mdata = mudata.MuData({"rna": rna_adata, "protein": protein_adata})
+
+    data_package = DataPackage()
+    data_package.multi_sc = {"multi_sc": mdata}
+    # data_package.annotation = {"multi_sc": mdata.obs}
+
+    return data_package, rna_adata, mdata
+
+
+# Pre-generated example data for direct import
+config = DefaultConfig()
+EXAMPLE_PROCESSED_DATA = generate_example_data(random_seed=config.global_seed)
+EXAMPLE_MULTI_BULK, raw_rna, raw_protein, annotation = generate_raw_datapackage(
+    data_case=DataCase.MULTI_BULK
+)
+EXAMPLE_MULTI_SC, sample_adata, sample_mudata = generate_raw_datapackage(
+    data_case=DataCase.MULTI_SINGLE_CELL
+)