flexynesis 0.1.0__py3-none-any.whl

flexynesis/__init__.py

@@ -0,0 +1,142 @@
+"""
+# Flexynesis
+
+Flexynesis is a deep-learning based multi-omics bulk sequencing data
+integration suite with a focus on (pre-)clinical endpoint prediction. The
+package includes multiple types of deep learning architectures such as simple
+fully connected networks, supervised variational autoencoders; different
+options of data layer fusion, and automates feature selection and
+hyperparameter optimisation. The tools are continuosly benchmarked on publicly
+available datasets mostly related to the study of cancer. Some of the
+applications of the methods we develop are drug response modeling in cancer
+patients or preclinical models (such as cell lines and patient-derived
+xenografts), cancer subtype prediction, or any other clinically relevant
+outcome prediction that can be formulated as a regression or classification
+problem.
+
+
+# Package Contents
+- data: Pytorch Dataset classes and functions to import, process multiomics data. 
+- main: High-level functions for training, evaluating, and using models
+- models: 
+    - direct_pred: A multi-task fully connected neural network for direct prediction of one ore more target variables 
+    - direct_pred_cnn: A multi-task one-dimensional convolutional neural network
+    - direct_pred_gcnn: A multi-task graph-convolutional neural network
+    - supervised_vae: A multi-task Supervised Variational Autoencoder model architecture
+    - triplet_encoder: A fully connected neural network implemented with a triplet loss-based contrastive learning 
+- modules: Reusable components for model architecture and training
+- feature_selection: Feature selection methods including Laplacian scoring and redundancy filtering
+- utils: General utility functions for data manipulation and visualization
+- config: Default hyperparameter optimisation spaces  
+
+# Main Features
+
+- Various multi-modal data fusion methods using different kinds of deep learning architectures.
+- Data management tools for loading, preprocessing, and augmenting data.
+- Feature selection methods for effective dimensionality reduction.
+- Utility functions to facilitate data manipulation and visualization.
+- High-level functions for training, evaluating, and using models.
+
+
+# Benchmarks
+
+For the latest benchmark results see:
+https://bimsbstatic.mdc-berlin.de/akalin/buyar/flexynesis-benchmark-datasets/dashboard.html
+
+The code for the benchmarking pipeline is at: https://github.com/BIMSBbioinfo/flexynesis-benchmarks
+
+# Environment
+
+To create a clone of the development environment, use the `spec-file.txt`:
+```
+conda create --name flexynesis --file spec-file.txt
+conda activate flexynesis
+```
+
+To export existing spec-file.txt:
+```
+conda list --explicit > spec-file.txt
+```
+
+# Guix
+
+You can also create a reproducible development environment with [GNU Guix](https://guix.gnu.org).  You will need [this Guix commit](https://git.savannah.gnu.org/cgit/guix.git/commit/?id=e3e011a08141058598cc7631aeb52d620a3ccb8c) or later.
+
+```
+guix shell
+```
+
+or
+
+```
+guix shell -m manifest.scm
+```
+
+You can build a Guix package from the current committed state of your git checkout like this:
+
+```
+guix pack -f guix.scm
+```
+
+Do this to build a Docker image containing this package together with a matching Python installation:
+
+```
+guix pack -C none \
+  -e '(load "guix.scm")' \
+  -f docker \
+  -S /bin=bin -S /lib=lib -S /share=share \
+  glibc-locales coreutils bash python
+```
+
+# Installation
+
+To install the project using setuptools, you can follow these steps:
+
+1. Clone the project from the Git repository:
+```
+git clone git@github.com:BIMSBbioinfo/flexynesis.git
+```
+2. Navigate to the project directory:
+```
+cd flexynesis
+```
+3. Create a clone of the development environment, use the `spec-file.txt`:
+```
+conda create --name flexynesis --file spec-file.txt
+conda activate flexynesis
+```
+4. Install the project:
+```
+pip install -e .
+```
+
+# Testing
+
+Run unit tests
+```python
+pytest -vvv tests/unit
+```
+
+This will run all the unit tests in the tests directory.
+
+# Contributing
+If you would like to contribute to the project, please open an issue or a pull request on the GitHub repository.
+
+# License
+This package is currently private and is not meant to be used outside of Arcas.ai
+
+# Authors
+
+- Bora Uyar, bora.uyar@mdc-berlin.de
+- Taras Savchyn, Taras.Savchyn@mdc-berlin.de
+- Ricardo Wurmus, Ricardo.Wurmus@mdc-berlin.de
+- Altuna Akalin, Altuna.Akalin@mdc-berlin.de
+"""
+
+from .modules import *
+from .data import *
+from .main import *
+from .models import *
+from .feature_selection import *
+from .utils import *
+from .config import *

flexynesis/__main__.py

@@ -0,0 +1,327 @@
+from lightning import seed_everything
+# Set the seed for all the possible random number generators.
+seed_everything(42, workers=True)
+import lightning as pl
+from typing import NamedTuple
+import os, yaml, torch, time, random, warnings, argparse 
+os.environ["OMP_NUM_THREADS"] = "1"
+import pandas as pd
+import flexynesis
+from flexynesis.models import *
+from lightning.pytorch.callbacks import EarlyStopping
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Flexynesis - Your PyTorch model training interface", 
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    
+    parser.add_argument("--data_path", help="(Required) Path to the folder with train/test data files", type=str, required = True)
+    parser.add_argument("--model_class", help="(Required) The kind of model class to instantiate", type=str, 
+                        choices=["DirectPred", "DirectPredGCNN", "supervised_vae", "MultiTripletNetwork", "CrossModalPred"], required = True)
+    parser.add_argument("--gnn_conv_type", help="If model_class is set to DirectPredGCNN, choose which graph convolution type to use", type=str, 
+                        choices=["GC", "GCN", "GAT", "SAGE"])
+    parser.add_argument("--target_variables", 
+                        help="(Optional if survival variables are not set to None)." 
+                        "Which variables in 'clin.csv' to use for predictions, comma-separated if multiple", 
+                        type = str, default = None)
+    parser.add_argument("--batch_variables", 
+                        help="(Optional) Which variables in 'clin.csv' to use for data integration / batch correction, comma-separated if multiple", 
+                        type = str, default = None)
+    parser.add_argument("--surv_event_var", help="Which column in 'clin.csv' to use as event/status indicator for survival modeling", type = str, default = None)
+    parser.add_argument("--surv_time_var", help="Which column in 'clin.csv' to use as time/duration indicator for survival modeling", type = str, default = None)
+    parser.add_argument('--config_path', type=str, default=None, help='Optional path to an external hyperparameter configuration file in YAML format.')
+    parser.add_argument("--fusion_type", help="How to fuse the omics layers", type=str, choices=["early", "intermediate"], default = 'intermediate')
+    parser.add_argument("--hpo_iter", help="Number of iterations for hyperparameter optimisation", type=int, default = 5)
+    parser.add_argument("--finetuning_samples", help="Number of samples from the test dataset to use for fine-tuning the model. Set to 0 to disable fine-tuning", type=int, default = 0)
+    parser.add_argument("--variance_threshold", help="Variance threshold (as percentile) to drop low variance features (default: 1; set to 0 for no variance filtering)", type=float, default = 1)
+    parser.add_argument("--correlation_threshold", help="Correlation threshold to drop highly redundant features (default: 0.8; set to 1 for no redundancy filtering)", type=float, default = 0.8)
+    parser.add_argument("--restrict_to_features", help="Restrict the analyis to the list of features provided by the user (default: None)", type = str, default = None)
+    parser.add_argument("--subsample", help="Downsample training set to randomly drawn N samples for training. Disabled when set to 0", type=int, default = 0)
+    parser.add_argument("--features_min", help="Minimum number of features to retain after feature selection", type=int, default = 500)
+    parser.add_argument("--features_top_percentile", help="Top percentile features (among the features remaining after variance filtering and data cleanup to retain after feature selection", type=float, default = 20)
+    parser.add_argument("--data_types", help="(Required) Which omic data matrices to work on, comma-separated: e.g. 'gex,cnv'", type=str, required = True)
+    parser.add_argument("--input_layers", 
+                        help="If model_class is set to CrossModalPred, choose which data types to use as input/encoded layers"
+                        "Comma-separated if multiple",
+                        type=str, default = None
+                        )
+    parser.add_argument("--output_layers", 
+                        help="If model_class is set to CrossModalPred, choose which data types to use as output/decoded layers"
+                        "Comma-separated if multiple",
+                        type=str, default = None
+                        )    
+    parser.add_argument("--outdir", help="Path to the output folder to save the model outputs", type=str, default = os.getcwd())
+    parser.add_argument("--prefix", help="Job prefix to use for output files", type=str, default = 'job')
+    parser.add_argument("--log_transform", help="whether to apply log-transformation to input data matrices", type=str, choices=['True', 'False'], default = 'False')
+    parser.add_argument("--early_stop_patience", help="How many epochs to wait when no improvements in validation loss is observed (default: 10; set to -1 to disable early stopping)", type=int, default = 10)
+    parser.add_argument("--hpo_patience", help="How many hyperparamater optimisation iterations to wait for when no improvements are observed (default: 10; set to 0 to disable early stopping)", type=int, default = 10)
+    parser.add_argument("--use_cv", action="store_true", 
+                        help="(Optional) If set, the a 5-fold cross-validation training will be done. Otherwise, a single trainign on 80% of the dataset is done.")
+    parser.add_argument("--use_loss_weighting", help="whether to apply loss-balancing using uncertainty weights method", type=str, choices=['True', 'False'], default = 'True')
+    parser.add_argument("--evaluate_baseline_performance", help="whether to run Random Forest + SVMs to see the performance of off-the-shelf tools on the same dataset", type=str, choices=['True', 'False'], default = 'True')
+    parser.add_argument("--threads", help="(Optional) How many threads to use when using CPU (default: 4)", type=int, default = 4)
+    parser.add_argument("--use_gpu", action="store_true", 
+                        help="(Optional) If set, the system will attempt to use CUDA/GPU if available.")
+    # DirectPredGCNN args.
+    parser.add_argument("--graph", help="Graph to use, name of the database or path to the edge list on the disk.", type=str,  default="STRING")
+    parser.add_argument("--string_organism", help="STRING DB organism id.", type=int, default=9606)
+    parser.add_argument("--string_node_name", help="Type of node name.", type=str, choices=["gene_name", "gene_id"], default="gene_name")
+
+
+    warnings.filterwarnings("ignore", ".*does not have many workers.*")
+    warnings.filterwarnings("ignore", "has been removed as a dependency of the")
+    warnings.filterwarnings("ignore", "The `srun` command is available on your system but is not used")
+
+    args = parser.parse_args()
+    
+    # do some sanity checks on input arguments
+    # 1. Check for survival variables consistency
+    if (args.surv_event_var is None) != (args.surv_time_var is None):
+        parser.error("Both --surv_event_var and --surv_time_var must be provided together or left as None.")
+
+    # 2. Check for required variables for model classes
+    if args.model_class != "supervised_vae" and args.model_class != 'CrossModalPred':
+        if not any([args.target_variables, args.surv_event_var, args.batch_variables]):
+            parser.error(''.join(["When selecting a model other than 'supervised_vae' or 'CrossModalPred',",
+                                  "you must provide at least one of --target_variables, ",
+                                  "survival variables (--surv_event_var and --surv_time_var)",
+                                  "or --batch_variables."]))
+
+    # 3. Check for compatibility of fusion_type with DirectPredGCNN
+    if args.fusion_type == "early":
+        if args.model_class == "DirectPredGCNN":
+            parser.error("The 'DirectPredGCNN' model cannot be used with early fusion type. "
+                         "Use --fusion_type intermediate instead.")
+        if args.model_class == 'CrossModalPred': 
+            parser.error("The 'CrossModalPred' model cannot be used with early fusion type. "
+                         "Use --fusion_type intermediate instead.")
+            
+    
+    # 4. Check for device availability if --accelerator is set. 
+    if args.use_gpu:
+        if not torch.cuda.is_available():
+            warnings.warn(''.join(["\n\n!!! WARNING: GPU REQUESTED BUT NOT AVAILABLE. FALLING BACK TO CPU.\n",
+                                   "PERFORMANCE MAY BE DEGRADED, PARTICULARLY FOR DirectPredGCNN.\n",
+                                   "OTHER MODELS SHOULD HAVE REASONABLE PERFORMANCE ON CPU. \n",
+                                   "IF USING A SLURM SCHEDULER, ENSURE YOU REQUEST A GPU WITH: ",
+                                   "`srun --gpus=1 --pty flexynesis <rest of your_command>` !!!\n\n"]))
+            time.sleep(3)  #wait a bit to capture user's attention to the warning
+            device_type = 'cpu'
+            torch.set_num_threads(args.threads)
+        else:
+            device_type = 'gpu'
+    else:
+        device_type = 'cpu'
+        torch.set_num_threads(args.threads)
+
+    # 5. check GNN arguments
+    if args.model_class == 'DirectPredGCNN':
+        if not args.gnn_conv_type:
+            warning_message = "\n".join([
+                "\n\n!!! When running DirectPredGCNN, a convolution type can be set",
+                "with the --gnn_conv_type flag. See `flexynesis -h` for full set of options.",
+                "Falling back on the default convolution type: GC !!!\n\n"
+            ])
+            warnings.warn(warning_message)
+            time.sleep(3)  #wait a bit to capture user's attention to the warning
+            gnn_conv_type = 'GC'
+        else:
+            gnn_conv_type = args.gnn_conv_type
+    else:
+        gnn_conv_type = None
+        
+    # 6. Check CrossModalPred arguments
+    input_layers = args.input_layers
+    output_layers = args.output_layers
+    datatypes = args.data_types.strip().split(',')
+    if args.model_class == 'CrossModalPred':
+        # check if input output layers are matching the requested data types 
+        if args.input_layers: 
+            input_layers = input_layers.strip().split(',')
+            # Check if input_layers are a subset of datatypes
+            if not all(layer in datatypes for layer in input_layers):
+                raise ValueError(f"Input layers {input_layers} are not a valid subset of the data types: ({datatypes}).")
+        # check if output_layers are a subset of datatypes
+        if args.output_layers: 
+            output_layers = output_layers.strip().split(',')
+            if not all(layer in datatypes for layer in output_layers):
+                raise ValueError(f"Output layers {output_layers} are not a valid subset of the data types: ({datatypes}).")
+        
+    # Validate paths
+    if not os.path.exists(args.data_path):
+        raise FileNotFoundError(f"Input --data_path doesn't exist at:",  {args.data_path})
+    if not os.path.exists(args.outdir):
+        raise FileNotFoundError(f"Path to --outdir doesn't exist at:",  {args.outdir})
+
+    class AvailableModels(NamedTuple):
+        # type AvailableModel = ModelClass: Type, ModelConfig: str
+        DirectPred: tuple[DirectPred, str] = DirectPred, "DirectPred"
+        supervised_vae: tuple[supervised_vae, str] = supervised_vae, "supervised_vae"
+        MultiTripletNetwork: tuple[MultiTripletNetwork, str] = MultiTripletNetwork, "MultiTripletNetwork"
+        DirectPredGCNN: tuple[DirectPredGCNN, str] = DirectPredGCNN, "DirectPredGCNN"
+        CrossModalPred: tuple[CrossModalPred, str] = CrossModalPred, "CrossModalPred"
+
+    available_models = AvailableModels()
+    model_class = getattr(available_models, args.model_class, None)
+    if model_class is None:
+        raise ValueError(f"Invalid model_class: {args.model_class}")
+    else:
+        model_class, config_name = model_class
+
+    # Fix graph argument for non GNN models.
+    graph = args.graph if config_name == "DirectPredGCNN" else None
+
+    # import assays and labels
+    inputDir = args.data_path
+    
+    # Set concatenate to True to use early fusion, otherwise it will run intermediate fusion
+    concatenate = False 
+    if args.fusion_type == 'early':
+        concatenate = True
+        
+    data_importer = flexynesis.DataImporter(path = args.data_path, 
+                                            data_types = datatypes,
+                                            concatenate = concatenate, 
+                                            log_transform = args.log_transform == 'True',
+                                            variance_threshold = args.variance_threshold/100,  
+                                            correlation_threshold = args.correlation_threshold,
+                                            restrict_to_features = args.restrict_to_features,
+                                            min_features= args.features_min, 
+                                            top_percentile= args.features_top_percentile,
+                                            graph=graph,
+                                            processed_dir = '_'.join(['processed', args.prefix]),
+                                            string_organism=args.string_organism,
+                                            string_node_name=args.string_node_name, 
+                                            downsample = args.subsample)
+    train_dataset, test_dataset = data_importer.import_data(force = True)
+    
+    # print feature logs to file (we use these tables to track which features are dropped/selected and why)
+    feature_logs = data_importer.feature_logs
+    for key in feature_logs.keys():
+        feature_logs[key].to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'feature_logs', key, 'csv'])), 
+                                 header=True, index=False)
+
+    # define a tuner object, which will instantiate a DirectPred class 
+    # using the input dataset and the tuning configuration from the config.py
+    tuner = flexynesis.HyperparameterTuning(dataset = train_dataset, 
+                                            model_class = model_class, 
+                                            target_variables = args.target_variables.strip().split(',') if args.target_variables is not None else [],
+                                            batch_variables = args.batch_variables.strip().split(',') if args.batch_variables is not None else None,
+                                            surv_event_var = args.surv_event_var,
+                                            surv_time_var = args.surv_time_var,
+                                            config_name = config_name, 
+                                            config_path = args.config_path,
+                                            n_iter=int(args.hpo_iter),
+                                            use_loss_weighting = args.use_loss_weighting == 'True',
+                                            use_cv = args.use_cv, 
+                                            early_stop_patience = int(args.early_stop_patience), 
+                                            device_type = device_type,
+                                            gnn_conv_type = gnn_conv_type,
+                                            input_layers = input_layers,
+                                            output_layers = output_layers)    
+    
+    # do a hyperparameter search training multiple models and get the best_configuration 
+    model, best_params = tuner.perform_tuning(hpo_patience = args.hpo_patience)
+        
+    # if fine-tuning is enabled; fine tune the model on a portion of test samples 
+    if args.finetuning_samples > 0:
+        finetuneSampleN = args.finetuning_samples
+        print("[INFO] Finetuning the model on ",finetuneSampleN,"test samples")
+        # split test dataset into finetuning and holdout datasets 
+        all_indices = range(len(test_dataset))
+        finetune_indices = random.sample(all_indices, finetuneSampleN)
+        holdout_indices = list(set(all_indices) - set(finetune_indices))
+        finetune_dataset = test_dataset.subset(finetune_indices)
+        holdout_dataset = test_dataset.subset(holdout_indices)
+        
+        # fine tune on the finetuning dataset; freeze the encoders 
+        finetuner = flexynesis.FineTuner(model, 
+                                         finetune_dataset)
+        finetuner.run_experiments()
+            
+        # update the model to finetuned model 
+        model = finetuner.model 
+        # update the test dataset to exclude finetuning samples
+        test_dataset = holdout_dataset 
+    
+    # evaluate predictions;  (if any supervised learning happened)
+    if any([args.target_variables, args.surv_event_var, args.batch_variables]):
+        print("[INFO] Computing model evaluation metrics")
+        metrics_df = flexynesis.evaluate_wrapper(model.predict(test_dataset), test_dataset, 
+                                                 surv_event_var=model.surv_event_var, 
+                                                 surv_time_var=model.surv_time_var)
+        metrics_df.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'stats.csv'])), header=True, index=False)
+
+        # print known/predicted labels 
+        predicted_labels = pd.concat([flexynesis.get_predicted_labels(model.predict(train_dataset), train_dataset, 'train'),
+                                      flexynesis.get_predicted_labels(model.predict(test_dataset), test_dataset, 'test')], 
+                                    ignore_index=True)
+        predicted_labels.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'predicted_labels.csv'])), header=True, index=False)
+        # compute feature importance values
+        print("[INFO] Computing variable importance scores")
+        for var in model.target_variables:
+            model.compute_feature_importance(train_dataset, var, steps = 50)
+        df_imp = pd.concat([model.feature_importances[x] for x in model.target_variables], 
+                           ignore_index = True)
+        df_imp.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'feature_importance.csv'])), header=True, index=False)
+
+    # get sample embeddings and save 
+    print("[INFO] Extracting sample embeddings")
+    embeddings_train = model.transform(train_dataset)
+    embeddings_test = model.transform(test_dataset)
+    
+    embeddings_train.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'embeddings_train.csv'])), header=True)
+    embeddings_test.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'embeddings_test.csv'])), header=True)
+    
+    # also filter embeddings to remove batch-associated dims and only keep target-variable associated dims 
+    if args.batch_variables is not None:
+        print("[INFO] Printing filtered embeddings")
+        embeddings_train_filtered = flexynesis.remove_batch_associated_variables(data = embeddings_train, 
+                                                                                 batch_dict={x: train_dataset.ann[x] for x in model.batch_variables} if model.batch_variables is not None else None, 
+                                                                                 target_dict={x: train_dataset.ann[x] for x in model.target_variables}, 
+                                                                                 variable_types=train_dataset.variable_types)
+        # filter test embeddings to keep the same dims as the filtered training embeddings
+        embeddings_test_filtered = embeddings_test[embeddings_train_filtered.columns]
+
+        # save 
+        embeddings_train_filtered.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'embeddings_train.filtered.csv'])), header=True)    
+        embeddings_test_filtered.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'embeddings_test.filtered.csv'])), header=True)    
+
+    # for architectures with decoders; print decoded output layers 
+    if args.model_class == 'CrossModalPred':
+        print("[INFO] Printing decoded output layers")
+        output_layers_train = model.decode(train_dataset)
+        output_layers_test = model.decode(test_dataset)
+        for layer in output_layers_train.keys():
+            output_layers_train[layer].to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'train_decoded', layer, 'csv'])), header=True)
+        for layer in output_layers_test.keys():
+            output_layers_test[layer].to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'test_decoded', layer, 'csv'])), header=True)
+
+    
+    # evaluate off-the-shelf methods on the main target variable 
+    if args.evaluate_baseline_performance == 'True':
+        print("[INFO] Computing off-the-shelf method performance on first target variable:",model.target_variables[0])
+        var = model.target_variables[0]
+        metrics = pd.DataFrame()
+        if var != model.surv_event_var: 
+            metrics = flexynesis.evaluate_baseline_performance(train_dataset, test_dataset, 
+                                                            variable_name = var, 
+                                                            n_folds=5,
+                                                            n_jobs = int(args.threads))
+        if model.surv_event_var and model.surv_time_var:
+            print("[INFO] Computing off-the-shelf method performance on survival variable:",model.surv_time_var)
+            metrics_baseline_survival = flexynesis.evaluate_baseline_survival_performance(train_dataset, test_dataset, 
+                                                                                             model.surv_time_var, 
+                                                                                             model.surv_event_var, 
+                                                                                             n_folds = 5,
+                                                                                             n_jobs = int(args.threads))
+            metrics = pd.concat([metrics, metrics_baseline_survival], axis = 0, ignore_index = True)
+        
+        if not metrics.empty:
+            metrics.to_csv(os.path.join(args.outdir, '.'.join([args.prefix, 'baseline.stats.csv'])), header=True, index=False) 
+    
+    # save the trained model in file
+    torch.save(model, os.path.join(args.outdir, '.'.join([args.prefix, 'final_model.pth'])))
+    
+if __name__ == "__main__":
+    main()