philm 0.0.1__py3-none-any.whl

philm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

philm/config/config_repr.test.yaml

@@ -0,0 +1,7 @@
+data:
+  test_X: data/Phage_test.tsv
+output_prefix: results/PHILM_test
+model:
+  path: results/PHILM_best_model.pth
+  output_dim: #OUTDIM#
+  batch_size: 512

philm/config/config_repr.train.yaml

@@ -0,0 +1,7 @@
+data:
+  test_X: data/Phage_train.tsv
+output_prefix: results/PHILM_train
+model:
+  path: results/PHILM_best_model.pth
+  output_dim: #OUTDIM#
+  batch_size: 512

philm/config/config_repr.val.yaml

@@ -0,0 +1,7 @@
+data:
+  test_X: data/Phage_val.tsv
+output_prefix: results/PHILM_val
+model:
+  path: results/PHILM_best_model.pth
+  output_dim: #OUTDIM#
+  batch_size: 512

philm/config/config_test.train.yaml

@@ -0,0 +1,7 @@
+data:
+  test_X: data/Phage_train.tsv
+  test_Y: data/Bact_arc_train.tsv
+output_prefix: results/PHILM_predict_train
+model:
+  path: results/PHILM_best_model.pth
+  batch_size: 512

philm/config/config_test.val.yaml

@@ -0,0 +1,7 @@
+data:
+  test_X: data/Phage_val.tsv
+  test_Y: data/Bact_arc_val.tsv
+output_prefix: results/PHILM_predict_val
+model:
+  path: results/PHILM_best_model.pth
+  batch_size: 512

philm/config/config_test.yaml

@@ -0,0 +1,7 @@
+data:
+  test_X: data/Phage_test.tsv
+  test_Y: data/Bact_arc_test.tsv
+output_prefix: results/PHILM_predict_test
+model:
+  path: results/PHILM_best_model.pth
+  batch_size: 512

philm/config/config_train.yaml

@@ -0,0 +1,12 @@
+data:
+  train_X: data/Phage_train.tsv
+  train_Y: data/Bact_arc_train.tsv
+  val_X: data/Phage_val.tsv
+  val_Y: data/Bact_arc_val.tsv
+model:
+  batch_size: 512 # Batch size
+  patience: 20 # Number of epochs with no improvement after which training will be stopped
+  num_epochs: 8000 # Maximal epochs for training
+  n_trials: 10 # Number of trials for choosing the optimal combinations of parameters, default=10
+  path: results/PHILM_best_model.pth
+  para_path: results/PHILM_best_params.yaml

philm/config/config_train_perm.yaml

@@ -0,0 +1,44 @@
+# Fixed-parameter PHILM null-permutation training config.
+# Place this file in config/config_train_permutation.yaml and run from the repo root:
+#   python model/train_permutation.py -c config/config_train_permutation.yaml --perm-id 0
+# For 100 HPC array jobs, pass --perm-id ${SLURM_ARRAY_TASK_ID}.
+
+data:
+  train_X: data/Phage_train.tsv
+  train_Y: data/Bact_arc_train.tsv
+  val_X: data/Phage_val.tsv
+  val_Y: data/Bact_arc_val.tsv
+
+permutation:
+  # Overridden by --perm-id.
+  id: 0
+
+  # Seed used as: seed = base_seed + perm_id, unless --seed is supplied.
+  base_seed: 19910728
+
+  # Null model: shuffle phage/X sample rows relative to fixed bacteria/Y rows.
+  # This breaks the paired phage-bacteria relationship before training.
+  method: shuffle
+
+  # Avoid cases where a Y row is still paired with its original X row.
+  avoid_self_pairs: true
+
+  # Recommended: permute both train and validation pairs for a matched null-training workflow.
+  permute_train_X: true
+  permute_val_X: true
+
+  map_path_template: results/permutation_null/perm_{perm_id_padded}/PHILM_perm_{perm_id_padded}.sample_permutation.tsv
+
+runtime:
+  device: auto
+
+model:
+  batch_size: 512
+  patience: 20
+  num_epochs: 8000
+
+  # Output paths. {perm_id_padded} will be 000, 001, ..., 099.
+  path_template: results/permutation_null/perm_{perm_id_padded}/PHILM_perm_{perm_id_padded}.pth
+  para_path_template: results/permutation_null/perm_{perm_id_padded}/PHILM_perm_{perm_id_padded}.params.yaml
+  log_path_template: results/permutation_null/perm_{perm_id_padded}/PHILM_perm_{perm_id_padded}.train.log
+

philm/model/data_loader.py

@@ -0,0 +1,27 @@
+# data_loader.py
+import torch
+from torch.utils.data import Dataset, DataLoader
+
+class MGX_MVX_Dataset(Dataset):
+    def __init__(self, train_X_data, train_Y_data):
+        self.train_X_data = train_X_data
+        self.train_Y_data = train_Y_data
+
+    def __len__(self):
+        return len(self.train_X_data)
+
+    def __getitem__(self, idx):
+        X = torch.tensor(self.train_X_data.loc[idx], dtype=torch.float32)
+        y = torch.tensor(self.train_Y_data.loc[idx], dtype=torch.float32)
+        return X, y
+
+class Predict_Dataset(Dataset):
+    def __init__(self, X_data):
+        self.X_data = X_data
+
+    def __len__(self):
+        return len(self.X_data)
+
+    def __getitem__(self, idx):
+        X = torch.tensor(self.X_data.loc[idx], dtype=torch.float32)
+        return X

philm/model/evaluate.py

@@ -0,0 +1,104 @@
+import torch
+import sys, yaml, argparse
+import pandas as pd
+from torch.utils.data import DataLoader
+
+from .data_loader import MGX_MVX_Dataset
+from .model import MLP_NODE
+from .utils import ft_pearson_corr_list, ft_cos_sim_list, ft_r2score_list, pearson_corr_list, cos_sim_list, r2score_list
+
+# Device configuration
+if torch.cuda.is_available():
+    device = torch.device(0)  # Use GPU 0
+else:
+    device = torch.device('cpu') # Fallback to CPU
+
+def load_config(config_path):
+    with open(config_path, 'r') as file:
+        return yaml.safe_load(file)
+    
+def test_model(args):
+    # Load configuration
+    config = load_config(args.config)
+
+    # read data
+    test_X_data = pd.read_table(config['data']['test_X'], header=None)
+    test_Y_data = pd.read_table(config['data']['test_Y'], header=None)
+    dataset = MGX_MVX_Dataset(test_X_data, test_Y_data)
+
+    # load model
+    # define hyperparameters (must be the same as during training)
+    input_dim = test_X_data.shape[1]  # Should match training input_dim
+    output_dim = test_Y_data.shape[1]  # Use the output_dim from training
+    
+    batch_size=config['model']['batch_size']
+    hidden_size=config['model']['hidden_size']
+
+    # load data
+    test_loader = DataLoader(dataset, batch_size=batch_size)
+
+    model = MLP_NODE(
+        input_dim=input_dim,
+        output_dim=output_dim,
+        hidden_size=hidden_size
+    )
+    model.load_state_dict(torch.load(config['model']['path'], weights_only=False, map_location=device))
+    model.to(device)
+    model.eval()
+
+    preds = torch.tensor([]).to(device)
+    grounds = torch.tensor([]).to(device)
+    with torch.no_grad():
+        for X_batch, Y_batch in test_loader:
+            X_batch, Y_batch = X_batch.to(device), Y_batch.to(device)
+            
+            pred = model(X_batch)
+            pred = pred.to(device)
+            preds = torch.cat((preds, pred), 0)
+            grounds = torch.cat((grounds, Y_batch), 0)
+            torch.cuda.empty_cache()            
+            
+    prediction = preds
+    ground_truth = grounds
+
+    print(prediction.shape)
+    prefix=config['output_prefix']
+    dt = pd.DataFrame(prediction.cpu().numpy())
+    dt.to_csv(prefix+"_pred.tsv", sep="\t", header=False, index=False)
+
+    # feature-wise metrics
+    pcc_lst = ft_pearson_corr_list(prediction, ground_truth)
+    pcc_dt = pd.DataFrame(pcc_lst)
+    pcc_dt.to_csv(prefix+"_pcc_ft.tsv", sep="\t", header=["PCC"], index=False)
+
+    cos_lst = ft_cos_sim_list(prediction, ground_truth)
+    cos_dt = pd.DataFrame(cos_lst.cpu().numpy())
+    cos_dt.to_csv(prefix+"_cos_sim_ft.tsv", sep="\t", header=["Cos_Sim"], index=False)
+
+    r2_lst = ft_r2score_list(prediction, ground_truth)
+    r2_dt = pd.DataFrame(r2_lst)
+    r2_dt.to_csv(prefix+"_r2_ft.tsv", sep="\t", header=["R2"], index=False)
+
+    # sample-wise metrics
+    pcc_lst = pearson_corr_list(prediction, ground_truth)
+    pcc_dt = pd.DataFrame(pcc_lst)
+    pcc_dt.to_csv(prefix+"_pcc.tsv", sep="\t", header=["PCC"], index=False)
+
+    cos_lst = cos_sim_list(prediction, ground_truth)
+    cos_dt = pd.DataFrame(cos_lst.cpu().numpy())
+    cos_dt.to_csv(prefix+"_cos_sim.tsv", sep="\t", header=["Cos_Sim"], index=False)
+
+    r2_lst = r2score_list(prediction, ground_truth)
+    r2_dt = pd.DataFrame(r2_lst)
+    r2_dt.to_csv(prefix+"_r2.tsv", sep="\t", header=["R2"], index=False)
+
+def main():
+    from philm import __version__
+    parser = argparse.ArgumentParser(description=f'PHILM {__version__} evaluate model') 
+    parser.add_argument('-c', '--config', required=True, type=str,
+                        help='config file path')
+    args = parser.parse_args()
+    test_model(args)
+
+if __name__ == '__main__':
+    main()

philm/model/extract_repr.py

@@ -0,0 +1,75 @@
+import torch
+import sys, yaml, argparse
+import pandas as pd
+from torch.utils.data import DataLoader
+
+from .data_loader import MGX_MVX_Dataset, Predict_Dataset
+from .model import MLP_NODE
+
+# Device configuration
+if torch.cuda.is_available():
+    device = torch.device(0)  # Use GPU 0
+else:
+    device = torch.device('cpu')     # Fallback to CPU
+
+def load_config(config_path):
+    with open(config_path, 'r') as file:
+        return yaml.safe_load(file)
+    
+def extract_repr(args):
+    # Load configuration
+    config = load_config(args.config)
+
+    # read data
+    test_X_data = pd.read_table(config['data']['test_X'], header=None)
+    dataset = Predict_Dataset(test_X_data)
+
+    # load model
+    # define hyperparameters (must be the same as during training)
+    input_dim = test_X_data.shape[1]  # Should match training input_dim
+    output_dim = config['model']['output_dim']
+    batch_size=config['model']['batch_size']
+    hidden_size=config['model']['hidden_size']
+
+    # load data
+    dataloader = DataLoader(dataset, batch_size=batch_size)
+
+    model = MLP_NODE(
+        input_dim=input_dim, 
+        output_dim=output_dim,
+        hidden_size=hidden_size
+    )
+
+    model.load_state_dict(torch.load(config['model']['path'], weights_only=False, map_location=device))
+    model.to(device)
+    model.eval()
+
+    repr1_list = torch.tensor([]).to(device)
+    repr2_list = torch.tensor([]).to(device)
+    with torch.no_grad():
+        for X_batch in dataloader:
+            X_batch = X_batch.to(device)
+            repr1 = model.get_repr1(X_batch)
+            repr1_list = torch.cat((repr1_list, repr1), 0)
+            repr2 = model.get_repr2(X_batch)
+            repr2_list = torch.cat((repr2_list, repr2), 0)
+            torch.cuda.empty_cache()
+
+    print(repr1_list.shape)
+    prefix=config['output_prefix']
+    dt = pd.DataFrame(repr1_list.cpu().numpy())
+    dt.to_csv(prefix+"_repr1.tsv", sep="\t", header=False, index=False)
+
+    dt = pd.DataFrame(repr2_list.cpu().numpy())
+    dt.to_csv(prefix+"_repr2.tsv", sep="\t", header=False, index=False)
+
+def main():
+    from philm import __version__
+    parser = argparse.ArgumentParser(description=f'PHILM {__version__} extract representations') 
+    parser.add_argument('-c', '--config', required=True, type=str,
+                        help='config file path')
+    args = parser.parse_args()
+    extract_repr(args)
+
+if __name__ == '__main__':
+    main()

philm/model/model.py

@@ -0,0 +1,48 @@
+import torch
+import torch.nn as nn
+import torchdiffeq
+
+class ODEFunc(nn.Module):
+    """
+    Defines the neural ODE function.
+    """
+    def __init__(self, hidden_dim):
+        super(ODEFunc, self).__init__()
+        layers = []
+        prev_dim = hidden_dim
+        for _ in range(2):
+            # print(prev_dim, h_dim)
+            layers.append(nn.Linear(prev_dim, hidden_dim))
+            layers.append(nn.Tanh())
+            prev_dim = hidden_dim
+        self.model = nn.Sequential(*layers)
+
+    def forward(self, t, x):
+        return self.model(x)
+
+class MLP_NODE(nn.Module):
+    def __init__(self, input_dim, hidden_size, output_dim):
+        super(MLP_NODE, self).__init__()
+        self.fc1 = nn.Linear(input_dim, hidden_size)
+        self.ode_func = ODEFunc(hidden_size)
+        self.ode_solver = torchdiffeq.odeint  # Using adjoint method for memory efficiency
+        self.fc2 = nn.Linear(hidden_size, output_dim)
+
+    def forward(self, x):
+        x = torch.flatten(x, 1)  # Flatten the input tensor except for the batch dimension
+        x = self.fc1(x)
+        t = torch.tensor([0, 1], dtype=torch.float32, device=x.device)  # Time interval for ODE solver, moved to the same device as x
+        x = self.ode_solver(self.ode_func, x, t, rtol=1e-4, atol=1e-4)[-1]  # Get the result at the final time point
+        x = self.fc2(x)
+        return x
+
+    def get_repr1(self, x):
+        repr1 = self.fc1(x)
+        return repr1
+    
+    def get_repr2(self, x):
+        x = torch.flatten(x, 1)  # Flatten the input tensor except for the batch dimension
+        x = self.fc1(x)
+        t = torch.tensor([0, 1], dtype=torch.float32, device=x.device)  # Time interval for ODE solver, moved to the same device as x
+        repr2 = self.ode_solver(self.ode_func, x, t, rtol=1e-4, atol=1e-4)[-1]  # Get the result at the final time point
+        return repr2

philm/model/optimize.py

@@ -0,0 +1,60 @@
+import optuna
+import torch, os
+from .trainer import train_model
+
+def determine_nodelayer_no(input_num, output_num):
+    pool = [4, 8, 16, 32, 64, 128, 256, 320, 512, 640, 1024, 2048, 2560, 4096]
+    print(input_num, output_num)
+    val = input_num + output_num
+    hidden_dim = min(pool, key=lambda x: abs(x - val/2))
+    
+    # print(hidden_dim)
+    hidden_size_list = [hidden_dim]
+    indices = [pool.index(hidden_dim)]
+    if (pool.index(hidden_dim)-1) > 0:
+        indices.append(pool.index(hidden_dim)-1)
+        hidden_size_list.append(pool[pool.index(hidden_dim)-1])
+    if (pool.index(hidden_dim)+1) < len(pool):
+        indices.append(pool.index(hidden_dim)+1)
+        hidden_size_list.append(pool[pool.index(hidden_dim)+1])
+    return hidden_size_list
+
+def objective(trial, train_X_data, train_Y_data, val_X_data, val_Y_data, batch_size, patience, num_epochs, device):
+    
+    input_num = train_X_data.shape[1]
+    output_num = train_Y_data.shape[1]
+    hidden_size_list = determine_nodelayer_no(input_num, output_num)
+    print("candidates:", hidden_size_list)
+    
+    # Suggest hyperparameters
+    learning_rate = trial.suggest_float("learning_rate", 1e-6, 1e-2, log=True)
+    hidden_size = trial.suggest_categorical("hidden_size", hidden_size_list)
+    weight_decay = trial.suggest_float("weight_decay", 1e-8, 1e-5, log=True)
+
+    # Ensure the directory exists
+    os.makedirs("checkpoints", exist_ok=True)
+
+    model_path = os.path.join("checkpoints", f"model_trial_{trial.number}.pth")  # Save models in a folder named 'models'
+    log_path = os.path.join("checkpoints", f"train_{trial.number}.log")
+
+    # Call the train function and get the validation loss
+    best_val_loss = train_model(
+        train_X_data=train_X_data,
+        train_Y_data=train_Y_data, 
+        val_X_data=val_X_data,
+        val_Y_data=val_Y_data,
+        device=device,
+        num_epochs=num_epochs,
+        batch_size=batch_size,
+        learning_rate=learning_rate,
+        hidden_size=hidden_size,
+        patience=patience,
+        weight_decay=weight_decay,
+        model_path=model_path,
+        log_path=log_path
+    )
+
+    # Store the model name in the trial's user attributes for later reference
+    trial.set_user_attr("model_path", model_path)
+
+    return best_val_loss

philm/model/predict.py

@@ -0,0 +1,72 @@
+import torch
+import sys, yaml, argparse
+import pandas as pd
+from torch.utils.data import DataLoader
+
+from .data_loader import Predict_Dataset
+from .model import MLP_NODE
+
+# Device configuration
+if torch.cuda.is_available():
+    device = torch.device(0)  # Use GPU 0
+else:
+    device = torch.device('cpu') # Fallback to CPU
+    
+def load_config(config_path):
+    with open(config_path, 'r') as file:
+        return yaml.safe_load(file)
+    
+def predict(args):
+    # Load configuration
+    config = load_config(args.config)
+
+    # read data
+    test_X_data = pd.read_table(config['data']['test_X'], header=None)
+    dataset = Predict_Dataset(test_X_data)
+
+    # load model
+    # define hyperparameters (must be the same as during training)
+    input_dim = test_X_data.shape[1]  # Should match training input_dim
+    output_dim = config['model']['output_dim'] # Use the output_dim from training
+    batch_size=config['model']['batch_size']
+    hidden_size=config['model']['hidden_size']
+
+    # load data
+    test_loader = DataLoader(dataset, batch_size=batch_size)
+
+    model = MLP_NODE(
+        input_dim=input_dim,
+        output_dim=output_dim,
+        hidden_size=hidden_size
+    )
+    model.load_state_dict(torch.load(config['model']['path'], weights_only=False, map_location=device))
+    model.to(device)
+    model.eval()
+
+    preds = torch.tensor([]).to(device)
+    with torch.no_grad():
+        for X_batch in test_loader:
+            X_batch = X_batch.to(device)
+            
+            pred = model(X_batch)
+            pred = pred.to(device)
+            preds = torch.cat((preds, pred), 0)
+            torch.cuda.empty_cache()
+            
+    prediction = preds
+
+    print(prediction.shape)
+    prefix=config['output_prefix']
+    dt = pd.DataFrame(prediction.cpu().numpy())
+    dt.to_csv(prefix+"_pred.tsv", sep="\t", header=False, index=False)
+
+def main():
+    from philm import __version__
+    parser = argparse.ArgumentParser(description=f'PHILM {__version__} predict model')
+    parser.add_argument('-c', '--config', required=True, type=str,
+                        help='config file path')
+    args = parser.parse_args()
+    predict(args)
+
+if __name__ == '__main__':
+    main()

philm/model/train.py

@@ -0,0 +1,75 @@
+import yaml
+import numpy as np
+import pandas as pd
+import torch
+import sys, os, argparse
+from .trainer import train_model
+from .optimize import objective
+import optuna
+import shutil
+
+
+def load_config(config_path):
+    with open(config_path, 'r') as file:
+        return yaml.safe_load(file)
+    
+def train_model(args):
+    # Load configuration
+    config = load_config(args.config)
+
+    # Load data
+    train_X_data = pd.read_table(config['data']['train_X'], header=None)  # Shape: (samples, bacterial_features)
+    train_Y_data = pd.read_table(config['data']['train_Y'], header=None)  # Shape: (samples, viral_features)
+    val_X_data = pd.read_table(config['data']['val_X'], header=None)  # Shape: (samples, bacterial_features)
+    val_Y_data = pd.read_table(config['data']['val_Y'], header=None)  # Shape: (samples, viral_features)
+    
+    # Load fixed parameters
+    batch_size = config['model']['batch_size']
+    patience = config['model']['patience']
+    num_epochs = config['model']['num_epochs']
+    n_trials = config['model']['n_trials']
+    
+    # Load target model path and parameter path
+    saved_model_path = config['model']['path']
+    para_path = config['model']['para_path']
+        
+    # Device configuration
+    if torch.cuda.is_available():
+        device = torch.device(0)  # Use GPU 0
+    else:
+        device = torch.device('cpu') # Fallback to CPU
+    print(f'Using device: {device}')
+
+    # Create an Optuna study and optimize
+    study = optuna.create_study(direction="minimize")
+    study.optimize(lambda trial: objective(trial, train_X_data, train_Y_data, val_X_data, val_Y_data, batch_size, patience, num_epochs, device), n_trials=n_trials)
+
+    # Print the best parameters, score, and model name
+    print("Best Parameters:", study.best_params)
+    print("Best Val Loss:", study.best_value)
+    print("Best Model Name:", study.best_trial.user_attrs["model_path"])
+    
+    ori_model_path = study.best_trial.user_attrs["model_path"]
+    # os.rename(ori_model_path, saved_model_path)
+    os.makedirs(os.path.dirname(saved_model_path), exist_ok=True)
+    shutil.copy2(ori_model_path, saved_model_path)
+
+    # Save best parameters and model name to a YAML file
+    best_params = {
+        'learning_rate': study.best_params.get('learning_rate', None),
+        'hidden_size': study.best_params.get('hidden_size', None),
+        'weight_decay': study.best_params.get('weight_decay', None)
+    }
+    with open(para_path, "w") as f:
+        yaml.dump(best_params, f, default_flow_style=False)
+    
+def main():
+    from philm import __version__
+    parser = argparse.ArgumentParser(description=f'PHILM {__version__} train model')
+    parser.add_argument('-c', '--config', required=True, type=str,
+                        help='config file path')
+    args = parser.parse_args()    
+    train_model(args)
+
+if __name__ == '__main__':
+    main()