gpbench 1.0.0__py3-none-any.whl

gpbench/method_reg/Cropformer/Cropformer_Hyperparameters.py

@@ -0,0 +1,250 @@
+import time
+import torch
+import numpy as np
+import torch.nn as nn
+import random
+import torch.optim  as optim 
+from torch.utils.data  import DataLoader, TensorDataset 
+from sklearn.preprocessing  import StandardScaler 
+from lightning.pytorch import LightningModule
+import optuna
+
+from torch.optim import Adam
+from torch.nn import MSELoss
+from sklearn.model_selection import KFold
+from sklearn.metrics  import mean_absolute_error, mean_squared_error, r2_score
+
+class LayerNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-12):
+        super(LayerNorm, self).__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.bias = nn.Parameter(torch.zeros(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, x):
+        u = x.mean(-1, keepdim=True)
+        s = (x - u).pow(2).mean(-1, keepdim=True)
+        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
+        return self.weight * x + self.bias
+        
+class SelfAttention(LightningModule):
+    def __init__(self, num_attention_heads, input_size, hidden_size, output_dim=1, kernel_size=3,
+                 hidden_dropout_prob=0.5, attention_probs_dropout_prob=0.5, learning_rate=0.001):
+        super(SelfAttention, self).__init__()
+        self.num_attention_heads = num_attention_heads
+        self.attention_head_size = int(hidden_size / num_attention_heads)
+        self.all_head_size = hidden_size
+
+        self.query = torch.nn.Linear(input_size, self.all_head_size)
+        self.key = torch.nn.Linear(input_size, self.all_head_size)
+        self.value = torch.nn.Linear(input_size, self.all_head_size)
+
+        self.attn_dropout = torch.nn.Dropout(attention_probs_dropout_prob)
+        self.out_dropout = torch.nn.Dropout(hidden_dropout_prob)
+        self.dense = torch.nn.Linear(hidden_size, input_size)
+        self.LayerNorm = torch.nn.LayerNorm(input_size, eps=1e-12)
+        self.relu = torch.nn.ReLU()
+        self.out = torch.nn.Linear(input_size, output_dim)
+        self.cnn = torch.nn.Conv1d(1, 1, kernel_size, stride=1, padding=1)
+
+        self.learning_rate = learning_rate
+        self.loss_fn = MSELoss()
+
+    def forward(self, input_tensor):
+        input_tensor = input_tensor.to(self.device)
+        self.cnn = self.cnn.to(self.device)
+
+        cnn_hidden = self.cnn(input_tensor.view(input_tensor.size(0), 1, -1))
+        input_tensor = cnn_hidden
+        mixed_query_layer = self.query(input_tensor)
+        mixed_key_layer = self.key(input_tensor)
+        mixed_value_layer = self.value(input_tensor)
+
+        query_layer = mixed_query_layer
+        key_layer = mixed_key_layer
+        value_layer = mixed_value_layer
+
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+        attention_scores = attention_scores / np.sqrt(self.attention_head_size)
+        attention_probs = torch.nn.Softmax(dim=-1)(attention_scores)
+        attention_probs = self.attn_dropout(attention_probs)
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+        hidden_states = self.dense(context_layer)
+        hidden_states = self.out_dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        output = self.out(self.relu(hidden_states.view(hidden_states.size(0), -1)))
+        return output
+
+    def training_step(self, batch, batch_idx):
+        x, y = batch
+        y_pred = self(x)
+        loss = self.loss_fn(y_pred, y)
+        return loss
+
+    def validation_step(self, batch, batch_idx):
+        x, y = batch
+        y_pred = self(x)
+        val_loss = self.loss_fn(y_pred, y)
+        return val_loss
+
+    def configure_optimizers(self):
+        return Adam(self.parameters(), lr=self.learning_rate)
+
+class EarlyStopping:
+    def __init__(self, patience=10, delta=0):
+        self.patience = patience
+        self.delta = delta
+        self.best_score = None
+        self.counter = 0
+        self.early_stop = False
+
+    def __call__(self, score):
+        if self.best_score is None:
+            self.best_score = score
+        elif score < self.best_score + self.delta:
+            self.counter += 1
+            if self.counter >= self.patience:
+                self.early_stop = True
+        else:
+            self.best_score = score
+            self.counter = 0
+
+
+def run_nested_cv_with_early_stopping(data, label, outer_cv, learning_rate, num_heads, dropout_prob, batch_size, hidden_dim,
+                                      output_dim, kernel_size, patience, DEVICE):
+    best_corr_coefs = []
+    best_maes = []
+    best_r2s = []
+    best_mses = []
+
+    time_star = time.time()
+    for fold, (train_idx, test_idx) in enumerate(outer_cv.split(data)):
+        x_train, x_test = data[train_idx], data[test_idx]
+        y_train, y_test = label[train_idx], label[test_idx]
+        
+        num_attention_heads = num_heads
+        attention_probs_dropout_prob = dropout_prob
+        hidden_dropout_prob = 0.5
+
+        model = SelfAttention(num_attention_heads, x_train.shape[1], hidden_dim, output_dim,
+                                hidden_dropout_prob=hidden_dropout_prob, kernel_size=kernel_size,
+                                attention_probs_dropout_prob=attention_probs_dropout_prob).to(DEVICE)
+        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+        loss_function = torch.nn.MSELoss()
+        scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=10)
+
+        scaler = StandardScaler()
+        x_train = scaler.fit_transform(x_train)
+        x_test = scaler.transform(x_test)
+
+        x_train_tensor = torch.from_numpy(x_train).float().to(DEVICE)
+        y_train_tensor = torch.from_numpy(y_train).float().to(DEVICE)
+        x_test_tensor = torch.from_numpy(x_test).float().to(DEVICE)
+        y_test_tensor = torch.from_numpy(y_test).float().to(DEVICE)
+
+        train_data = TensorDataset(x_train_tensor, y_train_tensor)
+        test_data = TensorDataset(x_test_tensor, y_test_tensor)
+
+        train_loader = DataLoader(train_data, batch_size, shuffle=True)
+        test_loader = DataLoader(test_data, batch_size, shuffle=False)
+
+        early_stopping = EarlyStopping(patience=patience)
+        best_corr_coef = -float('inf')
+        best_mae = float('inf')
+        best_mse = float('inf')
+        best_r2 = -float('inf')
+        for epoch in range(100):
+            model.train()
+            for x_batch, y_batch in train_loader:
+                optimizer.zero_grad()
+                y_pred = model(x_batch)
+                loss = loss_function(y_pred, y_batch.reshape(-1, 1))
+                loss.backward()
+                optimizer.step()
+
+            model.eval()
+            y_test_preds, y_test_trues = [], []
+            
+            with torch.no_grad():
+                for x_batch, y_batch in test_loader:
+                    y_test_pred = model(x_batch)
+                    y_test_preds.extend(y_test_pred.cpu().numpy().reshape(-1).tolist())
+                    y_test_trues.extend(y_batch.cpu().numpy().reshape(-1).tolist())
+
+            corr_coef = np.corrcoef(y_test_preds, y_test_trues)[0, 1]
+            mae = mean_absolute_error(np.array(y_test_trues), np.array(y_test_preds))
+            mse = mean_squared_error(np.array(y_test_trues), np.array(y_test_preds))
+            r2 = r2_score(np.array(y_test_trues), np.array(y_test_preds))
+            scheduler.step(corr_coef)
+
+            if corr_coef > best_corr_coef:
+                best_mae = mae
+                best_corr_coef = corr_coef
+                best_mse = mse
+                best_r2 = r2
+
+            early_stopping(corr_coef)
+            if early_stopping.early_stop:
+                print(f"Early stopping at epoch {epoch + 1}")
+                break
+        
+        best_corr_coefs.append(best_corr_coef)
+        best_maes.append(best_mae)
+        best_mses.append(best_mse)
+        best_r2s.append(best_r2)
+        print(f'Fold {fold + 1}: MAE={best_mae:.4f}, MSE={best_mse:.4f}, R2={best_r2:.4f}, Corr={best_corr_coef:.4f}')
+    
+    print("==== Final Results ====")
+    print(f"MAE: {np.mean(best_maes):.4f} ± {np.std(best_maes):.4f}")
+    print(f"MSE: {np.mean(best_mses):.4f} ± {np.std(best_mses):.4f}")
+    print(f"R2 : {np.mean(best_r2s):.4f} ± {np.std(best_r2s):.4f}")
+    print(f"Corr: {np.mean(best_corr_coefs):.4f} ± {np.std(best_corr_coefs):.4f}")
+
+    print(f"Time: {time.time() - time_star:.2f}s")
+    return best_corr_coefs
+
+
+def set_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+def Hyperparameter(X, label):
+    set_seed(42)
+    torch.cuda.empty_cache()
+    device = torch.device("cuda:0")
+    
+    def objective(trial):
+        lr = trial.suggest_loguniform("learning_rate", 1e-4, 1e-1)
+        heads = trial.suggest_int("heads", 1, 8, step=1)
+        dropout = trial.suggest_float("dropout", 0.1, 0.9, step=0.1)
+        batch_size = trial.suggest_categorical("batch_size", [32, 64, 128])
+        
+        outer_cv = KFold(n_splits=10, shuffle=True, random_state=42) 
+
+        corr_scores = run_nested_cv_with_early_stopping(
+            data=X,
+            label=label,
+            outer_cv=outer_cv,
+            learning_rate=lr,
+            num_heads=heads,
+            dropout_prob=dropout,
+            batch_size=batch_size,
+            hidden_dim=64,
+            output_dim=1,
+            kernel_size=3,
+            patience=5,
+            DEVICE=device,
+        )
+        return np.mean(corr_scores) 
+
+    study = optuna.create_study(direction="maximize")
+    study.optimize(objective, n_trials=20) 
+
+    print("best params:", study.best_params)
+    print("successfully")
+    return study.best_params

gpbench/method_reg/Cropformer/__init__.py

@@ -0,0 +1,5 @@
+from .Cropformer import Cropformer_reg
+
+Cropformer = Cropformer_reg
+
+__all__ = ["Cropformer","Cropformer_reg"]

gpbench/method_reg/DL_GWAS/DL_GWAS.py

@@ -0,0 +1,186 @@
+import os
+import time
+import psutil
+import argparse
+import random
+import torch
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+from . import DL_GWAS_Hyperparameters
+import keras
+import pynvml
+from keras import layers
+from keras import regularizers
+from keras.models import Model
+from keras.layers import *
+from scipy.stats import pearsonr
+from keras.callbacks import EarlyStopping
+from sklearn.model_selection import KFold, train_test_split
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+gpus = tf.config.experimental.list_physical_devices('GPU')
+if gpus:
+    for gpu in gpus:
+        tf.config.experimental.set_memory_growth(gpu, True)
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Argument parser")
+    parser.add_argument('--methods', type=str, default='DL_GWAS/', help='Random seed')
+    parser.add_argument('--species', type=str, default='')
+    parser.add_argument('--phe', type=str, default='', help='Dataset name')
+    parser.add_argument('--data_dir', type=str, default='../../data/')
+    parser.add_argument('--result_dir', type=str, default='result/')
+
+    parser.add_argument('--epochs', type=int, default=1000, help='Number of training rounds')
+    parser.add_argument('--batch_size', type=int, default=128, help='Batch size')
+    parser.add_argument('--learning_rate', type=float, default=0.01, help='Learning rate')
+    parser.add_argument('--patience', type=int, default=5, help='Patience for early stopping')
+    args = parser.parse_args()
+    return args
+
+def indices_to_one_hot(data,nb_classes):
+	targets = np.array(data).reshape(-1)
+	return np.eye(nb_classes)[targets]
+
+def load_data(args):
+    xData = np.load(os.path.join(args.data_dir, args.species, 'genotype.npz'))["arr_0"]
+    yData = np.load(os.path.join(args.data_dir, args.species, 'phenotype.npz'))["arr_0"]
+    names = np.load(os.path.join(args.data_dir, args.species, 'phenotype.npz'))["arr_1"]
+
+    nsample = xData.shape[0]
+    nsnp = xData.shape[1]
+    print("Number of samples: ", nsample)
+    print("Number of SNPs: ", nsnp)
+    xData = xData.astype(int)
+    arr = np.empty(shape=(nsample, nsnp, 4))
+    xData[xData == -9] = 0
+    for i in range(0, nsample):
+        arr[i] = indices_to_one_hot(pd.to_numeric(xData[i], downcast='signed'), 4)
+    
+    return arr, yData, nsample, nsnp, names
+
+def set_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    
+def get_gpu_mem_by_pid(pid):
+    procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
+    for p in procs:
+        if p.pid == pid:
+            return p.usedGpuMemory / 1024**2
+    return 0.0
+
+def resnet(args, nsnp):
+	
+	inputs = Input(shape=(nsnp,4))
+	
+	x = Conv1D(10,4,padding='same',activation = 'linear',kernel_initializer = 'TruncatedNormal', kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(inputs)
+	x = Conv1D(10,20,padding='same',activation = 'linear', kernel_initializer = 'TruncatedNormal',kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(x)	
+	x = Dropout(0.75)(x)
+	shortcut = Conv1D(10,4,padding='same',activation = 'linear',kernel_initializer = 'TruncatedNormal', kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(inputs)
+	x = layers.add([shortcut,x])
+	x = Conv1D(10,4,padding='same',activation = 'linear',kernel_initializer = 'TruncatedNormal', kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(x)
+	x = Dropout(0.75)(x)
+	x = Flatten()(x)
+	x = Dropout(0.75)(x)
+	outputs = Dense(1,activation = 'linear',bias_regularizer = regularizers.l2(0.01),kernel_initializer = 'TruncatedNormal',name = 'out')(x)
+	model = Model(inputs = inputs,outputs = outputs)
+	model.compile(loss='mean_squared_error',optimizer=keras.optimizers.Adam(learning_rate=args.learning_rate),metrics=['mae'])
+	return model
+
+def isru(x):
+    return x / (tf.sqrt(1 + 0.02 * tf.square(x)))
+	
+def run_nested_cv(args, data, label, nsnp):
+    result_dir = os.path.join(args.result_dir, args.methods + args.species + args.phe)
+    os.makedirs(result_dir, exist_ok=True)
+    print("Starting 10-fold cross-validation...")
+    kf = KFold(n_splits=10, shuffle=True, random_state=42)
+
+    all_mse, all_mae, all_r2, all_pcc = [], [], [], []
+    time_star = time.time()
+    early_stopping = EarlyStopping(monitor='val_loss', patience=args.patience)
+    for fold, (train_index, test_index) in enumerate(kf.split(data)):
+        print(f"Running fold {fold}...")
+        process = psutil.Process(os.getpid())
+        fold_start_time = time.time()
+
+        X_train, X_test = data[train_index], data[test_index]
+        y_train, y_test = label[train_index], label[test_index]
+        
+        X_train_sub, X_valid, y_train_sub, y_valid = train_test_split(X_train, y_train, test_size=0.1, random_state=42)
+
+        model = resnet(args, nsnp = nsnp)
+        model.fit(X_train_sub, y_train_sub, batch_size=args.batch_size, epochs=args.epochs, validation_data=(X_valid, y_valid),callbacks=[early_stopping],shuffle= True, verbose=0)
+        y_pred = model.predict(X_test)
+
+        y_pred = np.asarray(y_pred).flatten().astype(np.float64)
+        y_test = np.asarray(y_test).flatten().astype(np.float64)
+
+        mse = mean_squared_error(y_test, y_pred)
+        r2 = r2_score(y_test, y_pred)
+        mae = mean_absolute_error(y_test, y_pred)
+        pcc, _ = pearsonr(y_test, y_pred)
+
+        all_mse.append(mse)
+        all_r2.append(r2)
+        all_mae.append(mae)
+        all_pcc.append(pcc)
+
+        fold_time = time.time() - fold_start_time
+        fold_gpu_mem = get_gpu_mem_by_pid(os.getpid())
+        fold_cpu_mem = process.memory_info().rss / 1024**2
+        print(f'Fold {fold}:  Corr={pcc:.4f}, MAE={mae:.4f}, MSE={mse:.4f}, R2={r2:.4f}, Time={fold_time:.2f}s, '
+              f'GPU={fold_gpu_mem:.2f}MB, CPU={fold_cpu_mem:.2f}MB')
+     
+        results_df = pd.DataFrame({'Y_test': y_test, 'Y_pred': y_pred})
+        results_df.to_csv(os.path.join(result_dir, f"fold{fold}.csv"), index=False)
+
+    print("\n===== Cross-validation summary =====")
+    print(f"Average PCC: {np.mean(all_pcc):.4f} ± {np.std(all_pcc):.4f}")
+    print(f"Average MAE: {np.mean(all_mae):.4f} ± {np.std(all_mae):.4f}")
+    print(f"Average MSE: {np.mean(all_mse):.4f} ± {np.std(all_mse):.4f}")
+    print(f"Average R2 : {np.mean(all_r2):.4f} ± {np.std(all_r2):.4f}")
+    print(f"Time: {time.time() - time_star:.2f}s")
+
+
+def DL_GWAS_reg():
+    set_seed(42)
+    pynvml.nvmlInit()
+    handle = pynvml.nvmlDeviceGetHandleByIndex(0)
+    args = parse_args()
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    all_species =['Cotton/']
+    
+    for i in range(len(all_species)):
+        args.species = all_species[i]
+        args.device = device  
+        X, Y, nsamples, nsnp, names = load_data(args)
+        for j in range(len(names)):
+            args.phe = names[j]
+            print("starting run " + args.methods + args.species + args.phe)
+            label = Y[:, j]
+            label = np.nan_to_num(label, nan=np.nanmean(label))
+            best_params = DL_GWAS_Hyperparameters.Hyperparameter(X, label, nsnp)
+            args.learning_rate = best_params['learning_rate']
+            args.patience = best_params['patience']
+            args.batch_size = best_params['batch_size']
+            start_time = time.time()
+
+            process = psutil.Process(os.getpid())
+            run_nested_cv(args, data=X, label=label, nsnp = nsnp)
+
+            elapsed_time = time.time() - start_time
+            print(f"running time: {elapsed_time:.2f} s")
+            print("successfully")
+
+
+if __name__ == "__main__":
+    DL_GWAS_reg()

gpbench/method_reg/DL_GWAS/DL_GWAS_Hyperparameters.py

@@ -0,0 +1,125 @@
+import os
+import time
+import psutil
+import random
+import optuna
+import torch
+import numpy as np
+import tensorflow as tf
+import keras
+from keras import layers
+from keras import regularizers
+from keras.models import Model
+from keras.layers import *
+from scipy.stats import pearsonr
+from keras.callbacks import EarlyStopping
+from sklearn.model_selection import KFold, train_test_split
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+from optuna.exceptions import TrialPruned 
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+gpus = tf.config.experimental.list_physical_devices('GPU')
+if gpus:
+    for gpu in gpus:
+        tf.config.experimental.set_memory_growth(gpu, True)
+
+def resnet(nsnp, learning_rate):
+    inputs = Input(shape=(nsnp, 4))
+    x = Conv1D(10,4,padding='same',activation = 'linear',kernel_initializer = 'TruncatedNormal', kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(inputs)
+    x = Conv1D(10,20,padding='same',activation = 'linear', kernel_initializer = 'TruncatedNormal',kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(x)
+    x = Dropout(0.75)(x)
+    shortcut = Conv1D(10,4,padding='same',activation = 'linear',kernel_initializer = 'TruncatedNormal', kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(inputs)
+    x = layers.add([shortcut,x])
+    x = Conv1D(10,4,padding='same',activation = 'linear',kernel_initializer = 'TruncatedNormal', kernel_regularizer=regularizers.l2(0.1),bias_regularizer = regularizers.l2(0.01))(x)
+    x = Dropout(0.75)(x)
+    x = Flatten()(x)
+    x = Dropout(0.75)(x)
+    outputs = Dense(1,activation = 'linear', bias_regularizer = regularizers.l2(0.01),kernel_initializer = 'TruncatedNormal',name = 'out')(x)
+
+    model = Model(inputs = inputs,outputs = outputs)
+    model.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam(learning_rate=learning_rate),metrics=['mae'])
+    
+    return model
+
+def isru(x):
+    return x / (tf.sqrt(1 + 0.02 * tf.square(x)))
+	
+def run_nested_cv_with_early_stopping(data, label, nsnp, learning_rate, batch_size, patience):
+    print("Starting 10-fold cross-validation...")
+    kf = KFold(n_splits=10, shuffle=True, random_state=42)
+    
+    all_mse, all_mae, all_r2, all_pcc = [], [], [], []
+    time_star = time.time()
+    early_stopping = EarlyStopping(monitor='val_loss', patience=patience)
+    for fold, (train_index, test_index) in enumerate(kf.split(data)):
+        print(f"Running fold {fold}...")
+        process = psutil.Process(os.getpid())
+        fold_start_time = time.time()
+
+        X_train, X_test = data[train_index], data[test_index]
+        y_train, y_test = label[train_index], label[test_index]
+        
+        X_train_sub, X_valid, y_train_sub, y_valid = train_test_split(X_train, y_train, test_size=0.1, random_state=42)
+
+        model = resnet(nsnp = nsnp, learning_rate = learning_rate)
+        model.fit(X_train_sub, y_train_sub, batch_size=batch_size, epochs=1000, validation_data=(X_valid, y_valid),callbacks=[early_stopping],shuffle= True, verbose=0)
+        y_pred = model.predict(X_test)
+        
+        y_pred = np.asarray(y_pred).flatten().astype(np.float64)
+        y_test = np.asarray(y_test).flatten().astype(np.float64)
+
+        mse = mean_squared_error(y_test, y_pred)
+        r2 = r2_score(y_test, y_pred)
+        mae = mean_absolute_error(y_test, y_pred)
+        pcc, _ = pearsonr(y_test, y_pred)
+        
+        if np.isnan(pcc):
+            print(f"Fold {fold} resulted in NaN PCC, pruning the trial...")
+            raise TrialPruned()
+
+        all_mse.append(mse)
+        all_r2.append(r2)
+        all_mae.append(mae)
+        all_pcc.append(pcc)
+
+        fold_time = time.time() - fold_start_time
+        fold_cpu_mem = process.memory_info().rss / 1024**2
+        print(f'Fold {fold}:  Corr={pcc:.4f}, MAE={mae:.4f}, MSE={mse:.4f}, R2={r2:.4f}, Time={fold_time:.2f}s, '
+              f'CPU={fold_cpu_mem:.2f}MB')
+    return np.mean(all_pcc) if all_pcc else 0.0
+
+def set_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+def Hyperparameter(data, label, nsnp):
+    set_seed(42)
+    def objective(trial):
+        learning_rate = trial.suggest_float("learning_rate", 1e-4, 0.1)
+        batch_size = trial.suggest_categorical("batch_size", [32, 64, 128])
+        patience = trial.suggest_int("patience", 5, 10)
+        try:
+            corr_score = run_nested_cv_with_early_stopping(
+                data=data,
+                label=label,
+                nsnp=nsnp,
+                learning_rate=learning_rate,
+                batch_size=batch_size,
+                patience=patience
+            )
+        except TrialPruned:
+            return float("-inf")
+        return corr_score
+    
+    study = optuna.create_study(direction="maximize")
+    study.optimize(objective, n_trials=20)
+
+    print("best params:", study.best_params)
+    print("successfully")
+    return study.best_params

gpbench/method_reg/DL_GWAS/__init__.py

@@ -0,0 +1,5 @@
+from .DL_GWAS import DL_GWAS_reg
+
+DL_GWAS = DL_GWAS_reg
+
+__all__ = ["DL_GWAS","DL_GWAS_reg"]