gpbench 1.0.0__py3-none-any.whl

gpbench/method_reg/GEFormer/gMLP.py

@@ -0,0 +1,341 @@
+from random import randrange
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn, einsum
+
+from einops import rearrange, repeat
+from einops.layers.torch import Rearrange, Reduce
+
+# functions
+
+def exists(val):
+    return val is not None
+
+def pair(val):
+    return (val, val) if not isinstance(val, tuple) else val
+
+def dropout_layers(layers, prob_survival):
+    if prob_survival == 1:
+        return layers
+
+    num_layers = len(layers)
+    to_drop = torch.zeros(num_layers).uniform_(0., 1.) > prob_survival
+
+    # make sure at least one layer makes it
+    if all(to_drop):
+        rand_index = randrange(num_layers)
+        to_drop[rand_index] = False
+
+    layers = [layer for (layer, drop) in zip(layers, to_drop) if not drop]
+    return layers
+
+
+# helper classes
+
+class Residual(nn.Module):
+    def __init__(self, fn):
+        super().__init__()
+        self.fn = fn
+
+    def forward(self, x):
+        return self.fn(x) + x
+
+class PreNorm(nn.Module):
+    def __init__(self, dim, fn):
+        super().__init__()
+        self.fn = fn
+        self.norm = nn.LayerNorm(dim)
+
+    def forward(self, x, **kwargs):
+        x = self.norm(x)
+        return self.fn(x, **kwargs)
+
+class Attention(nn.Module):
+    def __init__(self, dim_in, dim_out, dim_inner, causal = False):
+        super().__init__()
+        self.scale = dim_inner ** -0.5
+        self.causal = causal
+
+        self.to_qkv = nn.Linear(dim_in, dim_inner * 3, bias = False)
+        self.to_out = nn.Linear(dim_inner, dim_out)
+
+    def forward(self, x):
+        device = x.device
+        q, k, v = self.to_qkv(x).chunk(3, dim = -1)
+        sim = einsum('b i d, b j d -> b i j', q, k) * self.scale
+
+        if self.causal:
+            mask = torch.ones(sim.shape[-2:], device = device).triu(1).bool()
+            sim.masked_fill_(mask[None, ...], -torch.finfo(q.dtype).max)
+
+        attn = sim.softmax(dim = -1)
+        out = einsum('b i j, b j d -> b i d', attn, v)
+        return self.to_out(out)
+
+class SpatialGatingUnit(nn.Module):
+    def __init__(
+        self,
+        dim,
+        dim_seq,
+        causal = False,
+        act = nn.Identity(),
+        heads = 1,
+        init_eps = 1e-3,
+        circulant_matrix = False
+    ):
+        super().__init__()
+        dim_out = dim // 2
+        self.heads = heads
+        self.causal = causal
+        self.norm = nn.LayerNorm(dim_out)
+
+        self.act = act
+
+        # parameters
+
+        if circulant_matrix:
+            self.circulant_pos_x = nn.Parameter(torch.ones(heads, dim_seq))
+            self.circulant_pos_y = nn.Parameter(torch.ones(heads, dim_seq))
+
+        self.circulant_matrix = circulant_matrix
+        shape = (heads, dim_seq,) if circulant_matrix else (heads, dim_seq, dim_seq)
+        weight = torch.zeros(shape)
+
+        self.weight = nn.Parameter(weight)
+        init_eps /= dim_seq
+        nn.init.uniform_(self.weight, -init_eps, init_eps)
+
+        self.bias = nn.Parameter(torch.ones(heads, dim_seq))
+
+    def forward(self, x, gate_res = None):
+        device, n, h = x.device, x.shape[1], self.heads
+
+        res, gate = x.chunk(2, dim = -1)
+        gate = self.norm(gate)
+
+        weight, bias = self.weight, self.bias
+
+        if self.circulant_matrix:
+            # build the circulant matrix
+
+            dim_seq = weight.shape[-1]
+            weight = F.pad(weight, (0, dim_seq), value = 0)
+            weight = repeat(weight, '... n -> ... (r n)', r = dim_seq)
+            weight = weight[:, :-dim_seq].reshape(h, dim_seq, 2 * dim_seq - 1)
+            weight = weight[:, :, (dim_seq - 1):]
+
+            # give circulant matrix absolute position awareness
+
+            pos_x, pos_y = self.circulant_pos_x, self.circulant_pos_y
+            weight = weight * rearrange(pos_x, 'h i -> h i ()') * rearrange(pos_y, 'h j -> h () j')
+
+        if self.causal:
+            weight, bias = weight[:, :n, :n], bias[:, :n]
+            mask = torch.ones(weight.shape[-2:], device = device).triu_(1).bool()
+            mask = rearrange(mask, 'i j -> () i j')
+            weight = weight.masked_fill(mask, 0.)
+
+        gate = rearrange(gate, 'b n (h d) -> b h n d', h = h)
+
+        gate = einsum('b h n d, h m n -> b h m d', gate, weight)
+        gate = gate + rearrange(bias, 'h n -> () h n ()')
+
+        gate = rearrange(gate, 'b h n d -> b n (h d)')
+
+        if exists(gate_res):
+            gate = gate + gate_res
+
+        return self.act(gate) * res
+
+class gMLPBlock(nn.Module):
+    def __init__(
+        self,
+        *,
+        dim,
+        dim_ff,
+        seq_len,
+        heads = 1,
+        attn_dim = None,
+        causal = False,
+        act = nn.Identity(),
+        circulant_matrix = False
+    ):
+        super().__init__()
+        self.proj_in = nn.Sequential(
+            nn.Linear(dim, dim_ff),
+            nn.GELU()
+        )
+
+        self.attn = Attention(dim, dim_ff // 2, attn_dim, causal) if exists(attn_dim) else None
+
+        self.sgu = SpatialGatingUnit(dim_ff, seq_len, causal, act, heads, circulant_matrix = circulant_matrix)
+        self.proj_out = nn.Linear(dim_ff // 2, dim)
+
+    def forward(self, x):
+        gate_res = self.attn(x) if exists(self.attn) else None
+        x = self.proj_in(x)
+        x = self.sgu(x, gate_res=gate_res)
+        x = self.proj_out(x)
+        return x
+
+
+# main classes
+class gMLPVision(nn.Module):
+    def __init__(
+        self,
+        *,
+        image_size,
+        patch_size,
+        num_classes,
+        dim,
+        depth,
+        snp_len,
+        heads = 1,
+        ff_mult = 4,
+        channels = 1,
+        attn_dim = None,
+        prob_survival = 1.
+    ):
+        super().__init__()
+        assert (dim % heads) == 0, 'dimension must be divisible by number of heads'
+
+        image_height, image_width = pair(image_size)       
+        patch_height, patch_width = pair(patch_size)      
+        #assert (image_height % patch_height) == 0 and (image_width % patch_width) == 0, 'image height and width must be divisible by patch size'
+        #num_patches = (image_height[0] // patch_height[0]) * (image_width[1] // patch_width[1])
+        num_patches = 200 
+        dim_ff = dim * ff_mult
+
+        self.to_patch_embed = nn.Sequential(
+            Rearrange('b c (h p1) (w p2) -> b (h w) (c p1 p2)', p1 = patch_height, p2 = patch_width),
+            nn.Linear(1*snp_len*1, dim) 
+        )
+        
+       
+        self.prob_survival = prob_survival
+
+        self.layers = nn.ModuleList([Residual(PreNorm(dim, gMLPBlock(dim = dim, heads = heads, dim_ff = dim_ff, seq_len = num_patches, attn_dim = attn_dim))) for i in range(depth)])
+
+        self.to_logits = nn.Sequential(
+            nn.LayerNorm(dim),
+            Reduce('b n d -> b d', 'mean'),
+            nn.Linear(dim, num_classes)
+        )
+
+    def forward(self, x):
+        x = self.to_patch_embed(x)
+        layers = self.layers if not self.training else dropout_layers(self.layers, self.prob_survival)
+        x = nn.Sequential(*layers)(x)
+        return self.to_logits(x)
+
+class EarlyStopping:
+    def __init__(self, patience=10, delta=0):
+        self.patience = patience
+        self.delta = delta
+        self.best_score = None
+        self.counter = 0
+        self.early_stop = False
+
+    def __call__(self, score):
+        if self.best_score is None:
+            self.best_score = score
+        elif score < self.best_score + self.delta:
+            self.counter += 1
+            if self.counter >= self.patience:
+                self.early_stop = True
+        else:
+            self.best_score = score
+            self.counter = 0
+
+def exists(val):
+    return val is not None
+class GEFormer(nn.Module):
+    def __init__(self,nsnp):
+        super(GEFormer, self).__init__()
+        self.gmlp = gMLPVision(image_size=(nsnp, 1),
+                               patch_size=(nsnp, 1),
+                               num_classes=126,
+                               dim=126,
+                               depth=1,
+                               snp_len=nsnp
+                               )
+
+        self.MLP = nn.Sequential(
+            nn.Linear(126, 128),
+            nn.LeakyReLU(),
+            nn.Dropout(0.4),
+            nn.Linear(128, 64),
+            nn.LeakyReLU(),
+            nn.Dropout(0.4),
+            nn.Linear(64, 1)
+        )
+        self.numsnp = nsnp
+    def forward(self, x):        
+        x = x.view(x.size(0), 1,self.numsnp, 1) 
+        x = self.gmlp(x)
+        predict = self.MLP(x)
+        return predict
+
+    def train_model(self, train_loader, valid_loader, num_epochs, learning_rate, patience, device):
+        optimizer = torch.optim.Adam(self.parameters(), lr=learning_rate, weight_decay=1e-4)
+        criterion = nn.MSELoss()
+
+        self.to(device)
+
+        best_loss = float('inf')
+        best_state = None
+        trigger_times = 0
+
+        for epoch in range(num_epochs):
+            self.train()
+            train_loss = 0.0
+            for inputs, labels in train_loader:
+                inputs, labels = inputs.to(device), labels.to(device)
+                optimizer.zero_grad()
+                outputs = self(inputs)
+                labels = labels.unsqueeze(1)
+                loss = criterion(outputs, labels)
+                loss.backward()
+                optimizer.step()
+                train_loss += loss.item() * inputs.size(0)
+
+            self.eval()
+            valid_loss = 0.0
+            with torch.no_grad():
+                for inputs, labels in valid_loader:
+                    inputs, labels = inputs.to(device), labels.to(device)
+                    outputs = self(inputs)
+                    labels = labels.unsqueeze(1)
+                    loss = criterion(outputs, labels)
+                    valid_loss += loss.item() * inputs.size(0)
+
+            train_loss /= len(train_loader.dataset)
+            valid_loss /= len(valid_loader.dataset)
+
+            # ---------- Early stopping ----------
+            if valid_loss < best_loss:
+                best_loss = valid_loss
+                best_state = self.state_dict()
+                trigger_times = 0
+            else:
+                trigger_times += 1
+                if trigger_times >= patience:
+                    print(f"Early stopping at epoch {epoch+1}")
+                    break
+
+        if best_state is not None:
+            self.load_state_dict(best_state)
+        return best_loss
+        
+    def predict(self, test_loader):
+        self.eval()
+        y_pred = []
+        with torch.no_grad():
+            for inputs, _ in test_loader:
+                outputs = self(inputs)  
+                y_pred.append(outputs.cpu().numpy())
+        y_pred = np.concatenate(y_pred, axis=0)
+        y_pred = np.squeeze(y_pred)
+        return y_pred
+

gpbench/method_reg/LightGBM/LightGBM.py

@@ -0,0 +1,237 @@
+import os
+import time
+import psutil
+import argparse
+import random
+import torch
+import pandas as pd
+import numpy as np
+import lightgbm as lgb
+from sklearn.model_selection import KFold
+from scipy.stats import pearsonr
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+import subprocess
+import threading
+import queue
+from . import LightGBM_Hyperparameters
+
+class GPUMonitor:
+    def __init__(self, gpu_id=0, interval=0.5):
+        self.gpu_id = gpu_id
+        self.interval = interval
+        self.max_memory = 0
+        self.current_memory = 0
+        self.monitoring = False
+        self.pid = os.getpid()
+        self.queue = queue.Queue()
+    
+    def _get_gpu_memory_by_pid(self):
+        try:
+            result = subprocess.check_output([
+                'nvidia-smi', 
+                '--query-compute-apps=pid,used_memory,gpu_bus_id',
+                '--format=csv,nounits,noheader'
+            ], timeout=5)
+            
+            lines = result.decode('utf-8').strip().split('\n')
+            for line in lines:
+                if not line.strip():
+                    continue
+                parts = line.split(',')
+                if len(parts) >= 2:
+                    pid = int(parts[0].strip())
+                    if pid == self.pid:
+                        mem_str = parts[1].strip()
+                        mem_value = ''.join(filter(str.isdigit, mem_str))
+                        if mem_value:
+                            return int(mem_value)
+            return 0
+        except Exception as e:
+            print(f"GPU memory query error: {e}")
+            return 0
+    
+    def _monitor_loop(self):
+        while self.monitoring:
+            try:
+                mem = self._get_gpu_memory_by_pid()
+                self.current_memory = mem
+                if mem > self.max_memory:
+                    self.max_memory = mem
+                time.sleep(self.interval)
+            except Exception as e:
+                print(f"Monitor loop error: {e}")
+                break
+    
+    def start(self):
+        self.max_memory = 0
+        self.current_memory = 0
+        self.monitoring = True
+        self.thread = threading.Thread(target=self._monitor_loop)
+        self.thread.daemon = True
+        self.thread.start()
+    
+    def stop(self):
+        self.monitoring = False
+        if hasattr(self, 'thread'):
+            self.thread.join(timeout=2)
+        return self.max_memory
+
+gpu_monitor = GPUMonitor(gpu_id=0, interval=0.2)
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="LightGBM GPU Benchmark")
+    parser.add_argument('--methods', type=str, default='LightGBM/', help='Method name')
+    parser.add_argument('--species', type=str, default='', help='Dataset name')
+    parser.add_argument('--phe', type=str, default='', help='Phenotype')
+    parser.add_argument('--data_dir', type=str, default='../../data/')
+    parser.add_argument('--result_dir', type=str, default='result/')
+    
+    parser.add_argument('--learning_rate', type=float, default=0.01)
+    parser.add_argument('--num_leaves', type=int, default=10)
+    parser.add_argument('--min_data_in_leaf', type=int, default=1)
+    parser.add_argument('--max_depth', type=int, default=1)
+    parser.add_argument('--lambda_l1', type=float, default=0.1)
+    parser.add_argument('--lambda_l2', type=float, default=0.1)
+    parser.add_argument('--min_gain_to_split', type=float, default=0.1)
+    parser.add_argument('--feature_fraction', type=float, default=0.9)
+    parser.add_argument('--bagging_fraction', type=float, default=0.9)
+    parser.add_argument('--bagging_freq', type=int, default=1)
+    parser.add_argument('--num_boost_round', type=int, default=100)
+    parser.add_argument('--objective', type=str, default='regression')
+    parser.add_argument('--device_type', type=str, default='gpu')
+    parser.add_argument('--early_stopping_rounds', type=int, default=50)
+    return parser.parse_args()
+
+def load_data(args):
+    xData = np.load(os.path.join(args.data_dir, args.species, 'genotype.npz'))["arr_0"]
+    yData = np.load(os.path.join(args.data_dir, args.species, 'phenotype.npz'))["arr_0"]
+    names = np.load(os.path.join(args.data_dir, args.species, 'phenotype.npz'))["arr_1"]
+
+    nsample = xData.shape[0]
+    nsnp = xData.shape[1]
+    print(f"Number of samples: {nsample}, SNPs: {nsnp}")
+    return xData, yData, nsample, nsnp, names
+
+def set_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+def run_nested_cv(args, data, label):
+    result_dir = os.path.join(args.result_dir, args.methods + args.species + args.phe)
+    os.makedirs(result_dir, exist_ok=True)
+    print("Starting 10-fold cross-validation...")
+    kf = KFold(n_splits=10, shuffle=True, random_state=42)
+    all_mse, all_mae, all_r2, all_pcc = [], [], [], []
+    time_star = time.time()
+
+    params = {
+        'objective': args.objective,
+        'metric': 'rmse',
+        'learning_rate': args.learning_rate,
+        'num_leaves': args.num_leaves,
+        'min_data_in_leaf': args.min_data_in_leaf,
+        'max_depth': args.max_depth,
+        'lambda_l1': args.lambda_l1,
+        'lambda_l2': args.lambda_l2,
+        'min_gain_to_split': args.min_gain_to_split,
+        'feature_fraction': args.feature_fraction,
+        'bagging_fraction': args.bagging_fraction,
+        'bagging_freq': args.bagging_freq,
+        'num_boost_round': args.num_boost_round,
+        'device_type': 'gpu',
+        'gpu_platform_id': 0,
+        'gpu_device_id': 0,
+        'num_threads': 8,
+        'verbose': -1
+    }
+
+    for fold, (train_index, test_index) in enumerate(kf.split(data)):
+        print(f"\n===== Running fold {fold} =====")
+        process = psutil.Process(os.getpid())
+        fold_start_time = time.time()
+
+        gpu_monitor.start()
+        time.sleep(0.5)
+
+        cpu_mem_before = process.memory_info().rss / 1024**2
+
+        X_train, X_test = data[train_index], data[test_index]
+        y_train, y_test = label[train_index], label[test_index]
+
+        train_set = lgb.Dataset(X_train, label=y_train)
+        test_set = lgb.Dataset(X_test, label=y_test)
+
+        model = lgb.train(
+            params,
+            train_set,
+            num_boost_round=args.num_boost_round,
+            valid_sets=[test_set]
+        )
+
+        y_pred = model.predict(X_test)
+        mse = mean_squared_error(y_test, y_pred)
+        r2 = r2_score(y_test, y_pred)
+        mae = mean_absolute_error(y_test, y_pred)
+        pcc, _ = pearsonr(y_test, y_pred)
+
+        all_mse.append(mse)
+        all_r2.append(r2)
+        all_mae.append(mae)
+        all_pcc.append(pcc)
+
+        fold_time = time.time() - fold_start_time
+        fold_gpu_mem = gpu_monitor.stop()   
+        fold_cpu_mem = process.memory_info().rss / 1024**2
+        print(f'Fold {fold}:  Corr={pcc:.4f}, MAE={mae:.4f}, MSE={mse:.4f}, R2={r2:.4f}, Time={fold_time:.2f}s, '
+              f'GPU={fold_gpu_mem:.2f}MB, CPU={fold_cpu_mem:.2f}MB')
+
+        results_df = pd.DataFrame({'Y_test': y_test, 'Y_pred': y_pred})
+        results_df.to_csv(os.path.join(result_dir, f"fold{fold}.csv"), index=False)
+
+    print("\n===== Cross-validation summary =====")
+    print(f"Average PCC: {np.mean(all_pcc):.4f} ± {np.std(all_pcc):.4f}")
+    print(f"Average MAE: {np.mean(all_mae):.4f} ± {np.std(all_mae):.4f}")
+    print(f"Average MSE: {np.mean(all_mse):.4f} ± {np.std(all_mse):.4f}")
+    print(f"Average R2 : {np.mean(all_r2):.4f} ± {np.std(all_r2):.4f}")
+    print(f"Total Time : {time.time() - time_star:.2f}s")
+
+
+def LightGBM_reg():
+    set_seed(42)
+    args = parse_args()
+    all_species =['Cotton/']
+    
+    for i in range(len(all_species)):
+        args.species = all_species[i]
+        X, Y, nsamples, nsnp, names = load_data(args)
+        for j in range(len(names)):
+            args.phe = names[j]
+            print(f"Starting run: {args.methods}{args.species}{args.phe}")
+            label = Y[:, j]
+            label = np.nan_to_num(label, nan=np.nanmean(label))
+
+            best_params = LightGBM_Hyperparameters.Hyperparameter(X, label)
+            args.learning_rate = best_params['learning_rate']
+            args.num_leaves = best_params['num_leaves']
+            args.min_data_in_leaf = best_params['min_data_in_leaf']
+            args.max_depth = best_params['max_depth']
+            args.lambda_l1 = best_params['lambda_l1']
+            args.lambda_l2 = best_params['lambda_l2']
+            args.min_gain_to_split = best_params['min_gain_to_split']
+            args.feature_fraction = best_params['feature_fraction']
+            args.bagging_fraction = best_params['bagging_fraction']
+            args.bagging_freq = best_params['bagging_freq']
+            start_time = time.time()
+            run_nested_cv(args, data=X, label=label)
+            elapsed_time = time.time() - start_time
+
+            print(f"running time: {elapsed_time:.2f} s")
+            print("✅ Successfully finished.\n")
+
+
+if __name__ == "__main__":
+    LightGBM_reg()

gpbench/method_reg/LightGBM/LightGBM_Hyperparameters.py

@@ -0,0 +1,77 @@
+import random
+import torch
+import numpy as np
+import lightgbm as lgb
+import optuna
+from sklearn.model_selection import KFold
+from scipy.stats import pearsonr
+
+
+def set_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+
+def run_nested_cv(data, label, params):
+    print("Starting 10-fold cross-validation...")
+    kf = KFold(n_splits=10, shuffle=True, random_state=42)
+    all_corr = []
+
+    for train_idx, test_idx in kf.split(data):
+        X_train, X_test = data[train_idx], data[test_idx]
+        y_train, y_test = label[train_idx], label[test_idx]
+
+        train_set = lgb.Dataset(X_train, label=y_train)
+        test_set = lgb.Dataset(X_test, label=y_test)
+
+        model = lgb.train(
+            params,
+            train_set,
+            valid_sets=[test_set],
+            num_boost_round=100,
+        )
+
+        y_pred = model.predict(X_test)
+        corr, _ = pearsonr(y_test, y_pred)
+        all_corr.append(corr)
+    return np.mean(all_corr)
+
+
+def Hyperparameter(X, label):
+    set_seed(42)
+    torch.cuda.empty_cache()
+
+    def objective(trial):
+        params = {
+        'objective': 'regression',
+        'metric': 'rmse',
+        'learning_rate': trial.suggest_float('learning_rate', 1e-3, 0.2, log=True),
+        'num_leaves': trial.suggest_int('num_leaves', 15, 255),
+        'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 100),
+        'max_depth': trial.suggest_int('max_depth', 3, 10),
+        'lambda_l1': trial.suggest_float('lambda_l1', 0.0, 5.0),
+        'lambda_l2': trial.suggest_float('lambda_l2', 0.0, 5.0),
+        'min_gain_to_split': trial.suggest_float('min_gain_to_split', 0.0, 5.0),
+        'feature_fraction': trial.suggest_float('feature_fraction', 0.6, 1.0),
+        'bagging_fraction': trial.suggest_float('bagging_fraction', 0.6, 1.0),
+        'bagging_freq': trial.suggest_int('bagging_freq', 0, 10),
+        'num_boost_round':trial.suggest_int('num_boost_round', 100, 1000),
+        'device_type': 'gpu',
+        'gpu_device_id': 1,
+        'num_threads': 8,
+        'verbose':-1, 
+        }
+
+        corr_scores = run_nested_cv(data=X, label=label, params=params)
+        return np.mean(corr_scores) 
+
+    study = optuna.create_study(direction="maximize")
+    study.optimize(objective, n_trials=20) 
+
+    print("best params:", study.best_params)
+    print("successfully")
+    return study.best_params

gpbench/method_reg/LightGBM/__init__.py

@@ -0,0 +1,5 @@
+from .LightGBM import LightGBM_reg
+
+LightGBM = LightGBM_reg
+
+__all__ = ["LightGBM","LightGBM_reg"]