ScreenPro2 0.5.0__py3-none-any.whl

screenpro/phenoscore/_annotate.py

@@ -0,0 +1,122 @@
+"""
+annotate module
+"""
+
+import numpy as np
+import pandas as pd
+
+hit_dict = {
+    'gamma':{
+        'up_hit':'up_hit',
+        'down_hit':'essential_hit'
+    },
+    'tau':{
+        'up_hit':'up_hit', 
+        'down_hit':'down_hit'
+    },
+    'rho':{
+        'up_hit':'resistance_hit', 
+        'down_hit':'sensitivity_hit'
+    },
+    'delta':{
+        'up_hit':'up_hit',
+        'down_hit':'down_hit'
+    }
+}
+
+
+def getCombinedScore(df_in, score_col='score', pvalue_col='pvalue', target_col='target', ctrl_label='negative_control'):
+    """
+    Calculate the combined score column based on the given phenotypic scores and p-values.
+    Combined score is calculated as:
+
+        $combined\_score = \frac{score}{pseudo\_sd} \times -\log_{10}(pvalue)$
+
+    Parameters:
+        df_in (pandas.DataFrame): The input DataFrame.
+        score_col (str): The column name for the individual scores. Default is 'score'.
+        pvalue_col (str): The column name for the p-values. Default is 'pvalue'.
+        target_col (str): The column name for the target variable. Default is 'target'.
+        combined_score_col (str): The column name for the combined scores. Default is 'combined_score'.
+        ctrl_label (str): The label for the control group. Default is 'control'.
+    
+    Returns:
+        pandas.Series: The calculated combined score column.
+    """
+    # make a copy of input dataframe
+    df = df_in.copy()
+
+    for col in [score_col, pvalue_col, target_col]:
+        if col not in df.columns:
+            raise ValueError(f'Column "{col}" not found in the input DataFrame.')
+    
+    # calculate pseudo_sd
+    pseudo_sd = df[df[target_col].eq(ctrl_label)][score_col].tolist()
+    pseudo_sd = np.std(pseudo_sd)
+
+    # calculate combined score
+    return df[score_col]/pseudo_sd * -np.log10(df[pvalue_col])
+
+
+def annotateScoreTable(df_in, up_hit, down_hit, threshold, score_col='score', pvalue_col='pvalue', target_col='target', ctrl_label='negative_control'):
+    """
+    Annotate the given score tabel 
+    
+
+    Parameters:
+        df_in (pd.DataFrame): score dataframe
+        up_hit (str): up hit label
+        down_hit (str): down hit label
+        threshold (int): threshold value
+        score_col (str): score column name. Default is 'score'.
+        target_col (str): column name for the target variable. Default is 'target'.
+        pvalue_col (str): pvalue column name. Default is 'pvalue'.
+        ctrl_label (str): control label value. Default is 'negative_control'.
+    
+    Returns:
+        pd.DataFrame: annotated score dataframe
+    """
+    # make a copy of input dataframe
+    df = df_in.copy()
+
+    for col in [score_col, pvalue_col, target_col]:
+        if col not in df.columns:
+            raise ValueError(f'Column "{col}" not found in the input DataFrame.')
+
+    df[score_col] = df[score_col].astype(float)
+    df[pvalue_col] = df[pvalue_col].astype(float)
+
+    # add combined score column
+    df['combined_score'] = getCombinedScore(
+        df,
+        score_col=score_col, pvalue_col=pvalue_col, target_col=target_col,
+        ctrl_label=ctrl_label)
+
+    # add label column
+    df['label'] = '.'
+
+    # annotate hits: up
+    df.loc[
+        (df[score_col] > 0) & (~df[target_col].eq(ctrl_label)) &
+        (df['combined_score'] >= threshold), 'label'
+    ] = up_hit
+
+    # annotate hits: down
+    df.loc[
+        (df[score_col] < 0) & (~df[target_col].eq(ctrl_label)) &
+        (df['combined_score'] <= -threshold), 'label'
+    ] = down_hit
+
+    # annotate control
+    df.loc[df[target_col].eq(ctrl_label), 'label'] = ctrl_label
+
+    # annotate non-hit
+    df.loc[df['label'] == '.', 'label'] = 'target_non_hit'
+
+    # reorder factors
+    df['label'] = pd.Categorical(
+        df['label'],
+        categories=[down_hit, up_hit, ctrl_label, 'target_non_hit']
+    )
+
+    return df

screenpro/phenoscore/delta.py

@@ -0,0 +1,375 @@
+"""
+delta module
+"""
+import numpy as np
+import anndata as ad
+import pandas as pd
+
+from .phenostat import (
+    matrixStat, multipleTestsCorrection
+)
+
+
+### Key functions for calculating delta phenotype score
+
+def compareByReplicates(adata, df_cond_ref, df_cond_test, var_names='target', test='ttest', ctrl_label='negative_control', growth_rate=1, filter_type='mean', filter_threshold=40):
+    """Calculate phenotype score and p-values comparing `cond_test` vs `cond_ref`.
+
+    In this function, the phenotype calculation is done by comparing multiple replicates of `cond_test` vs `cond_ref`.
+
+    Args:
+        adata (AnnData): AnnData object
+        df_cond_ref (pd.DataFrame): dataframe of condition reference
+        df_cond_test (pd.DataFrame): dataframe of condition test
+        var_names (str): variable names to use as index in the result dataframe
+        test (str): test to use for calculating p-value ('MW': Mann-Whitney U rank; 'ttest' : t-test)
+        ctrl_label (str): control label, default is 'negative_control'
+        growth_rate (int): growth rate
+        filter_type (str): filter type to apply to low counts ('mean', 'both', 'either')
+        filter_threshold (int): filter threshold for low counts (default is 40)
+    
+    Returns:
+        pd.DataFrame: result dataframe
+    """
+    adat = adata.copy()
+
+    # apply NA to low counts
+    df_cond_ref, df_cond_test = applyNAtoLowCounts(
+        df_cond_ref=df_cond_ref, df_cond_test=df_cond_test,
+        filter_type=filter_type, 
+        filter_threshold=filter_threshold
+    )
+
+    # convert to numpy arrays
+    x = df_cond_ref.to_numpy()
+    y = df_cond_test.to_numpy()
+
+    # get control values
+    x_ctrl = df_cond_ref[adat.var.targetType.eq(ctrl_label)].dropna().to_numpy()
+    y_ctrl = df_cond_test[adat.var.targetType.eq(ctrl_label)].dropna().to_numpy()
+
+    # calculate phenotype scores
+    scores = calculateDelta(
+        x = x, y = y, 
+        x_ctrl = x_ctrl, y_ctrl = y_ctrl, 
+        growth_rate = growth_rate,
+    )
+
+    # average scores across replicates
+    scores = [np.mean(s) for s in scores]
+    
+    # compute p-value
+    p_values = matrixStat(x, y, test=test, level = 'col')
+
+    # get adjusted p-values
+    adj_p_values = multipleTestsCorrection(p_values)
+
+    # get target information            
+    targets_df = adat.var[var_names].copy()
+
+    # combine results into a dataframe
+    result = pd.concat([
+        pd.Series(scores, index=adat.var.index, name='score'),
+        pd.Series(p_values, index=adat.var.index, name=f'{test} pvalue'),
+        pd.Series(adj_p_values, index=adat.var.index, name='BH adj_pvalue'),
+    ], axis=1)
+    
+    # add targets information 
+    result = pd.concat([targets_df, result], axis=1)
+
+    return result
+
+
+def compareByTargetGroup(adata, df_cond_ref, df_cond_test, keep_top_n, var_names='target', test='ttest', ctrl_label='negative_control', growth_rate=1, filter_type='mean', filter_threshold=40):
+    """Calculate phenotype score and p-values comparing `cond_test` vs `cond_ref`.
+
+    In this function, the phenotype calculation is done by comparing groups of 
+    guide elements (e.g. sgRNAs) that target the same gene or groups of pseudogene (i.e.
+    subsampled groups of non-targeting control elements) between `cond_test` vs `cond_ref`.
+
+    Args:
+        adata (AnnData): AnnData object
+        df_cond_ref (pd.DataFrame): dataframe of condition reference
+        df_cond_test (pd.DataFrame): dataframe of condition test
+        keep_top_n (int): number of top guide elements to keep
+        var_names (str): variable names to use as index in the result dataframe
+        test (str): test to use for calculating p-value ('MW': Mann-Whitney U rank; 'ttest' : t-test)
+        ctrl_label (str): control label, default is 'negative_control'
+        growth_rate (int): growth rate
+        filter_type (str): filter type to apply to low counts ('mean', 'both', 'either')
+        filter_threshold (int): filter threshold for low counts (default is 40)
+        
+    Returns:
+        pd.DataFrame: result dataframe
+    """
+
+    adat = adata.copy()
+
+    # apply NA to low counts
+    df_cond_ref, df_cond_test = applyNAtoLowCounts(
+        df_cond_ref=df_cond_ref, df_cond_test=df_cond_test,
+        filter_type=filter_type, 
+        filter_threshold=filter_threshold
+    )
+
+    # get control values
+    x_ctrl = df_cond_ref[adat.var.targetType.eq(ctrl_label)].dropna().to_numpy()
+    y_ctrl = df_cond_test[adat.var.targetType.eq(ctrl_label)].dropna().to_numpy()
+
+    targets = []
+    scores = []
+    p_values = []
+    target_sizes = []
+
+    # group by target genes or pseudogenes to aggregate counts for score calculation
+    for target_name, target_group in adat.var.groupby(var_names):
+
+        # calculate phenotype scores and p-values for each target group
+        target_score, target_p_value, target_size  = scoreTargetGroup(
+            target_group=target_group, 
+            df_cond_ref=df_cond_ref, 
+            df_cond_test=df_cond_test,
+            x_ctrl=x_ctrl, y_ctrl=y_ctrl, 
+            test=test, growth_rate=growth_rate, 
+            keep_top_n=keep_top_n
+        )
+        
+        scores.append(target_score)
+        p_values.append(target_p_value)
+        targets.append(target_name)
+        target_sizes.append(target_size)
+
+    # average scores across replicates
+    scores = [np.mean(s) for s in scores]
+
+    # get adjusted p-values
+    adj_p_values = multipleTestsCorrection(np.array(p_values))
+
+    # get target information
+    if type(var_names) == str:
+        targets_df = pd.DataFrame(targets, columns=[var_names])
+    elif type(var_names) == list:
+        targets_df = pd.DataFrame(targets, columns=var_names)
+
+    # combine results into a dataframe
+    result = pd.concat([
+        pd.Series(scores, name='score', dtype=float),
+        pd.Series(p_values, name=f'{test} pvalue', dtype=float),
+        pd.Series(adj_p_values, name='BH adj_pvalue', dtype=float),
+        pd.Series(target_sizes, name='number_of_guide_elements', dtype=int),
+    ], axis=1)
+
+    # add targets information
+    result = pd.concat([targets_df, result], axis=1)
+
+    # set index to var_names 
+    if type(var_names) == list and len(var_names) > 1:
+        result.index = result[var_names].agg('-'.join, axis=1)
+    else:
+        result.index = result[var_names]
+
+    return result
+
+
+def getPhenotypeData(adata, score_tag, cond_ref, cond_test, growth_rate_reps=None, ctrl_label='negative_control'):
+    """Calculate phenotype score for each pair of replicates
+
+    Args:
+        adata (AnnData): AnnData object
+        score_tag (str): score tag. e.g. 'delta', 'gamma', 'tau', 'rho'.
+        cond_ref (str): condition reference
+        cond_test (str): condition test
+        growth_rate_reps (dict): growth rate for each replicate. Key is replicate number, value is growth rate.
+        ctrl_label (str): control label, default is 'negative_control'
+    """
+    score_name = f'{score_tag}:{cond_test}_vs_{cond_ref}'
+
+    adat = adata.copy()
+
+    adat_ctrl = adat[:, adat.var.targetType.eq(ctrl_label)].copy()
+    
+    results = {}
+    
+    if growth_rate_reps is None:
+        growth_rate_reps = dict([(replicate, 1) for replicate in adat.obs.replicate.unique()])
+    
+    for replicate in adat.obs.replicate.unique():
+        x=adat[adat.obs.query(f'condition == "{cond_ref}" & replicate == {str(replicate)}').index].X.T
+        y=adat[adat.obs.query(f'condition == "{cond_test}" & replicate == {str(replicate)}').index].X.T
+    
+        x_ctrl=adat_ctrl[adat_ctrl.obs.query(f'condition == "{cond_ref}" & replicate == {str(replicate)}').index].X.T
+        y_ctrl=adat_ctrl[adat_ctrl.obs.query(f'condition == "{cond_test}" & replicate == {str(replicate)}').index].X.T
+    
+        res = calculateDelta(
+            x=x,y=y,x_ctrl=x_ctrl,y_ctrl=y_ctrl,
+            growth_rate=growth_rate_reps[replicate],
+        )
+    
+        results.update({f'{score_name}::replicate_{replicate}': res.reshape(-1)})
+    
+    out = ad.AnnData(
+        pd.DataFrame(results, index=adat.var.index).T
+    )
+
+    return out
+
+
+def calculateDelta(x, y, x_ctrl, y_ctrl, growth_rate):
+    """Calculate phenotype score normalized by negative control and growth rate.
+    
+    Args:
+        x (np.array): array of values
+        y (np.array): array of values
+        x_ctrl (np.array): array of values
+        y_ctrl (np.array): array of values
+        growth_rate (int): growth rate
+    
+    Returns:
+        np.array: array of scores
+    """
+    # calculate control median and std 
+    ctrl_median = np.median(
+        calculateLog2e(x=x_ctrl, y=y_ctrl), 
+        axis=0 # for each individual sample (i.e. replicate)
+    )
+
+    # calculate log2e (i.e. log2 fold change enrichment y / x)
+    log2e = calculateLog2e(x=x, y=y)
+
+    # calculate delta score normalized by control median and growth rate
+    delta = (log2e - ctrl_median) / growth_rate
+    
+    return delta
+
+
+### Utility functions
+
+def calculateLog2e(x, y):
+    return np.log2(y) - np.log2(x)
+
+
+def averageBestN(scores, numToAverage):
+    # Sort and find top n guide per target, see #18
+    return np.mean(sorted(scores, key=abs, reverse=True)[:numToAverage])
+
+
+def getBestTargetByTSS(score_df,target_col,pvalue_col):
+    """
+    collapse the gene-transcript indices into a single score for a gene by best p-value
+    """
+    return score_df.dropna().groupby(target_col).apply(
+        lambda x: x.loc[x[pvalue_col].idxmin()]
+    )
+
+
+def scoreTargetGroup(target_group, df_cond_ref, df_cond_test, x_ctrl, y_ctrl, test='ttest', growth_rate=1, keep_top_n=None):
+    # select target group and convert to numpy arrays
+    x = df_cond_ref.loc[target_group.index,:].dropna().to_numpy()
+    y = df_cond_test.loc[target_group.index,:].dropna().to_numpy()
+
+    # calculate phenotype scores
+    target_scores = calculateDelta(
+        x = x, y = y, 
+        x_ctrl = x_ctrl, y_ctrl = y_ctrl, 
+        growth_rate = growth_rate,
+    )
+
+    # get target size
+    target_size = target_scores.shape[0] # number of guide elements in the target group
+    
+    if target_size == 0:
+        target_score = np.full(target_scores.shape[1], np.nan)
+    
+    elif (keep_top_n is None or keep_top_n is False) or target_size <= keep_top_n:
+        # average scores across guides
+        target_score = np.mean(target_scores, axis=0)
+
+    elif keep_top_n > 0 or target_size > keep_top_n:
+        # get top n scores per target
+        target_score = np.apply_along_axis(averageBestN, axis=0, arr=target_scores, numToAverage=keep_top_n)
+        target_size = keep_top_n # update target size to keep_top_n
+    
+    else:
+        raise ValueError(f'Invalid value for keep_top_n: {keep_top_n}')
+
+    # compute p-value
+    target_p_value = matrixStat(x, y, test=test, level='all')
+    
+    return target_score, target_p_value, target_size
+
+
+def generatePseudoGeneAnnData(adata, num_pseudogenes='auto', pseudogene_size='auto', ctrl_label='negative_control'):
+    """Generate pseudogenes from negative control elements in the library.
+
+    Args:
+        adata (AnnData): AnnData object
+        num_pseudogenes (int): number of pseudogenes to generate
+        pseudogene_size (int): number of sgRNA elements in each pseudogene
+        ctrl_label (str): control label, default is 'negative_control'
+    
+    Returns:
+        AnnData: AnnData object with pseudogenes
+    """
+    #TODO: check for `target` and `targetType` columns in adata.var
+    #TODO: add input arg to replace "target" and name it `target_col` or something similar
+    
+    if pseudogene_size == 'auto':
+        # sgRNA elements / target in the library
+        pseudogene_size = int(adata.var[~adata.var.targetType.eq(ctrl_label)].groupby('target').size().mean())
+
+    if num_pseudogenes == 'auto':
+        # approx number of target in the library
+        num_pseudogenes = len(adata.var.loc[~adata.var.targetType.eq(ctrl_label),'target'].unique()) * pseudogene_size
+    
+    adata_ctrl = adata[:,adata.var.targetType.eq(ctrl_label)].copy()
+    ctrl_elements = adata_ctrl.var.index.to_list()
+    
+    adata_pseudo_list = []
+    pseudo_source_sgrna = []
+
+    for pseudo_num in range(0, num_pseudogenes, pseudogene_size):
+        pseudo_elements = np.random.choice(ctrl_elements, pseudogene_size, replace=False)
+        pseudo_labels = [f'pseudo_{pseudo_num}_{i}' for i in range(1,pseudogene_size+1)]
+        
+        adata_pseudo = ad.AnnData(
+            X = adata_ctrl.X[:,adata_ctrl.var.index.isin(pseudo_elements)],
+            obs = adata_ctrl.obs   
+        )
+        adata_pseudo.var_names = pseudo_labels
+        adata_pseudo_list.append(adata_pseudo)
+        
+        for element in pseudo_elements: 
+            pseudo_source_sgrna.append(element)
+        
+    out = ad.concat(adata_pseudo_list, axis=1)
+    out.var['target'] = out.var.index.str.split('_').str[:-1].str.join('_')
+    out.var['targetType'] = ctrl_label
+    out.var['source'] = pseudo_source_sgrna
+    out.obs = adata_ctrl.obs.copy()
+    
+    return out
+
+
+def applyNAtoLowCounts(df_cond_ref, df_cond_test, filter_type, filter_threshold):
+    # more flexible read filtering by adding np.nan scores and/or pvalues to low count rows
+    # keep row if either both/all columns are above threshold, or if either/any column is
+    # in other words, mask if any column is below threshold or only if all columns are below
+    # source https://github.com/mhorlbeck/ScreenProcessing/blob/master/process_experiments.py#L464C1-L478C1
+
+    df = pd.concat({'ref':df_cond_ref, 'test':df_cond_test},axis=1)
+    
+    if filter_type == 'mean':
+        filter = df.apply(
+            lambda row: np.mean(row) < filter_threshold, axis=1)
+    elif filter_type == 'both' or filter_type == 'all':
+        filter = df.apply(
+            lambda row: min(row) < filter_threshold, axis=1)
+    elif filter_type == 'either' or filter_type == 'any':
+        filter = df.apply(
+            lambda row: max(row) < filter_threshold, axis=1)
+    else:
+        raise ValueError('filter type not recognized or not implemented')
+
+    df.loc[filter, :] = np.nan
+
+    return df['ref'], df['test']

screenpro/phenoscore/deseq.py

@@ -0,0 +1,57 @@
+"""
+deseq module: adapt pyDESeq2 for use in ScreenPro2 package
+"""
+
+import numpy as np
+import pandas as pd
+import anndata as ad
+import os, contextlib
+
+from pydeseq2.dds import DeseqDataSet
+from pydeseq2.default_inference import DefaultInference
+from pydeseq2.ds import DeseqStats
+
+
+def runDESeq(adata, design, n_cpus=8,quiet=False):
+
+    inference = DefaultInference(n_cpus=n_cpus)
+
+    print(f'\tcreating `dds` object...')
+    
+    dds = DeseqDataSet(
+        counts=adata.to_df().astype(int),
+        metadata=adata.obs,
+        design_factors=design,  # compare samples based on the "condition"
+        refit_cooks=True,
+        inference=inference,
+        quiet=quiet
+    )
+    dds.var = adata.var.copy()
+
+    dds.deseq2()
+
+    return dds
+
+
+def extractDESeqResults(dds, design, ref_level, tested_level, n_cpus=8, quiet=False):
+
+    inference = DefaultInference(n_cpus=n_cpus)
+
+    result_name = f'{tested_level}_vs_{ref_level}'
+
+    print(f'\t{tested_level}_vs_{ref_level}')
+
+    stat_res = DeseqStats(
+        dds, 
+        contrast=[design, tested_level, ref_level], 
+        inference=inference,
+        quiet=quiet
+    )
+    
+    with open(os.devnull, 'w') as devnull:
+        with contextlib.redirect_stdout(devnull):
+            stat_res.summary()
+
+    results = pd.concat([dds.var['target'], stat_res.results_df], axis=1)
+
+    return result_name, results

screenpro/phenoscore/evaluate.py

@@ -0,0 +1,66 @@
+## Copyright (c) 2022-2024 ScreenPro2 Development Team.
+## All rights reserved.
+## Gilbart Lab, UCSF / Arc Institute.
+## Multi-Omics Tech Center, Arc Insititue.
+##
+## courtesy Tyler Fair (@tdfair), M. Horlbeck, (@mhorlbeck)
+
+'''
+evaluate module: evaluate essentiality detection performance
+'''
+
+import numpy as np
+from sklearn import metrics
+from sklearn.metrics import precision_recall_curve
+import matplotlib.lines as mlines
+
+
+def calcROC(df_in, essential, nonessential, score_col, target_col='target', verbose=False):
+    df = df_in.copy()
+    df[target_col] = df[target_col].str.split('-').str[0]
+    
+    # AUC-ROC
+    df['DepMap'] = np.nan
+    df.loc[df[target_col].isin(essential),'DepMap'] = 'essential'
+    df.loc[df[target_col].isin(nonessential),'DepMap'] = 'non_essential'
+    
+    y_true = df[df['DepMap'].notna()]['DepMap']
+    y_scores = df[df['DepMap'].notna()][score_col]
+    
+    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_scores, pos_label='non_essential')
+    
+    if verbose: print('ROC-AUC score:', metrics.roc_auc_score(y_true, y_scores))
+    
+    return fpr, tpr
+
+
+def calcPR(df_in, truePos, trueNeg, score_col, target_col='target', ascending=True, verbose=False):
+
+    df = df_in.copy()
+
+    scoreList = df.set_index(target_col)[score_col]
+
+    truePos = truePos.intersection(scoreList.index)
+    trueNeg = trueNeg.intersection(scoreList.index)
+    
+    cumulativeTup = [(0,1,np.nan)]
+    cumulativeTP = 0.0
+    cumulativeFP = 0.0
+    
+    tup_cross95 = None
+    
+    for gene, fold_change in scoreList.sort_values(inplace=False, ascending=ascending).items():
+        if gene in truePos or gene in trueNeg:
+            if gene in truePos:
+                cumulativeTP += 1
+
+            elif gene in trueNeg:
+                cumulativeFP += 1
+
+            cumulativeTup.append((cumulativeTP / len(truePos), cumulativeTP / (cumulativeTP + cumulativeFP), fold_change))
+    
+    tup_cross95 = [cross95 for cross95 in cumulativeTup[::-1] if cross95[1] >= 0.95][0]
+    
+    if verbose: print('Precision-Recall AUC:', tup_cross95)
+    
+    return cumulativeTup, tup_cross95

screenpro/phenoscore/phenostat.py

@@ -0,0 +1,84 @@
+"""
+phenostat module: internal module for statistical analysis of phenoscore data.
+"""
+
+from scipy.stats import ttest_rel
+import numpy as np
+from statsmodels.stats.multitest import multipletests
+
+
+def matrixStat(x, y, test, level, transform='log10'):
+    """
+    Get p-values comparing `y` vs `x` matrices.
+
+    Parameters:
+        x (np.array): array of values
+        y (np.array): array of values
+        test (str): test to use for calculating p-value
+        level (str): level at which to calculate p-value
+        transform (str): transformation to apply to values before running test
+    
+    Returns:
+        np.array: array of p-values
+    """
+    # log-transform values
+    if transform == None:
+        pass
+    elif transform == 'log10':
+        x = np.log10(x)
+        y = np.log10(y)
+    else:
+        raise ValueError(f'Transform "{transform}" not recognized')
+    
+    # calculate p-values
+    if test == 'MW':
+        # run Mann-Whitney U rank test
+        raise ValueError('Mann-Whitney U rank test not implemented')
+    
+    elif test == 'KS':
+        # run Kolmorogov-Smirnov test
+        raise ValueError('Kolmorogov-Smirnov test not implemented')
+    
+    elif test == 'ttest':
+        # run ttest
+        if level == 'col':
+            p_value = ttest_rel(y, x, axis=1)[1]
+        elif level == 'row':
+            p_value = ttest_rel(y, x, axis=0)[1]
+        elif level == 'all':
+            # average across all values
+            p_value = ttest_rel(y, x, axis=None)[1]
+        else:
+            raise ValueError(f'Level "{level}" not recognized')
+        return p_value
+    else:
+        raise ValueError(f'Test "{test}" not recognized')
+
+
+def multipleTestsCorrection(p_values, method='fdr_bh'):
+    """
+    Calculate adjusted p-values using multiple testing correction.
+
+    Parameters:
+        p_values (np.array): array of p-values
+        method (str): method to use for multiple testing correction
+    
+    Returns:
+        np.array: array of adjusted p-values
+    """
+    if method == 'fdr_bh':
+        # fill na with 1
+        p_values[np.isnan(p_values)] = 1
+        # Calculate the adjusted p-values using the Benjamini-Hochberg method
+        if p_values is None:
+            raise ValueError('p_values is None')
+        _, adj_p_values, _, _ = multipletests(p_values, alpha=0.05, method='fdr_bh')
+    
+    else:
+        raise ValueError(f'Method "{method}" not recognized')
+
+    return adj_p_values
+
+
+def empiricalFDR():
+    pass