maradoner 0.10__tar.gz → 0.12__tar.gz

{maradoner-0.10 → maradoner-0.12}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: maradoner
-Version: 0.10
+Version: 0.12
 Summary: Variance-adjusted estimation of motif activities.
 Home-page: https://github.com/autosome-ru/nemara
 Author: Georgy Meshcheryakov
@@ -25,15 +25,8 @@ Requires-Dist: statsmodels>=0.14
 Requires-Dist: datatable>=1.0.0
 Requires-Dist: dill>=0.3.9
 Requires-Dist: rich>=12.6.0
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
+Requires-Dist: tqdm>=4.0
+Requires-Dist: scikit-learn>=1.6
 
 
 **MARADONER**

{maradoner-0.10 → maradoner-0.12}/maradoner/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = '0.10'
+__version__ = '0.12'
 import importlib
 
 
@@ -16,6 +16,8 @@ __min_reqs__ = [
             'datatable>=1.0.0' ,
             'dill>=0.3.9',
             'rich>=12.6.0',
+            'tqdm>=4.0',
+            'scikit-learn>=1.6'
            ]
 
 def versiontuple(v):

{maradoner-0.10 → maradoner-0.12}/maradoner/create.py

@@ -37,7 +37,7 @@ def transform_loadings(df, mode: str, zero_cutoff=1e-9, prom_inds=None):
 
 def create_project(project_name: str, promoter_expression_filename: str, loading_matrix_filenames: list[str],
                    motif_expression_filenames=None, loading_matrix_transformations=None, sample_groups=None, motif_postfixes=None,
-                   promoter_filter_lowexp_cutoff=0.95, promoter_filter_plot_filename=None,
+                   promoter_filter_lowexp_cutoff=0.95, promoter_filter_plot_filename=None, promoter_filter_max=True,
                    motif_names_filename=None, compression='raw', dump=True, verbose=True):
     if not os.path.isfile(promoter_expression_filename):
         raise FileNotFoundError(f'Promoter expression file {promoter_expression_filename} not found.')
@@ -88,7 +88,8 @@ def create_project(project_name: str, promoter_expression_filename: str, loading
                             f'{len(loading_matrix_transformations)}.')
     
     logger_print('Filtering promoters of low expression...', verbose)
-    inds, weights = filter_lowexp(promoter_expression, cutoff=promoter_filter_lowexp_cutoff, fit_plot_filename=promoter_filter_plot_filename)
+    inds, weights = filter_lowexp(promoter_expression, cutoff=promoter_filter_lowexp_cutoff, fit_plot_filename=promoter_filter_plot_filename,
+                                  max_mode=promoter_filter_max)
     promoter_expression = promoter_expression.loc[inds]
     proms = promoter_expression.index
     loading_matrices = [transform_loadings(df, mode, prom_inds=inds) for df, mode in zip(loading_matrices, loading_matrix_transformations)]
@@ -115,6 +116,7 @@ def create_project(project_name: str, promoter_expression_filename: str, loading
         motif_expression = None
     loading_matrices = pd.concat(loading_matrices, axis=1)
     if motif_names is not None:
+        motif_names = list(set(motif_names) & set(loading_matrices.columns))
         loading_matrices = loading_matrices[motif_names]
     proms = list(promoter_expression.index)
     sample_names = list(promoter_expression.columns)

{maradoner-0.10 → maradoner-0.12}/maradoner/dataset_filter.py

@@ -6,7 +6,19 @@ import pandas as pd
 import numpy as np
 from scipy.optimize import minimize
 from functools import partial
+from sklearn.mixture import GaussianMixture
 
+def compute_leftmost_probability(Y):
+    Y = Y.reshape(-1, 1)
+    gmm = GaussianMixture(n_components=2, random_state=0)
+    gmm.fit(Y)
+    
+    means = gmm.means_.flatten()
+    leftmost_component_index = np.argmin(means)
+    probas = gmm.predict_proba(Y)
+    leftmost_probs = probas[:, leftmost_component_index]
+    
+    return leftmost_probs, gmm
 
 def normax_logpdf(x: jnp.ndarray, mu: float, sigma: float, n: int):
     x = (x - mu) / sigma
@@ -39,9 +51,33 @@ def loglik(params: jnp.ndarray, x: jnp.ndarray, n: int):
     w = params[-1]
     return -logmixture(x, mu, sigma, w, n).sum()
 
-def filter_lowexp(expression: pd.DataFrame, cutoff=0.95, fit_plot_filename=None, plot_dpi=200):
+def filter_lowexp(expression: pd.DataFrame, cutoff=0.95, max_mode=True, 
+                  fit_plot_filename=None, plot_dpi=200):
     expression = (expression - expression.mean()) / expression.std()
-    
+    if not max_mode:
+        expression = expression.mean(axis=1).values
+        probs, gmm = compute_leftmost_probability(expression)
+        inds = probs < (1-cutoff)
+        if fit_plot_filename:
+            import matplotlib.pyplot as plt
+            from matplotlib.collections import LineCollection
+            import seaborn as sns
+            x = np.array(sorted(expression))
+            pdf = np.exp(gmm.score_samples(expression[:, None]))
+            points = np.array([x, pdf]).T.reshape(-1, 1, 2)
+            segments = np.concatenate([points[:-1], points[1:]], axis=1)
+            plt.figure(dpi=plot_dpi, )
+            sns.histplot(expression, stat='density', color='grey')
+            lc = LineCollection(segments, cmap='winter')
+            lc.set_array(probs)
+            lc.set_linewidth(3)
+            line = plt.gca().add_collection(lc)
+            plt.colorbar(line)
+            plt.xlabel('Standardized expression')
+            plt.tight_layout()
+            plt.savefig(fit_plot_filename)
+        return inds, probs
+        
     expression_max = expression.max(axis=1).values
     
     mu = [-1.0, 0.0]
@@ -105,5 +141,6 @@ def filter_lowexp(expression: pd.DataFrame, cutoff=0.95, fit_plot_filename=None,
     inds[:k] = False
     # print(inds)
     # inds[:] = 1
+    print(x[inds].mean(), x[~inds].mean())
     inds = inds[inds_inv]
     return inds, ws

{maradoner-0.10 → maradoner-0.12}/maradoner/export.py

@@ -2,8 +2,9 @@
 # -*- coding: utf-8 -*-
 from pandas import DataFrame as DF
 # add dot
-from .utils import read_init, openers
+from .utils import read_init, openers, ProjectData
 from .fit import FOVResult, ActivitiesPrediction, FitResult
+from .grn import grn
 from scipy.stats import norm, chi2, multivariate_normal, Covariance
 from scipy.linalg import eigh, lapack, cholesky, solve
 from statsmodels.stats import multitest
@@ -80,7 +81,9 @@ class Information():
         try:
             x = chol_inv(x)
         except:
-            print('alarm')
+            print('Failed to compute inverse using Cholesky decomposition. ')
+            print('This can be a sign of a numerical errors during parameters estimation.')
+            print('Will use pseudo-inverse now. The minimal and maximal eigenvalues are:')
             # print(x.diagonal().min())
             assert np.allclose(x, x.T), x - x.T
             x = np.linalg.eigh(x)
@@ -155,11 +158,12 @@ def export_fov(fovs: tuple[FOVResult], folder: str,
     samples = [fov_null.sample[:, None], fov_means.sample[:, None], fov_motif_means.sample[:, None]]
     samples = np.concatenate(samples, axis=-1)
     DF(samples, index=sample_names, columns=cols).to_csv(os.path.join(folder, 'samples.tsv'), sep='\t')
-    
+
+
 
 
 def posterior_anova(activities: ActivitiesPrediction, fit: FitResult, 
-                    B: np.ndarray, corr_stat=False):
+                    B: np.ndarray, corr_stat=False, map_cov=False):
     precs = list()
     istds = list()
     covs = list()
@@ -170,22 +174,35 @@ def posterior_anova(activities: ActivitiesPrediction, fit: FitResult,
     #     mot = np.delete(mot, activities.filtered_motifs)
     #     ind = mot * nu < cov.diagonal() + 1e-9
     #     bad_inds[ind] = True
-
-    for cov, U, nu in zip(activities.cov(), activities.U.T, fit.motif_variance.group):
-        mot = fit.motif_variance.motif
-        mot = np.delete(mot, activities.filtered_motifs)[~bad_inds]
+    # mot = fit.motif_variance.motif
+    # mot = np.delete(mot, activities.filtered_motifs)[~bad_inds]
+    motif_variance = fit.motif_variance.motif
+    if activities.filtered_motifs is not None:
+        motif_variance = np.delete(motif_variance, activities.filtered_motifs)
+        B = np.delete(B, activities.filtered_motifs, axis=1)
+    U = activities.U
+    if map_cov:
+        # fit.motif_variance.m
+        BTB = B.T @ B
+        BTB_s = BTB * motif_variance ** 0.5
+        BTB_s = BTB_s @ BTB_s.T
+    for cov, U, sigma, n, nu in zip(activities.cov(), U.T, 
+                          activities._cov[-2], 
+                          fit.error_variance.variance, fit.motif_variance.group):
         # cov = cov[~bad_inds, ~bad_inds]
-        cov = cov[..., ~bad_inds]
-        cov = cov[~bad_inds]
+        # cov = cov[..., ~bad_inds]
+        # cov = cov[~bad_inds]
+        if map_cov:
+            D = BTB_s * nu  + np.identity(len(BTB)) * sigma
+            cov = cov @ D @ cov.T * n / sigma ** 2 
         covs.append(cov)
-        U = U[~bad_inds]
+        # U = U[~bad_inds]
         # prec = np.linalg.inv(np.diag(mot * nu) - cov)
-        prec = np.linalg.inv(cov)
+        prec = np.linalg.pinv(cov, hermitian=True)
         mean += prec @ U
         precs.append(prec)
-    print(bad_inds.sum())
     total_prec = sum(precs)
-    total_cov = np.linalg.inv(total_prec)
+    total_cov = np.linalg.pinv(total_prec, hermitian=True)
     mean = total_cov @ mean
     stats = activities.U[~bad_inds] - mean.reshape(-1, 1)
     # if corr_stat:
@@ -203,16 +220,11 @@ def posterior_anova(activities: ActivitiesPrediction, fit: FitResult,
     fdr = multitest.multipletests(pvalues, alpha=0.05, method='fdr_by')[1] 
     return stats, pvalues, fdr, bad_inds
     
-    
-    
-    
 
 def export_results(project_name: str, output_folder: str,
                    std_mode: Standardization, 
                    anova_mode: ANOVAType=ANOVAType.positive,
-                   compute_corrected_pvalues=False,
-                   corrected_numerical=False,
-                   corrected_num_samples=1e5,
+                   weighted_zscore=False,
                    alpha=0.05,
                    n_jobs=6):
     
@@ -243,7 +255,7 @@ def export_results(project_name: str, output_folder: str,
         motif_names_filtered = motif_names
     
     os.makedirs(output_folder, exist_ok=True)
-    
+    # grn(data, act, fit, os.path.join(output_folder, 'grn'))
     error_variance = fit.error_variance.variance
     error_variance_fim = Information(fit.error_variance.fim)
     error_variance_stat, error_variance_std = error_variance_fim.standardize(error_variance, 
@@ -272,9 +284,12 @@ def export_results(project_name: str, output_folder: str,
     
     folder = os.path.join(output_folder, 'params')
     os.makedirs(folder, exist_ok=True)
+    if os.path.isfile(f'{project_name}.promvar.{fmt}'):
+        with openers[fmt](f'{project_name}.promvar.{fmt}', 'rb') as f:
+            promvar: np.ndarray = dill.load(f)
+        DF(promvar, index=prom_names, columns=group_names).to_csv(os.path.join(folder, 'promoter_variances.tsv'), sep='\t')
     if excluded_motif_group is not None:
         motif_group_variance_std = np.insert(motif_group_variance_std, excluded_motif_group, np.nan)
-    print(error_variance.shape, error_variance_std.shape,   motif_group_variance.shape, motif_group_variance_std.shape)
     DF(np.array([error_variance, error_variance_std, motif_group_variance, motif_group_variance_std]).T,
                 index=group_names,
                 columns=['sigma', 'sigma_std', 'nu', 'nu_std']).to_csv(os.path.join(folder, 'group_variances.tsv'),
@@ -324,12 +339,11 @@ def export_results(project_name: str, output_folder: str,
     pval = calc_z_test(anova_ass)
 
     fdrs = multitest.multipletests(pval, alpha=0.05, method='fdr_bh')[1]
-    lrt = 2 * fit.motif_variance.logratios
-    lrt_pvalues = chi2.sf(lrt, 1)
-    lrt_fdr = multitest.multipletests(lrt_pvalues, alpha=0.05, method='fdr_bh')[1]
-    anova_ass = DF(np.array([anova_ass, pval, fdrs, lrt, lrt_pvalues, lrt_fdr]).T, index=motif_names_filtered,
-                   columns=['stat', 'p-value', 'FDR',
-                            'logratio', 'lrt_p-value', 'lrt_FDR'])
+    # lrt = 2 * fit.motif_variance.logratios
+    # lrt_pvalues = chi2.sf(lrt, 1)
+    # lrt_fdr = multitest.multipletests(lrt_pvalues, alpha=0.05, method='fdr_bh')[1]
+    anova_ass = DF(np.array([anova_ass, pval, fdrs]).T, index=motif_names_filtered,
+                   columns=['stat', 'p-value', 'FDR'])
     anova_ass.to_csv(os.path.join(folder, 'anova.tsv'), sep='\t')
     
     sign = motif_mean.flatten() / motif_mean_std
@@ -347,6 +361,28 @@ def export_results(project_name: str, output_folder: str,
                   index=motif_names)
     sign_ass.to_csv(os.path.join(folder, 'sign.tsv'), sep='\t')
     
+    folder = os.path.join(output_folder, 'activities')
+    os.makedirs(folder, exist_ok=True)
+    U = list()
+    stds = list()
+    for u, cov in zip(act.U.T, act.cov()):
+        std = cov.diagonal() ** 0.5
+        u = u / std
+        U.append(u)
+        stds.append(std)
+    U = np.array(U).T
+    DF(U, index=motif_names_filtered, columns=group_names).to_csv(os.path.join(folder, 'activity.tsv'), sep='\t')
+    U = U ** 2
+    if weighted_zscore:
+        U_total = U.sum(axis=1, keepdims=True) / (1 / np.array(stds).T ** 2).sum(axis=1, keepdims=True)
+    else:
+        U_total = U.mean(axis=1, keepdims=True)
+    
+    U = np.hstack((U_total, U)) ** 0.5
+    DF(U, index=motif_names_filtered,
+       columns=['overall'] + list(group_names)).to_csv(os.path.join(folder, 'z_score.tsv'), sep='\t')
+    DF(act.U_raw, index=motif_names_filtered, columns=data.sample_names).to_csv(os.path.join(folder, 'activity_raw.tsv'), sep='\t')
+    
     if os.path.isfile(f'{project_name}.fov.{fmt}'):
         with open(f'{project_name}.fov.{fmt}', 'rb') as f:
             fov = dill.load(f)
@@ -373,6 +409,48 @@ def export_results(project_name: str, output_folder: str,
                        sample_names=sample_names)
         
             
+def export_loadings_product(project_name: str, output_folder: str,
+                            use_hdf: bool = True, intercepts: bool = True,
+                            tsv_truncation=4):
     
-    # return {'z-test': z_test, 'anova': anova, 'off_test': off_test,
-    #         'anova_ass': anova_ass, 'sign_ass': sign_ass}
+
+    data = read_init(project_name)
+    fmt = data.fmt
+    motif_names = data.motif_names
+    prom_names = data.promoter_names
+    # del data
+    with openers[fmt](f'{project_name}.fit.{fmt}', 'rb') as f:
+        fit: FitResult = dill.load(f)
+    if fit.promoter_inds_to_drop:
+        prom_names = np.delete(prom_names, fit.promoter_inds_to_drop)
+    group_names = fit.group_names
+    with openers[fmt](f'{project_name}.predict.{fmt}', 'rb') as f:
+        act: ActivitiesPrediction = dill.load(f)
+    
+    output_folder = os.path.join(output_folder, 'loadings-product')
+    os.makedirs(output_folder, exist_ok=True)
+    
+    U = act.U
+    B = data.B
+    mu = fit.motif_mean.mean 
+    
+    if act.filtered_motifs is not None:
+        motif_names = np.delete(motif_names, act.filtered_motifs)
+        B = np.delete(B, act.filtered_motifs, axis=1)
+        mu = np.delete(mu, act.filtered_motifs)
+    BM = B * mu
+    for name, U in zip(group_names, U.T):
+        effect = B * U
+        if intercepts:
+            effect += BM
+        if use_hdf:
+            effect = effect.astype(np.half)
+            filename = os.path.join(output_folder, f'{name}.hdf')
+            DF(data=effect, index=prom_names, columns=motif_names).to_hdf(filename, key='lrt', mode='w', complevel=4)
+        else:
+            filename = os.path.join(output_folder, f'{name}.tsv')
+            DF(data=effect, index=prom_names, columns=motif_names).to_csv(filename, sep='\t',
+                                                                          float_format=f'%.{tsv_truncation}f')  
+        
+        
+