exprmat 0.1.5__tar.gz → 0.1.6__tar.gz

{exprmat-0.1.5 → exprmat-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: exprmat
-Version: 0.1.5
+Version: 0.1.6
 Summary: routines to process expression matrices
 Home-page: https://github.com/yang-z-03/exprmat
 Author: Zheng Yang
@@ -28,6 +28,7 @@ Requires-Dist: datashader
 Requires-Dist: fa2_modified
 Requires-Dist: pynndescent
 Requires-Dist: gseapy
+Requires-Dist: sh
 Dynamic: author
 Dynamic: author-email
 Dynamic: home-page

exprmat-0.1.6/exprmat/clustering/rogue.py

@@ -0,0 +1,333 @@
+
+import numpy as np
+import pandas as pd
+import statsmodels.api as sm
+from skmisc import loess
+from scipy.stats import norm
+from statsmodels.stats.multitest import multipletests
+from exprmat.ansi import error, warning, info
+
+
+# the entropy of the genes across cell j is defined to be:
+#
+#     s1 = ln(mean(expr_j))
+#
+# this assumes that all cells in the cluster is homogenous. while the alternative
+# assumption is that cells are totally heterogenous, and each cell represent its
+# distinct state. the entropy for a totally heterogenous dataset is:
+# 
+#     s2 = mean(log(expr_ij))
+#
+# for calculation, this is simply a conversion of mean and log transform.
+# r is a small positive value to avoid nan's been introduced due to invalid log.
+# we require that the expression matrix have gene names as row names.
+
+def rogue_entropy(C, varname, r = 1):
+    '''
+    Calculate entropy for genes. A completely homogenous gene will yield entropy
+    as mean logrithmized counts (mean.expr). Expect raw count matrix.
+    '''
+  
+    entropy = np.log(C + 1).mean(axis = 0) # per gene.
+    mean_expr = np.log(C.mean(axis = 0) + r)
+    return pd.DataFrame({
+        'gene': varname,
+        'entropy': entropy.tolist()[0],
+        'mean.expr': mean_expr.tolist()[0]
+    })
+
+
+# fit the relationship between expression entropy and mean gene expression:
+#
+# in real-world dataset, there must be to some extent, heterogenous components
+# in the dataset clusters. s1 > s2. and for different techniques for rna sequencing
+# there is a systematic bias on the data distribution. we need a fitting step
+# to make sure that the highly-variable genes in the clusters are picked.
+#
+#' @param span is a parameter for loess for controls of fit smoothness.
+#' @param mt.method multiple testing method used in p.adjust.
+#
+# the returning dataframe consists of [1] gene names, [2] log mean expression (s1),
+# [3] fit, the fitted curve of real world log mean expression trajectory (s1'),
+# [4] ds, delta of entropy, the difference between s2 and s1',
+# [5] p.value, assume that ds is normal, and the p value of normal distribution.
+# [6] p.adj, the multi-test adjusted p value for normal approximation. and
+# [7] entropy, the expression entropy per se.
+#
+# the returning values are sorted according to decreasing delta of entropy, so
+# that the most highly-variable genes ranked on the top.
+
+# available correction methods:
+#  * bonferroni : one-step correction
+#  * sidak : one-step correction
+#  * holm-sidak : step down method using Sidak adjustments
+#  * holm : step-down method using Bonferroni adjustments
+#  * simes-hochberg : step-up method (independent)
+#  * hommel : closed method based on Simes tests (non-negative)
+#  * fdr_bh : Benjamini/Hochberg (non-negative)
+#  * fdr_by : Benjamini/Yekutieli (negative)
+#  * fdr_tsbh : two stage fdr correction (non-negative)
+#  * fdr_tsbky : two stage fdr correction (non-negative)
+
+def smoothen_entropy_lowess(df, span = 0.5, correction = "fdr_bh"):
+    
+    df = df.loc[
+        np.isfinite(df['mean.expr']) & 
+        df['entropy'] > 0, :
+    ].copy()
+    
+    fit = sm.nonparametric.lowess(
+        exog = df['mean.expr'], 
+        endog = df['entropy'], frac = span,
+        xvals = df['mean.expr']
+    )
+
+    df['fit'] = fit
+    df['ds'] = df['fit'] - df['entropy']
+    df['pval'] = 1 - norm.cdf(df['ds'], np.mean(df['ds']), np.std(df['ds']))
+    training = df.loc[df['pval'] > 0.1, :].copy()
+    
+    fit2 = sm.nonparametric.lowess(
+        exog = training['mean.expr'], 
+        endog = training['entropy'], frac = span,
+        xvals = df['mean.expr']
+    )
+
+    df['fit'] = fit2
+    df['ds'] = df['fit'] - df['entropy']
+    df['pval'] = 1 - norm.cdf(df['ds'], np.mean(df['ds']), np.std(df['ds']))
+    training = df.loc[df['pval'] > 0.1, :].copy()
+
+    fit3 = sm.nonparametric.lowess(
+        exog = training['mean.expr'], 
+        endog = training['entropy'], frac = span,
+        xvals = df['mean.expr']
+    )
+
+    df['fit'] = fit3
+    df['ds'] = df['fit'] - df['entropy']
+    df = df.loc[np.isfinite(df['ds']), :]
+    df['pval'] = 1 - norm.cdf(df['ds'], np.mean(df['ds']), np.std(df['ds']))
+    df['qval'] = multipletests(df['pval'], method = correction)[1]
+    
+    return df.sort_values(['ds'], ascending = False)
+
+
+def smoothen_entropy_loess(df, span = 0.5, correction = "fdr_bh"):
+    
+    df = df.loc[
+        np.isfinite(df['mean.expr']) & 
+        df['entropy'] > 0, :
+    ].copy()
+    
+    model = loess.loess(df['mean.expr'], df['entropy'], span = span)
+    model.fit()
+    fit = model.predict(df['mean.expr'])
+
+    df['fit'] = fit.values
+    df['ds'] = df['fit'] - df['entropy']
+    df['pval'] = 1 - norm.cdf(df['ds'], np.mean(df['ds']), np.std(df['ds']))
+    training = df.loc[df['pval'] > 0.1, :].copy()
+    
+    df = df.loc[df['mean.expr'] <= np.max(training['mean.expr']), :].copy()
+    model = loess.loess(training['mean.expr'], training['entropy'], span = span)
+    model.fit()
+    fit2 = model.predict(df['mean.expr'])
+
+    df['fit'] = fit2.values
+    df['ds'] = df['fit'] - df['entropy']
+    df['pval'] = 1 - norm.cdf(df['ds'], np.mean(df['ds']), np.std(df['ds']))
+    training = df.loc[df['pval'] > 0.1, :].copy()
+
+    df = df.loc[df['mean.expr'] <= np.max(training['mean.expr']), :].copy()
+    model = loess.loess(training['mean.expr'], training['entropy'], span = span)
+    model.fit()
+    fit3 = model.predict(df['mean.expr'])
+
+    df['fit'] = fit3.values
+    df['ds'] = df['fit'] - df['entropy']
+    df = df.loc[np.isfinite(df['ds']), :]
+    df['pval'] = 1 - norm.cdf(df['ds'], np.mean(df['ds']), np.std(df['ds']))
+    df['qval'] = multipletests(df['pval'], method = correction)[1]
+    
+    return df.sort_values(['ds'], ascending = False)
+
+
+smoothen_entropy = smoothen_entropy_loess
+
+# identify highly informative genes using rogue's s-e model
+# 
+# returns a tibble object with seven columns:
+# [1] gene, the gene name.
+# [2] mean.expr, the mean expression levels of genes.
+# [3] entropy, the expected expression entropy from a given mean gene expression.
+# [4] fit, the mean expression levels of genes.
+# [5] ds, the entropy reduction against the null expectation.
+# [6] p.value, the significance of ds against normal distribution.
+# [7] p.adj, adjusted p value, or a copy of p.value if specifying not to adjust it.
+
+def rogue_hvg(C, varname, span = 0.5, r = 1, correction = "fdr_bh", adjust_p = True):
+  
+    entropy = rogue_entropy(C, varname, r = r)
+    entropy = smoothen_entropy(entropy, span = span, correction = correction)
+    if not adjust_p: entropy['qval'] = entropy["pval"]
+    return entropy
+
+
+# assess the purity of single cell population by rogue index.
+#
+# we may either specify the platform to access the default k value, or specify
+# a k value manually. the rogue test is meaned to be run with highly variable
+# genes. you can either specify them manually in `features`, or it is recommended
+# to use the s-e model to pick the hvgs for you.
+
+def rogue_index(df, platform = "umi", cutoff = 0.05, k = None, features = None):
+    
+    if k is None:
+        if platform is None:
+            warning("you should supply a constant k yourself, or specify the platform for ")
+            warning("us to automatically pick the k constant for you. possible platforms ")
+            warning("are `umi` or `full-length`, the default k value for each case is 45 ")
+            warning("and 500 respectively.")
+            error('routine exit.')
+
+        elif platform == 'umi': k = 45
+        elif platform == 'full-length': k = 500
+        else: error('platform should be one of `umi` or `full-length`.')
+    
+    if features is not None:
+        df = df.loc[[x in features for x in df['gene'].tolist()], :]
+        sig = np.abs(df['ds'].to_numpy()).sum()
+        rogue = 1 - sig / (sig + k)
+
+    else:
+        
+        df = df.loc[
+            (df['pval'] < cutoff) &
+            (df['qval'] < cutoff), :
+        ]
+
+        sig = np.abs(df['ds'].to_numpy()).sum()
+        rogue = 1 - sig / (sig + k)
+    
+    return rogue
+
+
+# remove outlier cells when calculating ROGUE
+#' @param n remove this many outlier cells.
+
+def remove_outliers(df, C, varname, n = 2, span = 0.5, r = 1, correction = 'fdr_bh'):
+    
+    from exprmat.utils import choose_layer
+    signif_genes = df.loc[df['qval'] < 0.05, :]['gene'].tolist()
+    ngenes = len(signif_genes)
+    bool_mask_gene = [x in signif_genes for x in varname]
+    expr_cutoff = df['mean.expr'].to_numpy().min()
+    
+    expr = C[:, bool_mask_gene]
+    signif_genes = []
+    for imask in range(len(bool_mask_gene)):
+        if bool_mask_gene[imask]: signif_genes.append(varname[imask])
+
+    for i in range(ngenes):
+        gene_expr = np.flip(np.array(np.sort(expr[:, i].T))[0])
+        gene_expr = gene_expr[n:]
+        mean = np.log(gene_expr.mean() + r)
+        entropy = np.mean(np.log(gene_expr + 1))
+        df.loc[df['gene'] == signif_genes[i], 'mean.expr'] = mean
+        df.loc[df['gene'] == signif_genes[i], 'entropy'] = entropy
+
+    df = df.loc[df['mean.expr'] > expr_cutoff, :]
+    del df['qval']
+    df = smoothen_entropy(df, span = span, correction = correction)
+    return df
+
+
+def rogue(
+    adata, counts = 'X', platform = "umi", k = None, 
+    min_cells = 10, min_genes = 10, outlier_n = 2,
+    span = 0.5, r = 1, correction = 'fdr_bh'
+):
+    '''
+    Calculate ROGUE heterogeneity index for the whole annotated data.
+    Users are recommended to filter the minimally expressing genes and cells,
+    and pick out the subset of interest before feeding into this method. Or
+    more conveniently, run the analysis on given categorical properties via
+    the wrapper function ``experiment.run_rna_rogue``.
+
+    Parameters
+    -----------
+
+    adata : anndata.AnnData
+        The set of data you would like to test for heterogeneity
+
+    counts : str = 'X'
+        ROGUE index is calculated on raw counts. Specify a layer of raw counts.
+    
+    platform : Literal['umi', 'full-length']
+        Choose k value with preset technology
+
+    k : int = None
+        Manually picks a k value. It can be estimated from dataset using
+        ``estimate_k`` function, or use the recommended value via ``platform``.
+
+    outlier_n : int = 2
+        Remove n top-expressing cells during analysis
+
+    span : float = 0.5
+        Controls the smoothness of LOESS regression
+    
+    correction : str = 'fdr_bh'
+        Multitest correction of p-value. Available correction methods:
+        *  bonferroni : one-step correction
+        *  sidak : one-step correction
+        *  holm-sidak : step down method using Sidak adjustments
+        *  holm : step-down method using Bonferroni adjustments
+        *  simes-hochberg : step-up method (independent)
+        *  hommel : closed method based on Simes tests (non-negative)
+        *  fdr_bh : Benjamini/Hochberg (non-negative)
+        *  fdr_by : Benjamini/Yekutieli (negative)
+        *  fdr_tsbh : two stage fdr correction (non-negative)
+        *  fdr_tsbky : two stage fdr correction (non-negative)
+
+    Returns
+    -----------
+        A floating point value, rogue index
+    '''
+    
+    from exprmat.utils import choose_layer
+    C = choose_layer(adata, layer = counts).todense()
+    hvg = rogue_hvg(
+        C, adata.var_names.tolist(), 
+        span = span, r = r, correction = correction
+    )
+
+    hvg = remove_outliers(
+        hvg, C, adata.var_names.tolist(),
+        n = outlier_n, span = span, r = r, correction = correction
+    )
+
+    return rogue_index(hvg, platform = platform, k = k)
+
+
+# calculate the value of the reference factor k
+
+def estimate_k(C, varname, span = 0.5, r = 1, correction = "fdr_bh", adjust_p = True):
+  
+    entropy = rogue_entropy(C, varname, r = r)
+    entropy = smoothen_entropy(entropy, span = span, correction = correction)
+    if not adjust_p: entropy['qval'] = entropy["pval"]
+    entropy = entropy.loc[entropy['qval'] < 0.05, :]
+    k = entropy['ds'].to_numpy().sum()
+    return k // 2
+
+
+def se_plot(df):
+
+    import seaborn as sns
+    return sns.scatterplot(
+        data = df, 
+        x = 'mean.expr', 
+        y = 'entropy', 
+        hue = df['pval'].to_numpy() < 0.05
+    )

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/data/finders.py

@@ -1,6 +1,7 @@
 
 import os
 import pandas
+from exprmat.ansi import warning, info
 from exprmat.configuration import default as cfg
 
 basepath = os.path.dirname(__file__)
@@ -124,6 +125,36 @@ def get_mapper_ensembl(taxa):
     return ensembl_finder
 
 
+def update_mapper(taxa, name, ens):
+    '''
+    Update the name ENSEMBL ID pair.
+    '''
+    
+    if taxa not in genome.keys():
+        get_mapper_name(taxa)
+
+    if 'mapper.name' not in genome[taxa].keys():
+        get_mapper_name(taxa)
+
+    if 'mapper.ensembl' not in genome[taxa].keys():
+        get_mapper_ensembl(taxa)
+    
+    if (name not in genome[taxa]['mapper.name'].keys()):
+        if (ens in genome[taxa]['mapper.ensembl'].keys()):
+            genome[taxa]['mapper.name'][name] = genome[taxa]['mapper.ensembl'][ens]
+        else: info(f'{ens} -> {name} pair could not be established.')
+
+    else:
+        if (ens not in genome[taxa]['mapper.ensembl'].keys()):
+            genome[taxa]['mapper.ensembl'][ens] = genome[taxa]['mapper.name'][name]
+        else: # both name and ens present in the dictionary, check for inconsistancy
+            if genome[taxa]['mapper.ensembl'][ens] != genome[taxa]['mapper.name'][name]:
+                warning(f'{ens} -> {name} is conflicted. ({ens} registered to {genome[taxa]["mapper.ensembl"][ens]}, while {name} to {genome[taxa]["mapper.name"][name]})')
+
+    
+    return
+
+
 def update_mapper_name(taxa, name, ugene):
     '''
     Update the gene name to unique gene id mapping.

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/plotting/palettes.py

@@ -29,7 +29,7 @@ module attribute, and the 'small' palettes (i.e. excluding the ones with 256
 colors) are collected and in a ``small_palettes`` attribute.
 
 This module includes some palettes that are useful for addressing color deficiencies,
-which contains `Paul Tol's color schemes`_, and ``Colorblind`` from 
+which contains Paul Tol's color schemes, and ``Colorblind`` from 
 https://jfly.uni-koeln.de/color/#pallet.
 
 The following palettes are also introduced in `Paul Tol's color schemes` but with 

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/preprocessing/__init__.py

@@ -84,7 +84,7 @@ def highly_variable(
     elif method == 'vst':
         df = highly_variable_genes_seurat(
             m_count, obs = adata.obs, var = adata.var, subset = False,
-            **kwargs
+            n_top_genes = n_top_genes, **kwargs
         )
         adata.var[dest + '.means'] = df['means']
         adata.var[dest + '.vars'] = df['variances']

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/preprocessing/plot.py

@@ -57,4 +57,24 @@ def rna_plot_qc_metrics(adata, sample_name, figsize = (10, 2.2)):
     fig.set_dpi(100)
     fig.tight_layout(pad = 0.1)
 
-    return fig
+    return fig
+
+
+def rna_plot_gene_histogram(adata, sample_name, ax):
+
+    h = ax.hist(adata.obs['n.genes'].tolist(), bins = 100)
+    ax.set_xlabel('Gene detection')
+    ax.set_ylabel('Frequency')
+    ax.set_title(sample_name if len(sample_name) < 20 else sample_name[:17] + ' ..')
+
+    ax.vlines(
+        x = [0, 150], ymin = 0, ymax = 10,
+        colors = 'red', linestyles = 'dotted'
+    )
+
+    ax.vlines(
+        x = [300, 600], ymin = 0, ymax = 10,
+        colors = 'black', linestyles = 'dotted'
+    )
+
+    ax.set_yscale('log')

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/preprocessing/qc.py

@@ -153,15 +153,16 @@ def rna_qc(
     adata.obs['score.doublet.se'] = 0.0
     adata.obs['is.doublet'] = False
 
-    adata.obs.loc[subset.obs['score.doublet'].index, 'score.doublet'] = \
-        subset.obs['score.doublet']
-    adata.obs.loc[subset.obs['score.doublet.se'].index, 'score.doublet.se'] = \
-        subset.obs['score.doublet.se']
-    if 'is.doublet' in subset.obs.columns.tolist():
-        adata.obs.loc[subset.obs['score.doublet'].index, 'is.doublet'] = \
-            subset.obs['is.doublet']
-    
-    adata.uns['scrublet'] = subset.uns['scrublet']
+    if doublet_method == 'scrublet':
+        adata.obs.loc[subset.obs['score.doublet'].index, 'score.doublet'] = \
+            subset.obs['score.doublet']
+        adata.obs.loc[subset.obs['score.doublet.se'].index, 'score.doublet.se'] = \
+            subset.obs['score.doublet.se']
+        if 'is.doublet' in subset.obs.columns.tolist():
+            adata.obs.loc[subset.obs['score.doublet'].index, 'is.doublet'] = \
+                subset.obs['is.doublet']
+        adata.uns['scrublet'] = subset.uns['scrublet']
+        
     adata.obs['qc'] = adata.obs['filter']
     adata.var['qc'] = f_var
 

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/reader/experiment.py

@@ -323,8 +323,10 @@ class experiment:
                 if not samp in self.modalities[mod].keys():
                     warning(f'{samp} not loaded in the {mod} modality.')
                     continue
-
-                results[samp] = func(self.modalities[mod][samp], samp, **kwargs)
+                
+                try:
+                    results[samp] = func(self.modalities[mod][samp], samp, **kwargs)
+                except: warning(f'method failed for sample {samp}')
         
         return results
 
@@ -694,6 +696,13 @@ class experiment:
         from exprmat.reduction.plot import embedding
         return embedding(adata, sample_name = sample_name, **kwargs)
     
+
+    @staticmethod
+    def rna_plot_embedding_atlas(adata, sample_name, **kwargs):
+        from exprmat.reduction.plot import embedding_atlas
+        return embedding_atlas(adata, sample_name = sample_name, **kwargs)
+    
+
     @staticmethod
     def rna_plot_gene_gene(adata, sample_name, **kwargs):
         from exprmat.reduction.plot import gene_gene
@@ -886,6 +895,43 @@ class experiment:
         return fig
     
 
+    @staticmethod
+    def rna_plot_multiple_embedding_atlas(
+        adata, sample_name, basis, features, ncols, 
+        figsize = (3, 3), dpi = 100, **kwargs
+    ):
+        from exprmat.reduction.plot import embedding_atlas
+        import matplotlib.pyplot as plt
+        import warnings
+        warnings.filterwarnings('ignore')
+
+        n_features = len(features)
+        nrows = n_features // ncols
+        if n_features % ncols != 0: nrows += 1
+        fig, axes = plt.subplots(nrows, ncols, dpi = dpi)
+
+        for feat_id in range(len(features)):
+
+            if len(axes.shape) == 2:
+                embedding_atlas(
+                    adata, basis, color = features[feat_id],
+                    ax = axes[feat_id // ncols, feat_id % ncols],
+                    sample_name = sample_name, dpi = dpi, **kwargs
+                )
+
+            elif len(axes.shape) == 1:
+                embedding_atlas(
+                    adata, basis, color = features[feat_id],
+                    ax = axes[feat_id], dpi = dpi,
+                    sample_name = sample_name, **kwargs
+                )
+        
+        fig.set_figwidth(figsize[0])
+        fig.set_figheight(figsize[1])
+        fig.tight_layout()
+        return fig
+    
+
     @staticmethod
     def rna_plot_multiple_gene_gene(
         adata, sample_name, color, features_xy, ncols, 
@@ -1000,9 +1046,15 @@ class experiment:
 
     def plot_rna_embedding(self, run_on_samples = False, **kwargs):
         return self.plot_for_rna(run_on_samples, experiment.rna_plot_embedding, **kwargs)
+    
+    def plot_rna_embedding_atlas(self, run_on_samples = False, **kwargs):
+        return self.plot_for_rna(run_on_samples, experiment.rna_plot_embedding_atlas, **kwargs)
 
     def plot_rna_embedding_multiple(self, run_on_samples = False, **kwargs):
         return self.plot_for_rna(run_on_samples, experiment.rna_plot_multiple_embedding, **kwargs)
+    
+    def plot_rna_embedding_atlas_multiple(self, run_on_samples = False, **kwargs):
+        return self.plot_for_rna(run_on_samples, experiment.rna_plot_multiple_embedding_atlas, **kwargs)
 
     def plot_rna_markers(self, run_on_samples = False, **kwargs):
         return self.plot_for_rna(run_on_samples, experiment.rna_plot_markers, **kwargs)
@@ -1028,6 +1080,50 @@ class experiment:
     def plot_rna_expression_bar(self, run_on_samples = False, **kwargs):
         return self.plot_for_rna(run_on_samples, experiment.rna_plot_expression_bar, **kwargs)
     
+    def plot_rna_qc_gene_counts(
+        self, ncols = 4, figsize = (3, 3)
+    ):
+        from exprmat.utils import setup_styles, plotting_styles
+        setup_styles(**plotting_styles)
+
+        from exprmat.preprocessing.plot import rna_plot_gene_histogram
+        import matplotlib.pyplot as plt
+        import warnings
+        warnings.filterwarnings('ignore')
+        if self.modalities is None: error('samples are not loaded')
+        if not 'rna' in self.modalities.keys(): error('samples are not loaded')
+
+        n_features = len(self.modalities['rna'])
+        if n_features == 0: error('samples are not loaded')
+
+        nrows = n_features // ncols
+        if n_features % ncols != 0: nrows += 1
+        fig, axes = plt.subplots(nrows, ncols)
+
+        samples = list(self.modalities['rna'].keys())
+        samples.sort()
+        for feat_id in range(n_features):
+
+            if len(axes.shape) == 2:
+                rna_plot_gene_histogram(
+                    self.modalities['rna'][samples[feat_id]],
+                    sample_name = samples[feat_id],
+                    ax = axes[feat_id // ncols, feat_id % ncols]
+                )
+
+            elif len(axes.shape) == 1:
+                rna_plot_gene_histogram(
+                    self.modalities['rna'][samples[feat_id]],
+                    sample_name = samples[feat_id],
+                    ax = axes[feat_id]
+                )
+        
+        fig.set_figwidth(figsize[0])
+        fig.set_figheight(figsize[1])
+        fig.tight_layout()
+        return fig
+    
+    
 
     # accessor wrappers
 
@@ -1164,30 +1260,36 @@ class experiment:
 
         else: print(red('[!]'), 'dataset not integrated.')
 
-        print(red('[*]'), 'composed of samples:')
-        for i_loc, i_sample, i_batch, i_grp, i_mod, i_taxa in zip(
-            self.metadata.dataframe['location'], 
-            self.metadata.dataframe['sample'], 
-            self.metadata.dataframe['batch'], 
-            self.metadata.dataframe['group'], 
-            self.metadata.dataframe['modality'], 
-            self.metadata.dataframe['taxa']
-        ):
-            loaded = False
-            if (self.modalities is not None) and \
-               (i_mod in self.modalities.keys()) and \
-               (i_sample in self.modalities[i_mod].keys()):
-                loaded = True
-
-            print(
-                f'  {i_sample:20}', cyan(f'{i_mod:4}'), yellow(f'{i_taxa:4}'),
-                f'batch {green(f"{i_batch:10}")}',
-                red('dataset not loaded') if not loaded else 
-                f'{green(str(self.modalities[i_mod][i_sample].n_obs))} × ' +
-                f'{yellow(str(self.modalities[i_mod][i_sample].n_vars))}'
-            )
+        if self.modalities is None or len(self.modalities) == 0:
+            print(red('[*]'), 'samples not loaded from disk.')
+        
+        else:
+            print(red('[*]'), 'composed of samples:')
+            for i_loc, i_sample, i_batch, i_grp, i_mod, i_taxa in zip(
+                self.metadata.dataframe['location'], 
+                self.metadata.dataframe['sample'], 
+                self.metadata.dataframe['batch'], 
+                self.metadata.dataframe['group'], 
+                self.metadata.dataframe['modality'], 
+                self.metadata.dataframe['taxa']
+            ):
+                loaded = False
+                if (self.modalities is not None) and \
+                   (i_mod in self.modalities.keys()) and \
+                   (i_sample in self.modalities[i_mod].keys()):
+                    loaded = True
+
+                p_sample = i_sample if len(i_sample) < 30 else i_sample[:27] + ' ..'
+                p_batch = i_batch if len(i_batch) < 30 else i_batch[:27] + ' ..'
+                print(
+                    f'  {p_sample:30}', cyan(f'{i_mod:4}'), yellow(f'{i_taxa:4}'),
+                    f'batch {green(f"{p_batch:30}")}',
+                    red('dataset not loaded') if not loaded else 
+                    f'{green(str(self.modalities[i_mod][i_sample].n_obs))} × ' +
+                    f'{yellow(str(self.modalities[i_mod][i_sample].n_vars))}'
+                )
 
-        return '<exprmat.reader.experiment>'
+        return f'<exprmat.reader.experiment> ({len(self.metadata.dataframe)} samples)'
 
     pass
 

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/reader/matcher.py

@@ -15,12 +15,53 @@ import numpy as np
 from scipy.sparse import csr_matrix
 from exprmat.ansi import error, warning, info
 from exprmat.data.finders import get_mapper_ensembl, get_mapper_name, get_genome
-from exprmat.data.finders import update_mapper_ensembl, update_mapper_name, save_genome_changes
+from exprmat.data.finders import update_mapper_ensembl, update_mapper, save_genome_changes
 from exprmat.reader.metadata import metadata
 from exprmat.configuration import default as cfg
 from exprmat.ansi import info
 
 
+def refine_finder(features_fpath, taxa = 'mmu'):
+    table = pd.read_table(features_fpath, sep = '\t', header = None)
+    assert len(table.columns) >= 2
+
+    name_finder = get_mapper_name(taxa)
+    ensembl_finder = get_mapper_ensembl(taxa)
+
+    ensembls = table[0].tolist()
+    names = table[1].tolist()
+    mentioned_taxa = []
+
+    for ens, nm in zip(ensembls, names):
+
+        # here, we will check whether the names and ensembls starts with a reference
+        # prefix. 10x's default will append something like mm10_ before the gene names
+        # and indices in multi-species joint analysis. if there is no such prefix detected
+        # this means the gene comes from the default taxa.
+        
+        if '_' in ens:
+            reference_name = ens.split('_')[0]
+            pure_ens = ens.replace(reference_name + '_', '')
+            pure_nm = nm.replace(reference_name + '_', '')
+            if not reference_name in cfg['taxa.reference'].keys():
+                warning(f'gene {ens} seems to have a reference prefix, but not registered to taxa.')
+                continue
+
+            reference_taxa = cfg['taxa.reference'][reference_name]
+            alt_name_finder = get_mapper_name(reference_taxa)
+            alt_ens_finder = get_mapper_ensembl(reference_taxa)
+            if ((pure_nm not in alt_name_finder.keys()) or (pure_ens not in alt_ens_finder.keys())):
+                update_mapper(reference_taxa, pure_nm, pure_ens)
+                if reference_taxa not in mentioned_taxa: mentioned_taxa += [reference_taxa]
+
+        elif ((nm not in name_finder.keys()) or (ens not in ensembl_finder.keys())):
+            update_mapper(taxa, nm, ens)
+            if taxa not in mentioned_taxa: mentioned_taxa += [taxa]
+
+    for taxa in mentioned_taxa:
+        save_genome_changes(taxa)
+
+
 def adjust_features(path, refine_finder = False, default_taxa = 'mmu'):
     '''
     There are three main types of features table that may occur in the matrix data folder.
@@ -91,12 +132,12 @@ def adjust_features(path, refine_finder = False, default_taxa = 'mmu'):
                 reference_taxa = cfg['taxa.reference'][reference_name]
                 alt_name_finder = get_mapper_name(reference_taxa)
                 alt_ens_finder = get_mapper_ensembl(reference_taxa)
-                if ((pure_nm not in alt_name_finder.keys()) and (pure_ens in alt_ens_finder.keys())):
-                    update_mapper_name(reference_taxa, pure_nm, alt_ens_finder[pure_ens])
+                if ((pure_nm not in alt_name_finder.keys()) or (pure_ens not in alt_ens_finder.keys())):
+                    update_mapper(reference_taxa, pure_nm, pure_ens)
                     if reference_taxa not in mentioned_taxa: mentioned_taxa += [reference_taxa]
 
-            elif ((nm not in name_finder.keys()) and (ens in ensembl_finder.keys())):
-                update_mapper_name(default_taxa, nm, ensembl_finder[ens])
+            elif ((nm not in name_finder.keys()) or (ens not in ensembl_finder.keys())):
+                update_mapper(default_taxa, nm, ens)
                 if default_taxa not in mentioned_taxa: mentioned_taxa += [default_taxa]
 
         for taxa in mentioned_taxa:

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat/reduction/plot.py

@@ -57,17 +57,23 @@ class index_object_handler:
 
 def embedding_atlas(
     adata, basis, color,
+    hue_order = None,
     title = None, figsize = (4, 4), ax = None,
-    cmap = 'Turbo', legend_loc = 'right margin',frameon = 'small', fontsize = 10
+    cmap = 'Turbo', cmap_reverse = False, cmap_lower = '#000000',
+    legend = True, legend_loc = 'right margin', legend_col = 1, 
+    frameon = 'small', fontsize = 10,
+    annotate = True, annotate_style = 'index', annot_size = 12,
+    annotate_foreground = 'black', annotate_stroke = 'white',
+    dpi = 100, sample_name = None
 ):
-    setup_styles(**plotting_styles)
+    res_factor = 1.5
     import pandas as pd
     import datashader as ds
     import datashader.transfer_functions as tf
     from scipy.sparse import issparse
     from exprmat.plotting import palettes
 
-    cvs = ds.Canvas(plot_width = 800, plot_height = 800)
+    cvs = ds.Canvas(plot_width = int(figsize[0] * dpi * res_factor), plot_height = int(figsize[1] * dpi * res_factor))
     embedding = adata.obsm[basis]
     df = pd.DataFrame(embedding, columns=['x', 'y'])
 
@@ -81,25 +87,51 @@ def embedding_atlas(
         X = adata.raw[:,color].X
         if issparse(X): labels = X.toarray().reshape(-1)
         else: labels = X.reshape(-1)
+        
+    # try some conventions
+    elif 'gene' in adata.var.keys() and color in adata.var['gene'].tolist():
+        genes = adata.var['gene'].tolist()
+        X = adata.X[:, genes.index(color)]
+        if issparse(X): labels = X.toarray().reshape(-1)
+        else: labels = X.reshape(-1)
+
+     # try some conventions
+    elif 'ensembl' in adata.var.keys() and color in adata.var['ensembl'].tolist():
+        genes = adata.var['ensembl'].tolist()
+        X = adata.X[:, genes.index(color)]
+        if issparse(X): labels = X.toarray().reshape(-1)
+        else: labels = X.reshape(-1)
+    
+    else: error(f'unable to find feature `{color}` in metadata or variables.')
 
     df['label'] = labels
     
+    if ax is None: fig, ax = plt.subplots(figsize = figsize, dpi = dpi)
+    else: fig = ax.figure
+
     if type(labels[0]) is str:
         df['label'] = df['label'].astype('category')
         agg = cvs.points(df, 'x', 'y', ds.count_cat('label'))
-        legend_tag = True
+        legend_tag = legend
 
-        default_palette = palettes.linear_palette(palettes.all_palettes[cmap][
+        original_cat = df['label'].value_counts().index.tolist()
+        original_cat = sorted(original_cat, key = lambda s: int(s) if str.isdigit(s) else s)
+        hue_order = original_cat if hue_order is None else hue_order
+        
+        default_palette = list(palettes.linear_palette(palettes.all_palettes[cmap][
             list(palettes.all_palettes[cmap].keys())[-1]
-        ], len(adata.obs[color].cat.categories))
+        ], len(adata.obs[color].cat.categories))) if isinstance(cmap, str) else cmap
 
         if f'{color}.colors' not in adata.uns.keys():
             adata.uns[f'{color}.colors'] = default_palette
         elif len(adata.uns[f'{color}.colors']) != len(default_palette):
             adata.uns[f'{color}.colors'] = default_palette
+        elif not isinstance(adata.uns[f'{color}.colors'], list):
+            adata.uns[f'{color}.colors'] = list(adata.uns[f'{color}.colors'])
         
+        default_palette = adata.uns[f'{color}.colors']
         color_key = dict(zip(
-            adata.obs[color].cat.categories,
+            hue_order,
             adata.uns[f'{color}.colors']
         ))
         
@@ -108,7 +140,65 @@ def embedding_atlas(
             color_key = [color_key[i] for i in df['label'].cat.categories], 
             how = 'eq_hist'
         )
+
+        ax.imshow(
+            img.to_pil(), 
+            aspect = 'auto', 
+            interpolation = 'bicubic',
+            extent = (0, dpi * figsize[0] * res_factor, 0, dpi * figsize[1] * res_factor)
+        )
+
+        if legend:
+            assert len(adata.uns[f'{color}.colors']) == len(hue_order)
+            dummy_objects = []
+            legend_artists = {}
+            for legend_t, legend_c, legend_id in zip(
+                hue_order, adata.uns[f'{color}.colors'], range(len(hue_order))
+            ):
+                dummy = index_object()
+                dummy_objects += [dummy]
+                handler = index_object_handler()
+                handler.set_option(legend_id + 1, legend_t, legend_c)
+                legend_artists[dummy] = handler
+                pass
+
+            plt.legend(
+                dummy_objects, hue_order, handler_map = legend_artists, ncol = legend_col,
+                loc = 'upper left', bbox_to_anchor = (1, 1), frameon = False
+            )
+
+        if annotate:
+            
+            scale_max_x = dpi * figsize[0] * res_factor
+            scale_max_y = dpi * figsize[1] * res_factor
+            min_x, min_y, max_x, max_y = np.min(df['x']), np.min(df['y']), np.max(df['x']), np.max(df['y'])
+
+            for legend_t, legend_c, legend_id in zip(
+                hue_order, adata.uns[f'{color}.colors'], range(len(hue_order))
+            ):
+                # calculate gravity for legend_t class.
+                xs = np.array(df.loc[df['label'] == legend_t, 'x'])
+                ys = np.array(df.loc[df['label'] == legend_t, 'y'])
+                
+                mx, my = np.mean(xs), np.mean(ys)
+                mx = scale_max_x * ((mx - min_x) / (max_x - min_x))
+                my = scale_max_y * ((my - min_y) / (max_y - min_y))
+                text = mtext.Text(
+                    x = mx, y = my, # fontproperties = 'bold',
+                    text = str(legend_id + 1) if annotate_style == 'index' else legend_t, 
+                    color = annotate_foreground,
+                    ha = 'center', va = 'center', size = annot_size
+                )
+                
+                text.set_path_effects([
+                    mpe.Stroke(linewidth = 3, foreground = annotate_stroke),
+                    mpe.Normal()
+                ])
+
+                ax.add_artist(text)
+                pass
         
+
     elif (type(labels[0]) is int) or \
          (type(labels[0]) is float) or \
          (type(labels[0]) is np.float32) or \
@@ -117,36 +207,49 @@ def embedding_atlas(
         
         agg = cvs.points(df, 'x', 'y', ds.mean('label'))
         legend_tag = False
-        if cmap in palettes.all_palettes.keys():
-            num = list(palettes.all_palettes[cmap].keys())[-1]
-            img = tf.shade(agg,cmap = palettes.all_palettes[cmap][num])
-        else: img = tf.shade(agg,cmap = cmap)
+
+        cmap = list(palettes.interp_palette(palettes.all_palettes[cmap][
+            list(palettes.all_palettes[cmap].keys())[-1]
+        ], 256)) if isinstance(cmap, str) else cmap
+
+        if cmap_reverse:
+            cmap = cmap[::-1]
+
+        # np.array([0.95, 0.95, 0.98, 1]) # a tint of bluish gray
+        if cmap_lower is not None: 
+            for ix in range(5): cmap[ix] = cmap_lower
+
+        cmap = listedcm(cmap)
+        img = tf.shade(agg,cmap = cmap)
+        
+        ax.imshow(
+            img.to_pil(), 
+            aspect = 'auto', 
+            interpolation = 'bicubic',
+            extent = (0, dpi * figsize[0] * res_factor, 0, dpi * figsize[1] * res_factor)
+        )
     
     else:
         warning('color label must be categorical (string) or numerical.')
         return None
     
-    if ax is None: fig, ax = plt.subplots(figsize=figsize)
-    else: fig = ax.figure
-    
-    ax.imshow(img.to_pil(), aspect = 'auto')
-    
+
     def format_coord(x, y): return f"x = {x:.2f}, y = {y:.2f}"
     ax.format_coord = format_coord
 
-    if legend_tag == True:
-        unique_labels = adata.obs[color].cat.categories
-        for label in unique_labels:
-            ax.scatter([], [], c = color_key[label], label = label)
-        
-        if legend_loc == 'right margin':
-            ax.legend(
-                frameon = False,
-                loc = 'center left',
-                bbox_to_anchor = (1, 0.5),
-                ncol = (1 if len(unique_labels) <= 14 else 2 if len(unique_labels) <= 30 else 3),
-                fontsize = fontsize,
-            )
+    # if legend_tag == True:
+    #     unique_labels = adata.obs[color].cat.categories
+    #     for label in unique_labels:
+    #         ax.scatter([], [], c = color_key[label], label = label)
+    #     
+    #     if legend_loc == 'right margin':
+    #         ax.legend(
+    #             frameon = False,
+    #             loc = 'center left',
+    #             bbox_to_anchor = (1, 0.5),
+    #             ncol = (1 if len(unique_labels) <= 14 else 2 if len(unique_labels) <= 30 else 3),
+    #             fontsize = fontsize,
+    #         )
 
     if frameon == False:
         ax.axis('off')
@@ -159,8 +262,6 @@ def embedding_atlas(
         ax.spines['bottom'].set_visible(True)
         ax.spines['top'].set_visible(False)
         ax.spines['right'].set_visible(False)
-        # ax.spines['bottom'].set_bounds(0,150)
-        # ax.spines['left'].set_bounds(0,150)
         ax.set_xlabel(f'{basis}.1',loc='left',fontsize = fontsize)
         ax.set_ylabel(f'{basis}.2',loc='bottom',fontsize = fontsize)
 
@@ -175,13 +276,15 @@ def embedding_atlas(
         ax.set_xlabel(f'{basis}.1',loc='center',fontsize=fontsize)
         ax.set_ylabel(f'{basis}.2',loc='center',fontsize=fontsize)
     
-    line_width = 1
+    line_width = 0.6
     ax.spines['left'].set_linewidth(line_width)
     ax.spines['bottom'].set_linewidth(line_width)
     if title is None: title = color
-    ax.set_title(title,fontsize = fontsize)
+    ax.set_title(title,fontsize = fontsize * 1.2)
+    ax.set_xlim(dpi * figsize[0] * -0.05 * res_factor, dpi * figsize[0] * 1.05 * res_factor)
+    ax.set_ylim(dpi * figsize[1] * -0.05 * res_factor, dpi * figsize[1] * 1.05 * res_factor)
 
-    return ax
+    return ax.figure
 
 
 def embedding(

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: exprmat
-Version: 0.1.5
+Version: 0.1.6
 Summary: routines to process expression matrices
 Home-page: https://github.com/yang-z-03/exprmat
 Author: Zheng Yang
@@ -28,6 +28,7 @@ Requires-Dist: datashader
 Requires-Dist: fa2_modified
 Requires-Dist: pynndescent
 Requires-Dist: gseapy
+Requires-Dist: sh
 Dynamic: author
 Dynamic: author-email
 Dynamic: home-page

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat.egg-info/SOURCES.txt

@@ -9,9 +9,9 @@ exprmat.egg-info/dependency_links.txt
 exprmat.egg-info/not-zip-safe
 exprmat.egg-info/requires.txt
 exprmat.egg-info/top_level.txt
-exprmat.egg-info/.ipynb_checkpoints/requires-checkpoint.txt
 exprmat/clustering/__init__.py
 exprmat/clustering/leiden.py
+exprmat/clustering/rogue.py
 exprmat/clustering/spectral.py
 exprmat/clustering/utils.py
 exprmat/cnv/__init__.py

{exprmat-0.1.5 → exprmat-0.1.6}/exprmat.egg-info/requires.txt

@@ -20,3 +20,4 @@ datashader
 fa2_modified
 pynndescent
 gseapy
+sh

{exprmat-0.1.5 → exprmat-0.1.6}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name                 = 'exprmat',
-    version              = '0.1.5',
+    version              = '0.1.6',
     description          = 'routines to process expression matrices',
     url                  = 'https://github.com/yang-z-03/exprmat',
     author               = 'Zheng Yang',
@@ -32,7 +32,8 @@ setup(
         'datashader',
         'fa2_modified',
         'pynndescent',
-        'gseapy'
+        'gseapy',
+        'sh'
     ],
     include_package_data = False,
     package_data         = {

exprmat-0.1.5/exprmat.egg-info/.ipynb_checkpoints/requires-checkpoint.txt

@@ -1,22 +0,0 @@
-umap-learn>=0.5.0
-scvi-tools>=1.3.0
-scanpy
-anndata
-mudata
-pandas
-numpy
-matplotlib
-seaborn
-torch>=2.0
-annoy
-scikit-image
-scikit-misc
-igraph
-bbknn
-scanorama
-harmonypy
-pymde
-datashader
-fa2_modified
-pynndescent
-gseapy