SURE-tools 2.2.10__tar.gz → 2.2.28__tar.gz

{sure_tools-2.2.10 → sure_tools-2.2.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SURE-tools
-Version: 2.2.10
+Version: 2.2.28
 Summary: Succinct Representation of Single Cells
 Home-page: https://github.com/ZengFLab/SURE
 Author: Feng Zeng

{sure_tools-2.2.10 → sure_tools-2.2.28}/SURE/DensityFlow.py

@@ -59,12 +59,13 @@ class DensityFlow(nn.Module):
                  input_size: int,
                  codebook_size: int = 200,
                  cell_factor_size: int = 0,
+                 turn_off_cell_specific: bool = False,
                  supervised_mode: bool = False,
                  z_dim: int = 10,
                  z_dist: Literal['normal','studentt','laplacian','cauchy','gumbel'] = 'gumbel',
-                 loss_func: Literal['negbinomial','poisson','multinomial','bernoulli'] = 'negbinomial',
+                 loss_func: Literal['negbinomial','poisson','multinomial','bernoulli'] = 'multinomial',
                  inverse_dispersion: float = 10.0,
-                 use_zeroinflate: bool = True,
+                 use_zeroinflate: bool = False,
                  hidden_layers: list = [500],
                  hidden_layer_activation: Literal['relu','softplus','leakyrelu','linear'] = 'relu',
                  nn_dropout: float = 0.1,
@@ -102,6 +103,7 @@ class DensityFlow(nn.Module):
         else:
             self.use_bias = [not zero_bias] * self.cell_factor_size
         #self.use_bias = not zero_bias
+        self.turn_off_cell_specific = turn_off_cell_specific
         
         self.codebook_weights = None
 
@@ -203,27 +205,51 @@ class DensityFlow(nn.Module):
             self.cell_factor_effect = nn.ModuleList()
             for i in np.arange(self.cell_factor_size):
                 if self.use_bias[i]:
-                    self.cell_factor_effect.append(MLP(
-                        [self.latent_dim+1] + self.decoder_hidden_layers + [self.latent_dim],
-                        activation=activate_fct,
-                        output_activation=None,
-                        post_layer_fct=post_layer_fct,
-                        post_act_fct=post_act_fct,
-                        allow_broadcast=self.allow_broadcast,
-                        use_cuda=self.use_cuda,
+                    if self.turn_off_cell_specific:
+                        self.cell_factor_effect.append(MLP(
+                            [1] + self.decoder_hidden_layers + [self.latent_dim],
+                            activation=activate_fct,
+                            output_activation=None,
+                            post_layer_fct=post_layer_fct,
+                            post_act_fct=post_act_fct,
+                            allow_broadcast=self.allow_broadcast,
+                            use_cuda=self.use_cuda,
+                            )
+                        )
+                    else:
+                        self.cell_factor_effect.append(MLP(
+                            [self.latent_dim+1] + self.decoder_hidden_layers + [self.latent_dim],
+                            activation=activate_fct,
+                            output_activation=None,
+                            post_layer_fct=post_layer_fct,
+                            post_act_fct=post_act_fct,
+                            allow_broadcast=self.allow_broadcast,
+                            use_cuda=self.use_cuda,
+                            )
                         )
-                    )
                 else:
-                    self.cell_factor_effect.append(ZeroBiasMLP(
-                        [self.latent_dim+1] + self.decoder_hidden_layers + [self.latent_dim],
-                        activation=activate_fct,
-                        output_activation=None,
-                        post_layer_fct=post_layer_fct,
-                        post_act_fct=post_act_fct,
-                        allow_broadcast=self.allow_broadcast,
-                        use_cuda=self.use_cuda,
+                    if self.turn_off_cell_specific:
+                        self.cell_factor_effect.append(ZeroBiasMLP(
+                            [1] + self.decoder_hidden_layers + [self.latent_dim],
+                            activation=activate_fct,
+                            output_activation=None,
+                            post_layer_fct=post_layer_fct,
+                            post_act_fct=post_act_fct,
+                            allow_broadcast=self.allow_broadcast,
+                            use_cuda=self.use_cuda,
+                            )
+                        )
+                    else:
+                        self.cell_factor_effect.append(ZeroBiasMLP(
+                            [self.latent_dim+1] + self.decoder_hidden_layers + [self.latent_dim],
+                            activation=activate_fct,
+                            output_activation=None,
+                            post_layer_fct=post_layer_fct,
+                            post_act_fct=post_act_fct,
+                            allow_broadcast=self.allow_broadcast,
+                            use_cuda=self.use_cuda,
+                            )
                         )
-                    )
             
         self.decoder_concentrate = MLP(
                     [self.latent_dim] + self.decoder_hidden_layers + [self.input_size],
@@ -234,16 +260,6 @@ class DensityFlow(nn.Module):
                     allow_broadcast=self.allow_broadcast,
                     use_cuda=self.use_cuda,
                 )
-        if self.loss_func == 'negbinomial':
-            self.decoder_total_count = MLP(
-                    [self.latent_dim] + self.decoder_hidden_layers + [self.input_size],
-                    activation=activate_fct,
-                    output_activation=Exp,
-                    post_layer_fct=post_layer_fct,
-                    post_act_fct=post_act_fct,
-                    allow_broadcast=self.allow_broadcast,
-                    use_cuda=self.use_cuda,
-                )
 
         if self.latent_dist == 'studentt':
             self.codebook = MLP(
@@ -324,9 +340,9 @@ class DensityFlow(nn.Module):
         batch_size = xs.size(0)
         self.options = dict(dtype=xs.dtype, device=xs.device)
         
-        #if self.loss_func=='negbinomial':
-        #    total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
-        #                             xs.new_ones(self.input_size), constraint=constraints.positive)
+        if self.loss_func=='negbinomial':
+            total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
+                                     xs.new_ones(self.input_size), constraint=constraints.positive)
             
         if self.use_zeroinflate:
             gate_logits = pyro.param("dropout_rate", xs.new_zeros(self.input_size))
@@ -370,7 +386,6 @@ class DensityFlow(nn.Module):
                     rate = theta * torch.sum(xs, dim=1, keepdim=True)
 
             if self.loss_func == 'negbinomial':
-                total_count = self.decoder_total_count(zs)
                 if self.use_zeroinflate:
                     pyro.sample('x', dist.ZeroInflatedDistribution(dist.NegativeBinomial(total_count=total_count, probs=theta),gate_logits=gate_logits).to_event(1), obs=xs)
                 else:
@@ -404,9 +419,9 @@ class DensityFlow(nn.Module):
         batch_size = xs.size(0)
         self.options = dict(dtype=xs.dtype, device=xs.device)
         
-        #if self.loss_func=='negbinomial':
-        #    total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
-        #                             xs.new_ones(self.input_size), constraint=constraints.positive)
+        if self.loss_func=='negbinomial':
+            total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
+                                     xs.new_ones(self.input_size), constraint=constraints.positive)
             
         if self.use_zeroinflate:
             gate_logits = pyro.param("dropout_rate", xs.new_zeros(self.input_size))
@@ -455,7 +470,6 @@ class DensityFlow(nn.Module):
                     rate = theta * torch.sum(xs, dim=1, keepdim=True)
 
             if self.loss_func == 'negbinomial':
-                total_count = self.decoder_total_count(zs)
                 if self.use_zeroinflate:
                     pyro.sample('x', dist.ZeroInflatedDistribution(dist.NegativeBinomial(total_count=total_count, probs=theta),gate_logits=gate_logits).to_event(1), obs=xs)
                 else:
@@ -489,9 +503,9 @@ class DensityFlow(nn.Module):
         batch_size = xs.size(0)
         self.options = dict(dtype=xs.dtype, device=xs.device)
         
-        #if self.loss_func=='negbinomial':
-        #    total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
-        #                             xs.new_ones(self.input_size), constraint=constraints.positive)
+        if self.loss_func=='negbinomial':
+            total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
+                                     xs.new_ones(self.input_size), constraint=constraints.positive)
             
         if self.use_zeroinflate:
             gate_logits = pyro.param("dropout_rate", xs.new_zeros(self.input_size))
@@ -552,7 +566,6 @@ class DensityFlow(nn.Module):
                     rate = theta * torch.sum(xs, dim=1, keepdim=True)
 
             if self.loss_func == 'negbinomial':
-                total_count = self.decoder_total_count(zs)
                 if self.use_zeroinflate:
                     pyro.sample('x', dist.ZeroInflatedDistribution(dist.NegativeBinomial(total_count=total_count, probs=theta),gate_logits=gate_logits).to_event(1), obs=xs)
                 else:
@@ -586,9 +599,9 @@ class DensityFlow(nn.Module):
         batch_size = xs.size(0)
         self.options = dict(dtype=xs.dtype, device=xs.device)
         
-        #if self.loss_func=='negbinomial':
-        #    total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
-        #                             xs.new_ones(self.input_size), constraint=constraints.positive)
+        if self.loss_func=='negbinomial':
+            total_count = pyro.param("inverse_dispersion", self.inverse_dispersion *
+                                     xs.new_ones(self.input_size), constraint=constraints.positive)
             
         if self.use_zeroinflate:
             gate_logits = pyro.param("dropout_rate", xs.new_zeros(self.input_size))
@@ -659,7 +672,6 @@ class DensityFlow(nn.Module):
                     rate = theta * torch.sum(xs, dim=1, keepdim=True)
 
             if self.loss_func == 'negbinomial':
-                total_count = self.decoder_total_count(zs)
                 if self.use_zeroinflate:
                     pyro.sample('x', dist.ZeroInflatedDistribution(dist.NegativeBinomial(total_count=total_count, probs=theta),gate_logits=gate_logits).to_event(1), obs=xs)
                 else:
@@ -690,9 +702,17 @@ class DensityFlow(nn.Module):
         zus = None
         for i in np.arange(self.cell_factor_size):
             if i==0:
-                zus = self.cell_factor_effect[i]([zns,us[:,i].reshape(-1,1)])
+                #if self.turn_off_cell_specific:
+                #    zus = self.cell_factor_effect[i](us[:,i].reshape(-1,1))
+                #else:
+                #    zus = self.cell_factor_effect[i]([zns,us[:,i].reshape(-1,1)])
+                zus = self._cell_response(zns, i, us[:,i].reshape(-1,1))
             else:
-                zus = zus + self.cell_factor_effect[i]([zns,us[:,i].reshape(-1,1)])
+                #if self.turn_off_cell_specific:
+                #    zus = zus + self.cell_factor_effect[i](us[:,i].reshape(-1,1))
+                #else:
+                #    zus = zus + self.cell_factor_effect[i]([zns,us[:,i].reshape(-1,1)])
+                zus = zus + self._cell_response(zns, i, us[:,i].reshape(-1,1))
         return zus
                         
     def _get_codebook_identity(self):
@@ -710,7 +730,7 @@ class DensityFlow(nn.Module):
         """
         Return the mean part of metacell codebook
         """
-        cb = self._get_metacell_coordinates()
+        cb = self._get_codebook()
         cb = tensor_to_numpy(cb)
         return cb
 
@@ -834,12 +854,12 @@ class DensityFlow(nn.Module):
             us_i = us[:,pert_idx].reshape(-1,1)
             
             # factor effect of xs 
-            dzs0 = self.get_cell_response(xs, factor_idx=pert_idx, perturb=us_i)
+            dzs0 = self.get_cell_response(zs, perturb_idx=pert_idx, perturb_us=us_i)
             
             # perturbation effect 
             ps = np.ones_like(us_i)
             if np.sum(np.abs(ps-us_i))>=1:
-                dzs = self.get_cell_response(xs, factor_idx=pert_idx, perturb=ps)          
+                dzs = self.get_cell_response(zs, perturb_idx=pert_idx, perturb_us=ps)          
                 zs = zs + dzs0 + dzs
             else:
                 zs = zs + dzs0
@@ -856,47 +876,48 @@ class DensityFlow(nn.Module):
         
         return counts, zs
     
-    def _cell_response(self, xs, factor_idx, perturb):
+    def _cell_response(self, zs, perturb_idx, perturb):
         #zns,_ = self.encoder_zn(xs)    
-        zns,_ = self._get_basal_embedding(xs)   
+        #zns,_ = self._get_basal_embedding(xs)   
+        zns = zs
         if perturb.ndim==2:
-            ms = self.cell_factor_effect[factor_idx]([zns, perturb])
+            if self.turn_off_cell_specific:
+                ms = self.cell_factor_effect[perturb_idx](perturb)
+            else:
+                ms = self.cell_factor_effect[perturb_idx]([zns, perturb])
         else:
-            ms = self.cell_factor_effect[factor_idx]([zns, perturb.reshape(-1,1)])
+            if self.turn_off_cell_specific:
+                ms = self.cell_factor_effect[perturb_idx](perturb.reshape(-1,1))
+            else:
+                ms = self.cell_factor_effect[perturb_idx]([zns, perturb.reshape(-1,1)])
             
         return ms 
 
     def get_cell_response(self, 
-                             xs, 
-                             factor_idx,
-                             perturb,
+                             zs, 
+                             perturb_idx,
+                             perturb_us,
                              batch_size: int = 1024):
         """
         Return cells' changes in the latent space induced by specific perturbation of a factor
 
         """
-        xs = self.preprocess(xs)
-        xs = convert_to_tensor(xs, device=self.get_device())
-        ps = convert_to_tensor(perturb, device=self.get_device())
-        dataset = CustomDataset2(xs,ps)
+        #xs = self.preprocess(xs)
+        zs = convert_to_tensor(zs, device=self.get_device())
+        ps = convert_to_tensor(perturb_us, device=self.get_device())
+        dataset = CustomDataset2(zs,ps)
         dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
 
         Z = []
         with tqdm(total=len(dataloader), desc='', unit='batch') as pbar:
-            for X_batch, P_batch, _ in dataloader:
-                zns = self._cell_response(X_batch, factor_idx, P_batch)
+            for Z_batch, P_batch, _ in dataloader:
+                zns = self._cell_response(Z_batch, perturb_idx, P_batch)
                 Z.append(tensor_to_numpy(zns))
                 pbar.update(1)
 
         Z = np.concatenate(Z)
         return Z
     
-    def get_metacell_response(self, factor_idx, perturb):
-        zs = self._get_codebook()
-        ps = convert_to_tensor(perturb, device=self.get_device())         
-        ms = self.cell_factor_effect[factor_idx]([zs,ps])
-        return tensor_to_numpy(ms)
-    
     def _get_expression_response(self, delta_zs):
         return self.decoder_concentrate(delta_zs)
     
@@ -1357,5 +1378,4 @@ def main():
 
 
 if __name__ == "__main__":
-
     main()

{sure_tools-2.2.10 → sure_tools-2.2.28}/SURE/perturb/perturb.py

@@ -1,5 +1,6 @@
 import re
 import numpy as np
+import pandas as pd
 from numba import njit
 from itertools import chain
 from joblib import Parallel, delayed
@@ -8,6 +9,8 @@ from typing import Literal
 class LabelMatrix:
     def __init__(self):
         self.labels_ = None
+        self.control_label = None 
+        self.sep_pattern = None
     
     def fit_transform(self, labels, control_label=None, sep_pattern=r'[,;_\s]', speedup: Literal['none','vectorize','parallel']='none'):
         if speedup=='none':
@@ -24,8 +27,31 @@ class LabelMatrix:
             mat = np.delete(mat, idx, axis=1)
             self.labels_ = np.delete(self.labels_, idx)
             
+        self.control_label = control_label
+        self.sep_pattern=sep_pattern
+        
         return mat
-            
+    
+    def transform(self, labels, speedup: Literal['none','vectorize','parallel']='none'):
+        sep_pattern = self.sep_pattern
+        if speedup=='none':
+            mat, labels_ = label_to_matrix(labels=labels, sep_pattern=sep_pattern)
+        elif speedup=='vectorize':
+            mat, labels_ = vectorized_label_to_matrix(labels=labels, sep_pattern=sep_pattern)
+        elif speedup=='parallel':
+            mat, labels_ = parallel_label_to_matrix(labels=labels, sep_pattern=sep_pattern)
+          
+        mat_df = pd.DataFrame(mat, columns=labels_)
+        
+        labels_valid = [x for x in labels_ if x in self.labels_]  
+        mat_df = mat_df[labels_valid]
+        
+        mat_valid = np.zeros([mat.shape[0], len(self.labels_)])
+        mat_valid_df = pd.DataFrame(mat_valid, columns=self.labels_)
+        mat_valid_df[labels_valid] = mat_df 
+        
+        return mat_valid_df.values
+        
     def inverse_transform(self, matrix):
         return matrix_to_labels(matrix=matrix, unique_labels=self.labels_)
 

{sure_tools-2.2.10 → sure_tools-2.2.28}/SURE/utils/custom_mlp.py

@@ -240,9 +240,15 @@ class ZeroBiasMLP(nn.Module):
         y = self.mlp(x)
         mask = torch.zeros_like(y)
         if len(y.shape)==2:
-            mask[x[1][:,0]>0,:] = 1
+            if type(x)==list:
+                mask[x[1][:,0]>0,:] = 1
+            else:
+                mask[x[:,0]>0,:] = 1
         elif len(y.shape)==3:
-            mask[:,x[1][:,0]>0,:] = 1
+            if type(x)==list:
+                mask[:,x[1][:,0]>0,:] = 1
+            else:
+                mask[:,x[:,0]>0,:] = 1
         return y*mask
         
 

{sure_tools-2.2.10 → sure_tools-2.2.28}/SURE_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SURE-tools
-Version: 2.2.10
+Version: 2.2.28
 Summary: Succinct Representation of Single Cells
 Home-page: https://github.com/ZengFLab/SURE
 Author: Feng Zeng

{sure_tools-2.2.10 → sure_tools-2.2.28}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setup(
     name='SURE-tools',
-    version='2.2.10',
+    version='2.2.28',
     description='Succinct Representation of Single Cells',
     long_description=long_description,
     long_description_content_type="text/markdown",