exprmat 0.1.40__tar.gz → 0.1.41__tar.gz

{exprmat-0.1.40 → exprmat-0.1.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: exprmat
-Version: 0.1.40
+Version: 0.1.41
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{exprmat-0.1.40 → exprmat-0.1.41}/exprmat/reader/experiment.py

@@ -44,10 +44,6 @@ class experiment:
         dump = '.', subset = None,
         version = em.SPECIFICATION
     ):
-        
-        from exprmat.reader.updater import update
-        version = update(meta, modalities, mudata, version)
-
         # TODO: we support rna only at present.
         table = meta.dataframe.to_dict(orient = 'list')
         self.mudata = mudata
@@ -265,6 +261,15 @@ class experiment:
             else: warning(f'sample {i_sample} have no supported modalities')
 
 
+        self.metadata.dataframe = self.metadata.dataframe.loc[
+            self.metadata.dataframe['modality'] != 'rna.splicing', :
+        ].copy()
+        
+        self.metadata.dataframe = self.metadata.dataframe.loc[
+            self.metadata.dataframe['modality'] != 'rna.tcr', :
+        ].copy()
+
+
         # attaching bulk datasets.
 
         for i_mod in ['rna-bulk', 'atac-bulk']:
@@ -894,6 +899,8 @@ class experiment:
                 p.imap_unordered(partial_task, args),
                 total = len(args), desc = 'processing anndata'
             ))
+
+            p.close()
             
             results = {}
             for data, key, out in output:
@@ -2439,6 +2446,17 @@ def load_experiment(direc, load_samples = True, load_subset = None):
     # read individual modality and sample
     meta = load_metadata(os.path.join(direc, 'metadata.tsv'))
 
+    # load specification
+    version = load_experiment_specification(direc)
+    
+    # update the object to the newest version
+    from exprmat.reader.updater import update
+    version = update(meta, direc, version)
+
+    meta.dataframe = meta.dataframe.loc[
+        ['.' in x for x in meta.dataframe['modality']], :
+    ].copy()
+
     modalities = {}
     if load_samples:
         for modal, samp in zip(meta.dataframe['modality'], meta.dataframe['sample']):
@@ -2459,9 +2477,6 @@ def load_experiment(direc, load_samples = True, load_subset = None):
         if os.path.exists(os.path.join(direc, 'subsets', load_subset + '.h5mu')):
             mdata = mu.read_h5mu(os.path.join(direc, 'subsets', load_subset + '.h5mu'))
 
-    # load specification
-    version = load_experiment_specification(direc)
-    
     expr = experiment(
         meta = meta, 
         mudata = mdata, 

{exprmat-0.1.40 → exprmat-0.1.41}/exprmat/reader/static.py

@@ -182,16 +182,17 @@ def rna_scvi(
     # save the model
     import os
     if sample_name == 'integrated':
-        attempt = os.path.join(savepath, 'scvi')
-    else: attempt = os.path.join(savepath, 'rna', 'scvi', sample_name)
+        attempt = os.path.join(savepath, 'scvi', key_added)
+    else: attempt = os.path.join(savepath, 'rna', 'scvi', sample_name, key_added)
     
     os.makedirs(attempt, exist_ok = True)
     model.save(attempt, overwrite = True)
     adata.obsm[key_added] = latent
 
     adata.uns[key_added] = {
-        'model': savepath,
+        'model': attempt,
         'n_latent': n_comps,
+        'batch_key': batch,
         'type': 'scvi'
     }
     

exprmat-0.1.41/exprmat/reader/updater.py

@@ -0,0 +1,111 @@
+
+import os
+import h5py
+from anndata.io import read_elem, write_elem
+
+import exprmat as em
+from exprmat.ansi import error, warning, info
+
+VERSION_CHECKPOINT = [1, em.SPECIFICATION]
+
+
+def update(meta, dump, version):
+
+    # current version
+    if version == em.SPECIFICATION: return version
+
+    # valid version
+    elif version in VERSION_CHECKPOINT:
+        ckpt = VERSION_CHECKPOINT.index(version)
+        for taskid in range(ckpt, len(VERSION_CHECKPOINT) - 1):
+            task = UPDATE_TASK[taskid]
+            info(f'updating experiment dump from {version} to {VERSION_CHECKPOINT[taskid + 1]} ...')
+            
+            # update samples
+            for irow in range(len(meta.dataframe)):
+                metadata = meta.dataframe.iloc[irow, :].copy()
+                if '.' in metadata['modality']: continue
+                info(f'updating sample ./{metadata["modality"]}/{metadata["sample"]} ...')
+                task(metadata['modality'], os.path.join(dump, metadata["modality"], metadata["sample"] + '.h5ad'))
+            
+            # update subsets
+            if os.path.exists(os.path.join(dump, 'subsets')):
+                for subset in os.listdir(os.path.join(dump, 'subsets')):
+                    if not subset.endswith('.h5mu'): continue
+                    if subset.startswith('.'): continue
+                    info(f'updating subset {subset.replace(".h5mu", "")} ...')
+                    task('subset', os.path.join(dump, 'subsets', subset))
+            
+            if os.path.exists(os.path.join(dump, 'integrated.h5mu')):
+                info(f'updating integrated dataset ...')
+                task('integrated', os.path.join(dump, 'integrated.h5mu'))
+
+
+        return VERSION_CHECKPOINT[taskid + 1]
+    
+    # invalid version
+    else: error(f'illegal version specification of experiment dump: [{version}]')
+
+
+def update_39(mod, fpath):
+
+    def update_rna_var(table):
+        if 'ensembl' in table.columns:
+            import pandas as pd
+            obsn = table[['ensembl']].copy()
+            taxa = obsn.index.tolist()[0].split(':')[1]
+            from exprmat.data.finders import get_genome
+            gtable = get_genome(taxa)
+            gtable.index = gtable['id'].tolist()
+            obsn = obsn.join(gtable, on = 'ensembl', how = 'left')
+            assert len(obsn) == len(table)
+            del obsn['ensembl']
+
+            old_columns = [
+                '.seqid', '.source', '.type', '.start', '.end', '.score', '.strand', '.phase',
+                'biotype', 'source', 'gene', 'description', 'ensembl', 'uid', 'version', 'len.mean',
+                'len.median', 'len.cds', '.ucsc'
+            ]
+
+            for o in old_columns:
+                if o in table.columns: del table[o]
+            return pd.concat([obsn, table], axis = 1)
+
+    def update_atac_var(table):
+        return table.rename(columns = {
+            '.seqid': 'chr',
+            '.start': 'start',
+            '.end': 'end',
+        }, inplace = False)
+
+
+    file = h5py.File(fpath, 'r+')
+
+    if mod in ['subset', 'integrated']:
+        for m in list(file['mod'].keys()):
+            if m in ['rna', 'rnasp-b', 'rnasp-c', 'rnasp-s', 'atac-g']:
+                table = read_elem(file['mod'][m]['var'])
+                table = update_rna_var(table)
+                write_elem(file['mod'][m], 'var', table)
+            
+            elif m in ['atac', 'atac-p']:
+                table = read_elem(file['mod'][m]['var'])
+                table = update_atac_var(table)
+                write_elem(file['mod'][m], 'var', table)
+    
+    if mod in ['rna', 'rnasp-b', 'rnasp-c', 'rnasp-s', 'atac-g']:
+        table = read_elem(file['var'])
+        table = update_rna_var(table)
+        write_elem(file, 'var', table)
+    
+    elif mod in ['atac', 'atac-p']:
+        table = read_elem(file['var'])
+        table = update_atac_var(table)
+        write_elem(file, 'var', table)
+    
+    file.close()
+
+
+UPDATE_TASK = [
+    update_39
+]

{exprmat-0.1.40 → exprmat-0.1.41}/exprmat/reduction/scvi.py

@@ -57,29 +57,27 @@ def scvi(
     )
 
     import warnings
-    import contextlib
     with warnings.catch_warnings():
         warnings.simplefilter('ignore')
-        with contextlib.redirect_stderr(None):
-            model.train(
-                max_epochs = max_epochs, 
-                accelerator = accelerator, 
-                devices = devices, 
-                train_size = train_size, 
-                validation_size = validation_size, 
-                shuffle_set_split = shuffle_set_split, 
-                load_sparse_tensor = load_sparse_tensor, 
-                batch_size = batch_size, 
-                early_stopping = early_stopping, 
-                datasplitter_kwargs = datasplitter_kwargs, 
-                plan_kwargs = plan_kwargs, 
-                datamodule = datamodule,
-                enable_progress_bar = enable_progress_bar, 
-                progress_bar_refresh_rate = progress_bar_refresh_rate, 
-                simple_progress_bar = simple_progress_bar, 
-                logger = logger, 
-                log_every_n_steps = log_every_n_steps,
-            )
+        model.train(
+            max_epochs = max_epochs, 
+            accelerator = accelerator, 
+            devices = devices, 
+            train_size = train_size, 
+            validation_size = validation_size, 
+            shuffle_set_split = shuffle_set_split, 
+            load_sparse_tensor = load_sparse_tensor, 
+            batch_size = batch_size, 
+            early_stopping = early_stopping, 
+            datasplitter_kwargs = datasplitter_kwargs, 
+            plan_kwargs = plan_kwargs, 
+            datamodule = datamodule,
+            enable_progress_bar = enable_progress_bar, 
+            progress_bar_refresh_rate = progress_bar_refresh_rate, 
+            simple_progress_bar = simple_progress_bar, 
+            logger = logger, 
+            log_every_n_steps = log_every_n_steps,
+        )
 
     latent = model.get_latent_representation()
     return model, latent

{exprmat-0.1.40 → exprmat-0.1.41}/pyproject.toml

@@ -20,7 +20,7 @@ description = "exprmat: Routines for expression matrices"
 readme = "readme.md"
 license = {file = "license"}
 requires-python = ">=3.9, <3.13"
-version = "0.1.40"
+version = "0.1.41"
 authors = [
     {name = "Zheng Yang", email = "xornent@outlook.com"}
 ]

exprmat-0.1.40/exprmat/reader/updater.py

@@ -1,83 +0,0 @@
-
-import exprmat as em
-from exprmat.ansi import error, warning, info
-
-VERSION_CHECKPOINT = [1, em.SPECIFICATION]
-
-
-def update(meta, modalities, mudata, version):
-
-    # current version
-    if version == em.SPECIFICATION: return version
-
-    # valid version
-    elif version in VERSION_CHECKPOINT:
-        ckpt = VERSION_CHECKPOINT.index(version)
-        for taskid in range(ckpt, len(VERSION_CHECKPOINT) - 1):
-            task = UPDATE_TASK[taskid]
-            info(f'updating experiment dump from {version} to {VERSION_CHECKPOINT[taskid + 1]} ...')
-            task(meta, modalities, mudata)
-
-        return VERSION_CHECKPOINT[taskid + 1]
-    
-    # invalid version
-    else: error(f'illegal version specification of experiment dump: [{version}]')
-
-
-def do_for(modalities, mudata, modality, func, **params):
-
-    if modalities:
-        if modality in modalities.keys():
-            for k in modalities[modality].keys():
-                func(modalities[modality][k], **params)
-    if mudata:
-        if modality in mudata.mod.keys():
-            func(mudata.mod[modality], **params)
-
-    
-def update_1_39(meta, modalities, mudata):
-
-    def update_rna(data):
-
-        if 'ensembl' in data.var.columns:
-            import pandas as pd
-            obsn = data.var[['ensembl']].copy()
-            taxa = obsn.index.tolist()[0].split(':')[1]
-            from exprmat.data.finders import get_genome
-            gtable = get_genome(taxa)
-            gtable.index = gtable['id'].tolist()
-            obsn = obsn.join(gtable, on = 'ensembl', how = 'left')
-            assert len(obsn) == data.n_vars
-            del obsn['ensembl']
-
-            old_columns = [
-                '.seqid', '.source', '.type', '.start', '.end', '.score', '.strand', '.phase',
-                'biotype', 'source', 'gene', 'description', 'ensembl', 'uid', 'version', 'len.mean',
-                'len.median', 'len.cds', '.ucsc'
-            ]
-
-            for o in old_columns:
-                if o in data.var.columns: del data.var[o]
-            data.var = pd.concat([obsn, data.var], axis = 1)
-
-    def update_atac(data):
-        data.var.rename(columns = {
-            '.seqid': 'chr',
-            '.start': 'start',
-            '.end': 'end',
-        }, inplace = True)
-
-    do_for(modalities, mudata, 'rna', update_rna)
-    do_for(modalities, mudata, 'rnasp-c', update_rna)
-    do_for(modalities, mudata, 'rnasp-b', update_rna)
-    do_for(modalities, mudata, 'rnasp-s', update_rna)
-    do_for(modalities, mudata, 'atac-g', update_rna)
-
-    do_for(modalities, mudata, 'atac', update_atac)
-    do_for(modalities, mudata, 'atac-p', update_atac)
-    pass
-
-
-UPDATE_TASK = [
-    update_1_39
-]