scte-quant 1.3.3__py3-none-any.whl

scTE/__init__.py

@@ -0,0 +1,11 @@
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("scTE")
+except PackageNotFoundError:
+    # package is not installed
+    __version__ = "unknown"
+
+from .miniglbase import genelist, location, glload
+
+__all__ = ["genelist", "location", "glload",]

scTE/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '1.3.3'
+__version_tuple__ = version_tuple = (1, 3, 3)
+
+__commit_id__ = commit_id = None

scTE/annotation.py

@@ -0,0 +1,158 @@
+import os,sys,gzip,time
+import numpy as np
+from scTE.miniglbase import genelist, glload, location
+
+form ={'force_tsv': True, 'loc': 'location(chr=column[0], left=column[1], right=column[2])', 'annot': 3}
+
+def cleanexon(filename, genefilename, exons):
+    if not os.path.exists('%s_scTEtmp/index'%filename):
+        os.system('mkdir -p %s_scTEtmp/index'%filename)
+
+    oh=gzip.open('%s_scTEtmp/index/%s.bed.gz'%(filename,genefilename),'wt')
+    for k in sorted(exons):
+        E=[]
+        chr_val = exons[k][0][0]  # get chromosome from first exon
+        for it in exons[k]:
+            E+=list(range(it[1],it[2]))
+        E=sorted(set(E))
+
+        s=0
+        tmp=[]
+        for id in range(0,len(E)-1):
+            if E[id+1]-E[id] >1:
+                en=id
+                tmp.append([E[s],E[en]])
+                s=en+1
+        tmp.append([E[s],E[id+1]])
+
+        for item in tmp:
+            oh.write('%s\t%s\t%s\t%s\n'%(chr_val,item[0],item[1],k))
+    oh.close()
+
+def annoGtf(filename, genefile, tefile, mode):
+
+    genefilename = genefile.split('/')[-1:][0].replace('.gtf','').replace('.gz','')
+    tefilename = tefile.split('/')[-1:][0].replace('.bed','').replace('.gz','')
+
+    raw = {}
+    clean = {}
+    if '.gz' in genefile:
+        o = gzip.open(genefile,'rb')
+    else:
+        o=open(genefile,'r')
+    for l in o:
+        if '.gz' in genefile:
+            l=l.decode('ascii')
+        if l.startswith('#'):
+            continue
+        t=l.strip().split('\t')
+        if t[2]=='exon' or t[2]=='UTR':
+            chr = t[0].replace('chr','')
+            left = int(t[3])
+            riht =  int(t[4])
+            name=t[8].split('gene_name "')[1].split('";')[0]
+
+            if name not in raw:
+                raw[name] = []
+            raw[name].append([chr,left,riht])
+
+            if 'protein_coding' not in l and 'lincRNA' not in l:
+                continue
+            if name not in clean:
+                clean[name] = []
+            clean[name].append([chr,left,riht])
+    o.close()
+
+    cleanexon(filename,'%s.raw'%genefilename,raw)
+    cleanexon(filename,'%s.clean'%genefilename,clean)
+
+    if mode == 'exclusive':
+        gene ={}
+        o = gzip.open('%s_scTEtmp/index/%s.clean.bed.gz'%(filename,genefilename),'rb')
+        for l in o:
+            t = l.decode('ascii').strip().split('\t')
+            chr = t[0].replace('chr','')
+            left = int(t[1])
+            rite = int(t[2])
+
+            left_buck = int((left-1)/10000) * 10000
+            right_buck = int((rite)/10000) * 10000
+            buckets_reqd = range(left_buck, right_buck+10000, 10000)
+
+            if chr not in gene:
+                gene[chr] = {}
+
+            if buckets_reqd:
+                for buck in buckets_reqd:
+                    if buck not in gene[chr]:
+                        gene[chr][buck] = []
+                    gene[chr][buck].append([left, rite])
+        o.close()
+
+        noverlap = []
+        if '.gz' in tefile:
+            o = gzip.open(tefile,'rb')
+        else:
+            o = open(tefile,'r')
+        for n,l in enumerate(o):
+            if '.gz' in tefile:
+                l = l.decode('ascii')
+            t = l.strip().split('\t')
+            chr = t[0]
+            left = int(t[1])
+            rite = int(t[2])
+            
+            if chr not in gene:
+                noverlap.append('%s\t%s\t%s\t%s\n'%(chr,left,rite,t[3]))
+                continue
+            
+            left_buck = int((left-1)/10000) * 10000
+            right_buck = int((rite)/10000) * 10000
+            buckets_reqd = range(left_buck, right_buck+10000, 10000)
+
+            if buckets_reqd:
+                i = 1
+                for buck in buckets_reqd:
+                    if buck not in gene[chr]:
+                        pass
+                    else:
+                        for k in gene[chr][buck]:
+                            if left < k[1] and rite > k[0]:
+                                i = 0
+                                break
+                        if i == 0:
+                            break
+                if i == 1:
+                    noverlap.append('%s\t%s\t%s\t%s\n'%(chr,left,rite,t[3]))
+
+        oh = gzip.open('%s_scTEtmp/index/%s.exclusive.gz'%(filename, tefilename),'wt')
+        for k in noverlap:
+            oh.write(k)
+        oh.close()
+
+        genes = genelist('%s_scTEtmp/index/%s.raw.bed.gz'%(filename, genefilename), format=form, gzip=True)
+        TEs = genelist('%s_scTEtmp/index/%s.exclusive.gz'%(filename, tefilename), format=form, gzip=True)
+        print(genes)
+        print(TEs)
+        
+        all_annot = genes + TEs
+        all_annot.save('%s_scTEtmp/index/custome.exclusive.glb'%filename)
+        annot = '%s_scTEtmp/index/custome.exclusive.glb'%filename
+
+    elif mode == 'inclusive':
+        genes = genelist('%s_scTEtmp/index/%s.raw.bed.gz'%(filename,genefilename), format=form, gzip=True)
+        if tefilename.endswith('.gz'):
+            TEs = genelist(tefile, format=form, gzip=True)
+        else:
+            TEs = genelist(tefile, format=form)
+
+        all_annot = genes + TEs
+        all_annot.save('%s_scTEtmp/index/custome.inclusive.glb'%filename)
+        annot = '%s_scTEtmp/index/custome.inclusive.glb'%filename
+
+    return annot
+
+
+
+
+

scTE/base.py

@@ -0,0 +1,473 @@
+import pandas as pd
+import multiprocessing
+import argparse
+from functools import partial
+import logging
+logging.getLogger("zarr").setLevel(logging.WARNING)
+import os, sys, glob, datetime, time, gzip
+import collections
+from collections import defaultdict
+from math import log
+from scTE.miniglbase import genelist, glload, location
+from scTE.annotation import annoGtf
+import subprocess
+
+import numpy as np
+import scipy
+import anndata as ad
+
+def read_opts(parser):
+    args = parser.parse_args()
+    if args.format == "BAM" :
+        args.parser = "BAM"
+    elif args.format == "SAM" :
+        args.parser = "SAM"
+    else :
+        logging.error("The input file must be SAM/BAM format: %s !\n" % (args.format))
+        sys.exit(1)
+    
+    args.error = logging.critical
+    args.warn = logging.warning
+    args.debug = logging.debug
+    args.info = logging.info
+    
+    args.argtxt ="\n".join(("Parameter list:", \
+                "Sample = %s" % (args.out), \
+                "Reference annotation index = %s" %(args.annoglb[0]), \
+                "Minimum number of genes required = %s" % (args.genenumber), \
+                "Minimum number of counts required = %s"% (args.countnumber),\
+                "Number of threads = %s " % (args.thread),\
+    ))
+    return args
+
+def Readanno(filename, annoglb): #genome
+    glannot = glload(annoglb)
+    allelement = set(glannot['annot'])
+#     if genome in ['mm10']:
+#         chr_list = ['chr'+ str(i) for i in range(1,20) ] + [ 'chrX','chrY', 'chrM' ]
+#     elif genome in ['hg38']:
+#         chr_list = ['chr'+ str(i) for i in range(1,23) ] + [ 'chrX','chrY', 'chrM' ]
+    
+    chr_list = list(set([ k['chr'] for k in glannot['loc']])) #this is useful for costume chromsome
+    return(allelement, chr_list, annoglb, glannot)
+
+def checkCBUMI(filename,out,CB,UMI):
+    def _run_and_wait(cmd, outfile, timeout=30):
+        """Run a command and wait for its output file with polling."""
+        subprocess.run(cmd, shell=True)
+        # Poll for file instead of fixed sleep
+        for _ in range(timeout):
+            if os.path.exists(outfile):
+                time.sleep(0.1)  # brief settle for file flush
+                return
+            time.sleep(1)
+        logging.error("Timeout waiting for output: %s" % outfile)
+        sys.exit(1)
+
+    if CB == 'CR':
+        _run_and_wait(
+            'samtools view %s | head -100| grep "CR:Z:" | wc -l > %s_scTEtmp/o1/testCR.txt' % (filename, out),
+            '%s_scTEtmp/o1/testCR.txt' % out)
+        with open('%s_scTEtmp/o1/testCR.txt' % out, 'r') as o:
+            for l in o:
+                l = l.strip()
+                if int(l) < 100:
+                    logging.error("The input file %s has no cell barcodes information, plese make sure the aligner have add the cell barcode key, or set CB to False" % filename)
+                    sys.exit(1)
+    elif CB == 'CB':
+        _run_and_wait(
+            'samtools view %s | head -100| grep "CB:Z:" | wc -l > %s_scTEtmp/o1/testCR.txt' % (filename, out),
+            '%s_scTEtmp/o1/testCR.txt' % out)
+        with open('%s_scTEtmp/o1/testCR.txt' % out, 'r') as o:
+            for l in o:
+                l = l.strip()
+                if int(l) < 100:
+                    logging.error("The input file %s has no cell barcodes information, plese make sure the aligner have add the cell barcode key, or set CB to False" % filename)
+                    sys.exit(1)
+
+    if UMI == 'UR':
+        _run_and_wait(
+            'samtools view %s | head -100| grep "UR:Z:" | wc -l > %s_scTEtmp/o1/testUMI.txt' % (filename, out),
+            '%s_scTEtmp/o1/testUMI.txt' % out)
+        with open('%s_scTEtmp/o1/testUMI.txt' % out, 'r') as o:
+            for l in o:
+                l = l.strip()
+                if int(l) < 100:
+                    logging.error("The input file %s has no UR:Z information, plese make sure the aligner have add the UMI key, or set UMI to False" % filename)
+                    sys.exit(1)
+    elif UMI == 'UB':
+        _run_and_wait(
+            'samtools view %s | head -100| grep "UB:Z:" | wc -l > %s_scTEtmp/o1/testUMI.txt' % (filename, out),
+            '%s_scTEtmp/o1/testUMI.txt' % out)
+        with open('%s_scTEtmp/o1/testUMI.txt' % out, 'r') as o:
+            for l in o:
+                l = l.strip()
+                if int(l) < 100:
+                    logging.error("The input file %s has no UB:Z information, plese make sure the aligner have add the UMI key, or set UMI to False" % filename)
+                    sys.exit(1)
+
+def Bam2bed(filename, CB, UMI, out, num_threads):
+    if not os.path.exists('%s_scTEtmp/o1'%out):
+        os.system('mkdir -p %s_scTEtmp/o1'%out)
+
+    sample=filename.split('/')[-1].replace('.bam','')
+    if sys.platform == 'darwin': # Mac OSX has BSD sed
+        switch = '-E'
+    else:
+        switch = '-r'
+
+    if UMI == 'False':
+        if CB == 'False':
+            # Put the sample name in the barcode slot
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{print $3,$4,$4+100,"%s"}\' | sed %s \'s/^chr//g\'| gzip -c > %s_scTEtmp/o1/%s.bed.gz' % (num_threads, filename, out, switch, out, out))
+        elif CB == 'CR':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CR:Z:/)n=i}{print $3,$4,$4+100,$n}\' | sed %s \'s/CR:Z://g\' | sed %s \'s/^chr//g\' | gzip -c > %s_scTEtmp/o1/%s.bed.gz' % (num_threads, filename, switch, switch, out, out))
+        elif CB == 'CB':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CB:Z:/)n=i}{print $3,$4,$4+100,$n}\' | sed %s \'s/CB:Z://g\' | sed %s \'s/^chr//g\' | gzip -c > %s_scTEtmp/o1/%s.bed.gz' % (num_threads, filename, switch, switch, out, out))
+    elif UMI == 'UR':
+        if CB == 'CR':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CR:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UR:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CR:Z://g\' | sed %s \'s/UR:Z://g\'| sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip -c > %s_scTEtmp/o1/%s.bed.gz' % (num_threads, filename, switch, switch, switch, out,out))
+        elif CB == 'CB':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CB:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UR:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CB:Z://g\' | sed %s \'s/UR:Z://g\'| sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip -c > %s_scTEtmp/o1/%s.bed.gz' % (num_threads, filename, switch, switch, switch, out,out))
+    elif UMI == 'UB':
+        if CB == 'CR':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CR:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UB:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CR:Z://g\' | sed %s \'s/UB:Z://g\'| sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip -c > %s_scTEtmp/o1/%s.bed.gz' % (num_threads, filename, switch, switch, switch, out,out))
+        elif CB == 'CB':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CB:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UB:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CB:Z://g\' | sed %s \'s/UB:Z://g\'| sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip -c > %s_scTEtmp/o1/%s.bed.gz' % (num_threads, filename, switch, switch, switch, out,out))
+
+def Para_bam2bed(filename, CB, UMI, out, num_threads=1):
+    if not os.path.exists('%s_scTEtmp/o0'%out):
+        os.system('mkdir -p %s_scTEtmp/o0'%out)
+
+    sample=filename.split('/')[-1].replace('.bam','')
+    
+    if sys.platform == 'darwin': # Mac OSX has BSD sed
+        switch = '-E'
+    else:
+        switch = '-r'
+    
+    if UMI == 'False':
+        if CB == 'False':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{print $3,$4,$4+100,"%s"}\' | sed %s \'s/^chr//g\' | gzip > %s_scTEtmp/o0/%s.bed.gz'%(num_threads, filename, sample, switch, out, sample))
+        elif CB == 'CR':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CR:Z:/)n=i}{print $3,$4,$4+100,$n}\' | sed %s \'s/CR:Z://g\' | sed %s \'s/^chr//g\' | gzip > %s_scTEtmp/o0/%s.bed.gz'%(num_threads, filename, switch, switch, out,sample))
+        elif CB == 'CB':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CB:Z:/)n=i}{print $3,$4,$4+100,$n}\' | sed %s \'s/CB:Z://g\' | sed %s \'s/^chr//g\' | gzip > %s_scTEtmp/o0/%s.bed.gz'%(num_threads, filename, switch, switch, out,sample))
+    elif UMI == 'UR':
+        if CB == 'CR':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CR:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UR:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CR:Z://g\' | sed %s \'s/UR:Z://g\' | sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip > %s_scTEtmp/o0/%s.bed.gz'%(num_threads, filename, switch, switch, switch, out,sample))
+        elif CB == 'CB':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CB:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UR:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CB:Z://g\' | sed %s \'s/UR:Z://g\' | sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip > %s_scTEtmp/o0/%s.bed.gz'%(num_threads, filename, switch, switch, switch, out,sample))
+    elif UMI == 'UB':
+        if CB == 'CR':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CR:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UB:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CR:Z://g\' | sed %s \'s/UB:Z://g\' | sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip > %s_scTEtmp/o0/%s.bed.gz'%(num_threads, filename, switch, switch, switch, out,sample))
+        elif CB == 'CB':
+            os.system('samtools view -@ %s %s | awk \'{OFS="\t"}{for(i=12;i<=NF;i++)if($i~/CB:Z:/)n=i}{for(i=12;i<=NF;i++)if($i~/UB:Z:/)m=i}{print $3,$4,$4+100,$n,$m}\' | sed %s \'s/CB:Z://g\' | sed %s \'s/UB:Z://g\' | sed %s \'s/^chr//g\' | awk \'!x[$4$5]++\' | gzip > %s_scTEtmp/o0/%s.bed.gz'%(num_threads, filename, switch, switch, switch, out,sample))
+
+def splitAllChrs(chromosome_list, filename, genenumber, countnumber, UMI=True):
+    '''
+    **Purpose**
+        Split the data into separate beds, and count up all the times each barcode appears
+
+        This variant uses more memory, but does it all at the same time and gets the filtered whitelist for free
+
+    **Arguments**
+        chromosome_list
+            List of chromosome names
+
+        filename (Required)
+            filename stub to use for tmp files
+
+        genenumber (Required)
+            Minimum number of genes expressed required for a cell to pass filtering
+
+        countnumber (Required)
+            Minimum number of counts required for a cell to pass filtering.
+
+        UMI (optional, default=True)
+            use the UMI
+
+    **Returns**
+        The barcode whitelist
+    '''
+
+    if not os.path.exists('%s_scTEtmp/o2' % filename):
+        os.system('mkdir -p %s_scTEtmp/o2'%filename)
+
+    chromosome_list = set([c.replace('chr', '') for c in chromosome_list])
+
+    file_handle_in = gzip.open('%s_scTEtmp/o1/%s.bed.gz' % (filename,filename), 'rt')
+    file_handles_out = {chr: gzip.open('%s_scTEtmp/o2/%s.chr%s.bed.gz' % (filename,filename,chr), 'wt') for chr in chromosome_list}
+
+    CRs = defaultdict(int)
+
+    if UMI:
+        uniques = {chrom: set([]) for chrom in chromosome_list}
+
+    # Make a BED for each chromosome
+    for line in file_handle_in:
+        t = line.strip().split('\t')
+        chrom = t[0].replace('chr', '') # strip chr
+
+        if chrom not in chromosome_list: # remove the unusual chromosomes
+            # Force chrMT -> chrM
+            if chrom == 'MT':
+                chrom = 'M'
+            else:
+                continue
+
+        if UMI:
+            if line in uniques[chrom]:
+                continue
+            uniques[chrom].add(line)
+            CRs[t[3]] += 1
+        else:
+            CRs[t[3]] += 1
+
+        file_handles_out[chrom].write(line)
+
+    [file_handles_out[k].close() for k in file_handles_out]
+    file_handle_in.close()
+
+    if not countnumber:
+        mincounts = 2 * genenumber
+    else:
+        mincounts = countnumber
+
+    CRs = {k: v for k, v in CRs.items() if v >= mincounts}
+
+    return list(CRs.keys())
+
+def filterCRs(filename, genenumber, countnumber):
+    CRs = defaultdict(int)
+    for f in sorted(glob.glob('%s_scTEtmp/o2/%s*.count.gz'%(filename,filename))):
+        logging.info('Reading %s '%os.path.split(f)[1])
+        o = gzip.open(f,'rt')
+        for l in o:
+            t = l.strip().split('\t')
+            CRs[t[0]] += int(t[1])
+        o.close()
+
+    if not countnumber:
+        mincounts = 2* genenumber
+    else:
+        mincounts = countnumber
+    
+    logging.info('Before filter %s'%len(CRs))
+    CRs = {k: v for k, v in CRs.items() if v >= mincounts}
+    logging.info('Before filter %s'%len(CRs))
+
+    return list(CRs.keys())
+
+def splitChr(chr, filename, CB, UMI):
+    if not os.path.exists('%s_scTEtmp/o2'%filename):
+        os.system('mkdir -p %s_scTEtmp/o2'%filename)
+
+    chr=chr.replace('chr','')
+    if CB == 'CR' or CB == 'CB': CB = True
+    else: CB = False
+    if UMI == 'UR' or UMI == 'UB': UMI = True
+    else: UMI= False
+    
+    if not CB: # C1-style data is a cell per BAM, so no barcode;
+        if not UMI:
+            if chr == '1':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^1\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '2':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^2\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '3':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^3\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            else:
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+        else:
+            if chr == '1':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^1\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '2':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^2\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '3':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^3\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            else:
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+    else:
+        if not UMI: # did not remove the potential PCR duplicates for scRNA-seq
+            if chr == '1':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^1\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '2':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^2\'[0-9]\' | grep ^%s  | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '3':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^3\'[0-9]\' | grep ^%s  | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            else:
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+        else:
+            if chr == '1':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^1\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '2':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^2\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            elif chr == '3':
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep -v ^3\'[0-9]\' | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+            else:
+                os.system('gunzip -c -f %s_scTEtmp/o1/%s.bed.gz | grep ^%s | gzip -c > %s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr,filename,filename,chr))
+
+    CRs = defaultdict(int)
+    o = gzip.open('%s_scTEtmp/o2/%s.chr%s.bed.gz'%(filename,filename,chr),'rt')
+    for l in o:
+        t = l.strip().split('\t')
+        CRs[t[3]] += 1
+    o.close()
+
+    o = gzip.open('%s_scTEtmp/o2/%s.chr%s.count.gz'%(filename,filename,chr),'wt')
+    for k in CRs:
+        o.write('%s\t%s\n'%(k,CRs[k]))
+    o.close()
+
+def align(chr, filename, all_annot, glannot, whitelist): #CB
+    '''
+    **Purpose**
+        For each read, align it to the index and assign a TE, gene.
+
+    This is the speed critical part.
+
+    '''
+    s1 = time.time()
+    chr = 'chr' + chr
+
+    if not os.path.exists('%s_scTEtmp/o3'%filename):
+        os.system('mkdir -p %s_scTEtmp/o3'%filename)
+
+    if not glannot: # Load separately for the multicore pipeline, share the index for the single core pipeline
+        glannot = glload(all_annot)
+
+    # Only keep the glbase parts we need.
+    buckets = glannot.buckets[chr.replace('chr', '')]
+    all_annot = glannot.linearData
+
+    oh = gzip.open('%s_scTEtmp/o2/%s.%s.bed.gz' % (filename, filename, chr), 'rt')
+    res = {}
+    for line in oh:
+        t = line.strip().split('\t')
+        barcode = t[3]
+        if barcode not in whitelist:
+            continue
+        if barcode not in res:
+            res[barcode] = defaultdict(int)
+
+        #chrom = t[0].replace('chr', '') # Don't need as each align is already split for each chrom;
+        left = int(t[1])
+        rite = int(t[2])
+
+        #loc = location(chr=chrom, left=left, right=rite)
+        left_buck = ((left-1)//10000) * 10000
+        right_buck = ((rite)//10000) * 10000
+        buckets_reqd = range(left_buck, right_buck+10000, 10000)
+
+        if buckets_reqd:
+            loc_ids = set()
+            loc_ids_update = loc_ids.update
+
+            # get the ids reqd.
+            [loc_ids_update(buckets[buck]) for buck in buckets_reqd if buck in buckets]
+
+            result = [all_annot[index]['annot'] for index in loc_ids if (rite >= all_annot[index]['loc'].loc['left'] and left <= all_annot[index]['loc'].loc["right"])]
+
+            if result:
+                for gene in result:
+                    res[barcode][gene] += 1
+
+    oh.close()
+
+    oh = gzip.open('%s_scTEtmp/o3/%s.%s.bed.gz' % (filename,filename,chr), 'wt')
+    for bc in sorted(res):
+        for gene in sorted(res[bc]):
+            oh.write('%s\t%s\t%s\n' % (bc, gene, res[bc][gene]))
+    oh.close()
+
+def Countexpression(filename, allelement, genenumber, cellnumber, hdf5):
+    gene_seen = allelement
+
+    whitelist={}
+    o = gzip.open('%s_scTEtmp/o4/%s.bed.gz'%(filename, filename), 'rt')
+    for n,l in enumerate(o):
+        t = l.strip().split('\t')
+        if t[0] not in whitelist:
+            whitelist[t[0]] = 0
+        whitelist[t[0]] += 1
+    o.close()
+
+    CRlist = []
+    sortcb = sorted(whitelist.items(), key=lambda item:item[1], reverse=True)
+    for n,k in enumerate(sortcb):
+        if k[1] < genenumber:
+            break
+        if n >= cellnumber:
+            break
+        CRlist.append(k[0])
+    CRlist = set(CRlist)
+
+    res = {}
+    genes_oh = gzip.open('%s_scTEtmp/o4/%s.bed.gz' % (filename,filename), 'rt')
+    for n, l in enumerate(genes_oh):
+        t = l.strip().split('\t')
+        if t[0] not in CRlist:
+            continue
+        if t[0] not in res:
+            res[t[0]] = {}
+        if t[1] not in res[t[0]]:
+            res[t[0]][t[1]] = 0
+        res[t[0]][t[1]] += int(t[2])
+
+    genes_oh.close()
+
+    s=time.time()
+
+    # Save out the final file
+
+    gene_seen = list(gene_seen) # Do the sort once;
+    gene_seen.sort()
+
+    #==== save results =====
+    if not hdf5: # save as csv
+        res_oh = open('%s.csv'%filename, 'w')
+        res_oh.write('barcodes,')
+        res_oh.write('%s\n' % (','.join([str(i) for i in gene_seen])))
+
+        for k in sorted(res):
+            l = ["0"] * len(gene_seen) # Avoid all the appends
+            for idx, gene in enumerate(gene_seen):
+                if gene in res[k]:
+                    l[idx] = str(res[k][gene])
+            res_oh.write('%s,%s\n' % (k, ','.join(l)))
+        res_oh.close()
+    
+    else: # save as hdf5
+        data = []
+        CBs = []
+        for k in sorted(res):
+            l = ["0"] * len(gene_seen) # Avoid all the appends
+            for idx, gene in enumerate(gene_seen):
+                if gene in res[k]:
+                    l[idx] = str(res[k][gene])
+            data.append(l)
+            CBs.append(k)
+
+        obs = pd.DataFrame(index = CBs)
+        var = pd.DataFrame(index = gene_seen)
+        adata = ad.AnnData(np.asarray(data).astype(int),var = var,obs = obs)
+        adata.X = scipy.sparse.csr_matrix(adata.X)
+        adata.write('%s.h5ad'%filename)
+    
+    #========================
+
+
+    return len(res), genenumber, filename
+
+def timediff(timestart, timestop):
+    t  = (timestop-timestart)
+    time_day = t.days
+    s_time = t.seconds
+    ms_time = t.microseconds / 1000000
+    usedtime = int(s_time + ms_time)
+    time_hour = int(usedtime / 60 / 60 )
+    time_minute = int((usedtime - time_hour * 3600 ) / 60 )
+    time_second =  int(usedtime - time_hour * 3600 - time_minute * 60 )
+    retstr = "%dd %dh %dm %ds"  %(time_day, time_hour, time_minute, time_second,)
+    return retstr