celescope 2.2.2__tar.gz → 2.3.0.dev1__tar.gz

{celescope-2.2.2/celescope.egg-info → celescope-2.3.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: celescope
-Version: 2.2.2
+Version: 2.3.0.dev1
 Summary: Single Cell Analysis Pipelines
 Home-page: https://github.com/singleron-RD/CeleScope
 Author: zhouyiqi

{celescope-2.2.2 → celescope-2.3.0.dev1}/celescope/__init__.py

@@ -1,6 +1,6 @@
 import os
 
-__VERSION__ = "2.2.2"
+__VERSION__ = "2.3.0dev1"
 __version__ = __VERSION__
 
 ASSAY_LIST = [
@@ -58,7 +58,7 @@ HELP_DICT = {
     "genomeSAindexNbases": """For small genomes, the parameter --genomeSAindexNbases must to be scaled down, with a typical 
 value of min(14, log2(GenomeLength)/2 - 1). For example, for 1 megaBase genome, this is equal 
 to 9, for 100 kiloBase genome, this is equal to 7.""",
-    "chemistry": "`--chemistry auto` can auto-detect scopeV2 mRNA, scopeV3 mRNA, full length VDJ mRNA(flv_rna) and full length VDJ(flv). You need to explicitly use `--chemistry scopeV1` for legacy chemistry scopeV1. `--chemistry customized` is used for user defined combinations that you need to provide `--pattern`, `--whitelist` and `--linker` at the same time.",
+    "chemistry": "`--chemistry auto` can auto-detect most chemistry. You need to explicitly use `--chemistry GEXSCOPE-MicroBead` for legacy chemistry GEXSCOPE-MicroBead as it has no barcode whitelists. `--chemistry customized` is used for user defined combinations that you need to provide `--pattern` and optional `--whitelist` at the same time.",
 }
 
 # report metrics help

celescope-2.3.0.dev1/celescope/bulk_rna/__init__.py

@@ -0,0 +1,5 @@
+STEPS = ["sample", "starsolo", "cells", "split_fastq"]
+__ASSAY__ = "bulk_rna"
+REMOVE_FROM_MULTI = {
+    "cells",
+}

celescope-2.3.0.dev1/celescope/bulk_rna/multi_bulk_rna.py

@@ -0,0 +1,33 @@
+from celescope.bulk_rna.__init__ import __ASSAY__
+from celescope.tools.multi import Multi
+from celescope.tools.__init__ import (
+    OUTS_DIR,
+)
+
+
+class Multi_bulk_rna(Multi):
+    def starsolo(self, sample):
+        step = "starsolo"
+        arr = self.fq_dict[sample]
+        cmd_line = self.get_cmd_line(step, sample)
+        cmd = f'{cmd_line} ' f'--fq1 {arr["fq1_str"]} --fq2 {arr["fq2_str"]} '
+        self.process_cmd(cmd, step, sample, m=self.args.starMem, x=self.args.thread)
+
+    def split_fastq(self, sample):
+        step = "split_fastq"
+        arr = self.fq_dict[sample]
+        cmd_line = self.get_cmd_line(step, sample)
+        cmd = (
+            f'{cmd_line} '
+            f'--fq1 {arr["fq1_str"]} --fq2 {arr["fq2_str"]} --barcodes {self.outdir_dic[sample][OUTS_DIR]}/filtered/barcodes.tsv.gz'
+        )
+        self.process_cmd(cmd, step, sample, m=5, x=1)
+
+
+def main():
+    multi = Multi_bulk_rna(__ASSAY__)
+    multi.run()
+
+
+if __name__ == "__main__":
+    main()

celescope-2.3.0.dev1/celescope/bulk_rna/starsolo.py

@@ -0,0 +1,247 @@
+import subprocess
+import sys
+
+import numpy as np
+import pandas as pd
+
+from celescope.__init__ import HELP_DICT
+from celescope.tools.__init__ import COUNTS_FILE_NAME
+from celescope.chemistry_dict import chemistry_dict
+from celescope.tools.emptydrop_cr import get_plot_elements
+from celescope.tools.starsolo import (
+    Demultiplexing,
+    Mapping,
+    create_solo_args,
+)
+from celescope.tools.starsolo import (
+    Starsolo as tools_Starsolo,
+)
+from celescope.tools.matrix import CountMatrix
+from celescope.tools import utils
+from celescope.tools.step import Step, s_common
+
+SAM_attributes = "NH HI nM AS CR UR CB UB GX GN "
+
+
+class Starsolo(tools_Starsolo):
+    def __init__(self, args, display_title=None):
+        super().__init__(args, display_title=display_title)
+        self.tsv_matrix_file = f"{self.out_prefix}_matrix.tsv.gz"
+        self.outs.append(self.tsv_matrix_file)
+
+    def run_starsolo(self):
+        cmd = create_solo_args(
+            pattern_args=self.pattern_args,
+            whitelist_args=self.whitelist_args,
+            outFileNamePrefix=self.out_prefix + "_",
+            fq1=self.args.fq1,
+            fq2=self.args.fq2,
+            genomeDir=self.args.genomeDir,
+            soloCellFilter=self.args.soloCellFilter,
+            runThreadN=self.args.thread,
+            clip3pAdapterSeq=self.args.adapter_3p,
+            outFilterMatchNmin=self.args.outFilterMatchNmin,
+            soloFeatures=self.args.soloFeatures,
+            outSAMattributes=self.outSAMattributes,
+            soloCBmatchWLtype=self.args.soloCBmatchWLtype,
+            extra_starsolo_args=self.extra_starsolo_args,
+        )
+        if self.chemistry == "bulk_rna-bulk_vdj_match":
+            cmd += "--soloStrand Reverse \\\n"
+        sys.stderr.write(cmd)
+        subprocess.check_call(cmd, shell=True)
+        cmd = f"chmod -R 755 {self.solo_out_dir}"
+        sys.stderr.write(cmd)
+        subprocess.check_call(cmd, shell=True)
+
+    @utils.add_log
+    def keep_barcodes(self):
+        """take in raw matrix, only keep barcodes in the input file, and convert barcodes to sample names"""
+        matrix = CountMatrix.from_matrix_dir(self.raw_matrix)
+        barcode_sample = utils.two_col_to_dict(self.args.barcode_sample)
+        for barcode in barcode_sample:
+            if barcode not in matrix.get_barcodes():
+                barcode_sample.pop(barcode)
+                sys.stderr.write(
+                    f"WARNING: {barcode} {barcode_sample[barcode]} not found in raw matrix!\n"
+                )
+        filtered = matrix.slice_matrix_bc(barcode_sample.keys())
+        filtered.to_matrix_dir(self.filtered_matrix)
+        samples = [barcode_sample[bc] for bc in filtered.get_barcodes()]
+        converted = CountMatrix(filtered.get_features(), samples, filtered.get_matrix())
+        df = converted.to_df()
+        df.to_csv(self.tsv_matrix_file, sep="\t")
+        return filtered
+
+    def run(self):
+        self.run_starsolo()
+        filtered = self.keep_barcodes()
+        self.gzip_matrix()
+        q30_cb, q30_umi = self.get_Q30_cb_UMI()
+        return q30_cb, q30_umi, filtered
+
+
+class Cells(Step):
+    def __init__(self, args, display_title=None):
+        super().__init__(args, display_title=display_title)
+        solo_dir = f"{self.outdir}/{self.sample}_Solo.out/GeneFull_Ex50pAS"
+        self.summary_file = f"{solo_dir}/Summary.csv"
+        self.counts_file = f"{self.outs_dir}/{COUNTS_FILE_NAME}"
+
+    @utils.add_log
+    def parse_summary(self):
+        df = pd.read_csv(self.summary_file, index_col=0, header=None)
+        s = df.iloc[:, 0]
+        saturation = float(s["Sequencing Saturation"])
+        n_reads = int(s["Number of Reads"])
+        q30_RNA = float(s["Q30 Bases in RNA read"])
+
+        return n_reads, q30_RNA, saturation
+
+    def run(self, filtered: CountMatrix):
+        df_counts = pd.read_csv(self.counts_file, index_col=0, header=0, sep="\t")
+        reads_total = df_counts["countedU"].sum()
+        bcs = filtered.get_barcodes()
+        n_cells = len(bcs)
+        reads_cell = df_counts.loc[bcs, "countedU"].sum()
+        fraction_reads_in_cells = float(reads_cell / reads_total)
+        mean_used_reads_per_cell = int(reads_cell // len(bcs))
+        median_umi_per_cell = int(df_counts.loc[bcs, "UMI"].median())
+
+        bc_geneNum, total_genes = filtered.get_bc_geneNum()
+        median_genes_per_cell = int(np.median(list(bc_geneNum.values())))
+
+        df_counts.loc[:, "mark"] = "UB"
+        df_counts.loc[bcs, "mark"] = "CB"
+        df_counts.to_csv(self.counts_file, sep="\t", index=True)
+
+        self.add_metric(
+            "Number of Wells",
+            n_cells,
+            help_info="number of barcodes with at least one UMI",
+        )
+        self.add_metric(
+            "Fraction of Reads in Wells",
+            fraction_reads_in_cells,
+            value_type="fraction",
+            help_info="Fraction of reads which were mapped to a barcode",
+        )
+        self.add_metric(
+            "Mean Reads per Well",
+            mean_used_reads_per_cell,
+            help_info="Mean number of reads per barcode",
+        )
+        self.add_metric(
+            "Median UMI per Well",
+            median_umi_per_cell,
+            help_info="Median UMI count per barcode",
+        )
+        self.add_metric(
+            "Median Genes per Well",
+            median_genes_per_cell,
+            help_info="Median number of genes per barcode",
+        )
+        self.add_data(chart=get_plot_elements.plot_barcode_rank(self.counts_file))
+
+        n_reads, q30_RNA, saturation = self.parse_summary()
+        self.add_metric(
+            "Saturation",
+            saturation,
+            value_type="fraction",
+            help_info="the fraction of read originating from an already-observed UMI.",
+        )
+        return n_reads, q30_RNA
+
+
+def starsolo(args):
+    with Starsolo(args) as runner:
+        q30_cb, q30_umi, filtered = runner.run()
+
+    with Mapping(args) as runner:
+        valid_reads, corrected = runner.run()
+
+    with Cells(args) as runner:
+        n_reads, q30_RNA = runner.run(filtered)
+
+    with Demultiplexing(args) as runner:
+        runner.run(valid_reads, n_reads, corrected, q30_cb, q30_umi, q30_RNA)
+
+
+def get_opts_starsolo(parser, sub_program=True):
+    parser.add_argument(
+        "--chemistry",
+        help=HELP_DICT["chemistry"],
+        choices=list(chemistry_dict.keys()),
+        default="auto",
+    )
+    parser.add_argument(
+        "--pattern",
+        help="""The pattern of R1 reads, e.g. `C8L16C8L16C8L1U12T18`. The number after the letter represents the number 
+        of bases.  
+        - `C`: cell barcode  
+        - `L`: linker(common sequences)  
+        - `U`: UMI    
+        - `T`: poly T""",
+    )
+    parser.add_argument(
+        "--whitelist",
+        help="whitelist file path.",
+    )
+    parser.add_argument(
+        "--adapter_3p",
+        help="Adapter sequence to clip from 3 prime. Multiple sequences are seperated by space",
+        default="AAAAAAAAAAAA",
+    )
+    parser.add_argument(
+        "--genomeDir",
+        help=HELP_DICT["genomeDir"],
+    )
+    parser.add_argument(
+        "--outFilterMatchNmin",
+        help="""Alignment will be output only if the number of matched bases 
+is higher than or equal to this value.""",
+        default=50,
+    )
+    parser.add_argument(
+        "--soloCellFilter",
+        help="The same as the argument in STARsolo",
+        default="None",
+    )
+    parser.add_argument(
+        "--starMem", help="Maximum memory that STAR can use.", default=32
+    )
+    parser.add_argument("--STAR_param", help=HELP_DICT["additional_param"], default="")
+    parser.add_argument(
+        "--SAM_attributes",
+        help=f"Additional attributes(other than {SAM_attributes}) to be added to SAM file",
+        default="",
+    )
+    parser.add_argument(
+        "--soloFeatures",
+        help="The same as the argument in STARsolo",
+        default="GeneFull_Ex50pAS Gene",
+    )
+    parser.add_argument(
+        "--soloCBmatchWLtype",
+        help="The same as the argument in STARsolo. Please note `EditDist 2` only works with `--soloType CB UMI Complex`. ",
+        default="1MM",
+    )
+    parser.add_argument(
+        "--barcode_sample",
+        help="tsv file of well barcodes and sample names. The first column is well barcodes and the second column is sample names.",
+        required=True,
+    )
+    if sub_program:
+        parser.add_argument(
+            "--fq1",
+            help="R1 fastq file. Multiple files are separated by comma.",
+            required=True,
+        )
+        parser.add_argument(
+            "--fq2",
+            help="R2 fastq file. Multiple files are separated by comma.",
+            required=True,
+        )
+        parser = s_common(parser)
+
+    return parser

celescope-2.3.0.dev1/celescope/chemistry_dict.py

@@ -0,0 +1,84 @@
+from importlib import resources
+
+chemistry_dict = {
+    "auto": {
+        "pattern": ""  # auto detect
+    },
+    "customized": {
+        "pattern": ""  # user defined
+    },
+    "GEXSCOPE-MicroBead": {
+        "pattern": "C12U8",
+        "bc": [],
+    },
+    "GEXSCOPE-V1": {
+        "pattern": "C8L16C8L16C8L1U12",
+        "bc": ["bc.txt", "bc.txt", "bc.txt"],
+    },
+    "GEXSCOPE-V2": {
+        "pattern": "C9L16C9L16C9L1U12",
+        "bc": ["bc1.txt", "bc2.txt", "bc3.txt"],
+    },
+    "GEXSCOPE-V3": {
+        # 0-3 bases offset at the begining
+        "pattern": "C9L6C9L6C9L1U12",
+        "bc": ["bc1.txt", "bc2.txt", "bc3.txt"],
+        "linker": ["linker1.txt", "linker2.txt"],
+    },
+    "flv_rna": {
+        "pattern": "C8L16C8L16C8U9L6",
+        "bc": ["bc.txt", "bc.txt", "bc.txt"],
+    },
+    "flv": {
+        "pattern": "U9C8L16C8L16C8",
+        "bc": ["bc.txt", "bc.txt", "bc.txt"],
+    },
+    "rna_5p-1": {
+        "pattern": "U8C9L4C9L4C9",
+    },
+    "rna_3p-1": {
+        "pattern": "C9L4C9L4C9U8",
+    },
+    "5p3p-1": {
+        "pattern": "C9C9C9U8",  # converted
+        "bc": ["bc1.txt", "bc2.txt", "bc3.txt"],
+    },
+    "rna_5p-2": {
+        "pattern": "C8L4U9C8L4C8",
+    },
+    "rna_3p-2": {
+        "pattern": "C8L4C8U9L4C8",
+    },
+    "5p3p-2": {
+        "pattern": "C8C8C8U9",  # converted
+        "bc": ["bc1.txt", "bc2.txt", "bc3.txt"],
+    },
+    "rna_5p-3": {
+        "pattern": "C9L6U9C9L4C9",
+    },
+    "rna_3p-3": {
+        "pattern": "C9L4C9U9L6C9",
+    },
+    "5p3p-3": {
+        "pattern": "C9C9C9U9",  # converted
+        "bc": ["bc1.txt", "bc2.txt", "bc3.txt"],
+    },
+    "bulk_vdj": {
+        "pattern": "L18C6U16",
+        "bc": ["bc.txt"],
+    },
+    "bulk_rna-V1": {
+        "pattern": "C9U12",
+        "bc": ["bc.txt"],
+    },
+    "bulk_rna-V2": {
+        "pattern": "L9C9U12",
+        "bc": ["bc.txt"],
+    },
+    "bulk_rna-bulk_vdj_match": {
+        "pattern": "L18C6U16",
+        "bc": ["bc.txt"],
+    },
+}
+
+chemistry_dir = str(resources.files("celescope.data.chemistry"))

celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V1/bc.txt

@@ -0,0 +1,96 @@
+AACGTGAT
+AAACATCG
+ATGCCTAA
+AGTGGTCA
+ACCACTGT
+ACATTGGC
+CAGATCTG
+CATCAAGT
+CGCTGATC
+ACAAGCTA
+CTGTAGCC
+AGTACAAG
+AACAACCA
+AACCGAGA
+AACGCTTA
+AAGACGGA
+AAGGTACA
+ACACAGAA
+ACAGCAGA
+ACCTCCAA
+ACGCTCGA
+ACGTATCA
+ACTATGCA
+AGAGTCAA
+AGATCGCA
+AGCAGGAA
+AGTCACTA
+ATCCTGTA
+ATTGAGGA
+CAACCACA
+GACTAGTA
+CAATGGAA
+CACTTCGA
+CAGCGTTA
+CATACCAA
+CCAGTTCA
+CCGAAGTA
+CCGTGAGA
+CCTCCTGA
+CGAACTTA
+CGACTGGA
+CGCATACA
+CTCAATGA
+CTGAGCCA
+CTGGCATA
+GAATCTGA
+CAAGACTA
+GAGCTGAA
+GATAGACA
+GCCACATA
+GCGAGTAA
+GCTAACGA
+GCTCGGTA
+GGAGAACA
+GGTGCGAA
+GTACGCAA
+GTCGTAGA
+GTCTGTCA
+GTGTTCTA
+TAGGATGA
+TATCAGCA
+TCCGTCTA
+TCTTCACA
+TGAAGAGA
+TGGAACAA
+TGGCTTCA
+TGGTGGTA
+TTCACGCA
+AACTCACC
+AAGAGATC
+AAGGACAC
+AATCCGTC
+AATGTTGC
+ACACGACC
+ACAGATTC
+AGATGTAC
+AGCACCTC
+AGCCATGC
+AGGCTAAC
+ATAGCGAC
+ATCATTCC
+ATTGGCTC
+CAAGGAGC
+CACCTTAC
+CCATCCTC
+CCGACAAC
+CCTAATCC
+CCTCTATC
+CGACACAC
+CGGATTGC
+CTAAGGTC
+GAACAGGC
+GACAGTGC
+GAGTTAGC
+GATGAATC
+GCCAAGAC

celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V1/linker1.txt

@@ -0,0 +1,4 @@
+ATCCACGTGCTTGAGA
+TCGGTGACAGCCATAT
+CGAACATGTAGGTCTC
+GATTGTCACTAACGCG

celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V1/linker2.txt

@@ -0,0 +1,4 @@
+TCAGCATGCGGCTACG
+CGTAGTCAGAAGCTGA
+GACTACGTATTAGCAT
+ATGCTGACTCCTAGTC

celescope-2.2.2/celescope/data/chemistry/scopeV3.0.1/bclist2 → celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V2/bc2.txt

@@ -93,4 +93,4 @@ GATAACCGC
 GCTGCGATA
 TACTCACCA
 ACCAGGTCA
-GCAACTTCA
+GCAACTTCA

celescope-2.2.2/celescope/data/chemistry/scopeV3.0.1/bclist3 → celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V2/bc3.txt

@@ -189,4 +189,4 @@ CTAGGTTGC
 AGGCAGAAC
 CTACTATGG
 CTCAATCGT
-GTAACGCTT
+GTAACGCTT

celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V2/linker1.txt

@@ -0,0 +1,4 @@
+ATCCACGTGCTTGAGA
+TCGGTGACAGCCATAT
+CGAACATGTAGGTCTC
+GATTGTCACTAACGCG

celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V2/linker2.txt

@@ -0,0 +1,4 @@
+TCAGCATGCGGCTACG
+CGTAGTCAGAAGCTGA
+GACTACGTATTAGCAT
+ATGCTGACTCCTAGTC

celescope-2.2.2/celescope/data/chemistry/scopeV3.0.1/bclist1 → celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V3/bc1.txt

@@ -93,4 +93,4 @@ TACCGTCTG
 TCTTGCTTG
 CCAGCTAAC
 CTTACGCAG
-AGTAGGAGG
+AGTAGGAGG

celescope-2.3.0.dev1/celescope/data/chemistry/GEXSCOPE-V3/bc2.txt

@@ -0,0 +1,96 @@
+TTGGTGACC
+CATCGGTTC
+CAATGCAAC
+AAGGTGGTA
+TCAGGTAGA
+CACTAGGCA
+GGCATGCAA
+ATGCGATAC
+AGACGAAGT
+CGATAAGGC
+CGAGTTGCA
+GGTGATCAG
+AGTGGTGGT
+TAGCGATGA
+ATGCCTAAG
+CCATAATCG
+CTAACCAGA
+TGATGTGCC
+TAGCTACAC
+TCCTGGCTT
+GTAGTTCCT
+GTATCCTTC
+GCCATAACC
+GCATGTTGG
+AATCCGGTG
+ACGTTACGA
+AGCATAGCG
+GGTCCGTAA
+GTTCTACCG
+CGCTGTAAG
+CGTCATACC
+TGGTAACCG
+ACAACAGGT
+TGACTTCCG
+CACGCAATA
+ACCGTACTC
+GATGTGTGT
+ACACCAACG
+TTGAGACAG
+CGGATCATC
+TAATGGCCG
+ACCTCGACT
+CGGCTAGAT
+TGGACTTGT
+TGATCCTCT
+ATAGCGTGT
+TGTCGGTGT
+AGCCACATA
+GAAGAAGCC
+CTGTGGTAT
+CAGCCGAAT
+ACCTGCTAC
+TCTTCTCAG
+GATCAGGAC
+TAGACCACT
+TGAGTAGTC
+TTCGAGGAT
+ATGTATCGG
+TCTGTCTGC
+GCTTACAGG
+ATACCAGTC
+AACGTCCAA
+CTCCTCAAT
+TTCCAATCG
+AACGCTAGT
+TTACACGAC
+CATACGACC
+CTAATCGCG
+CGTAATTGG
+GTAGTGTTG
+TATAGCGGT
+CGTACTGAA
+TCGATGTGG
+CCAGAAGAT
+AGGCTGTTG
+TGAGGCCTT
+CCAGTCCTA
+CAATTGCGC
+TTGCCGTCA
+GAGTTGACA
+TGGTCAGTT
+GGTAGTCCA
+ATCCTTCCA
+ATGTGCAGC
+GGAAGACTC
+GTAATGGAC
+TATCGTGCA
+CCTACAAGG
+TAGTCCGGA
+AGATACGCA
+ACTCATCGT
+GATAACCGC
+GCTGCGATA
+TACTCACCA
+ACCAGGTCA
+GCAACTTCA