offtracker 2.7.10__zip → 2.10.0__zip

offtracker-2.10.0/offtracker/snakefile/Snakefile_offtracker.smk

@@ -0,0 +1,249 @@
+# 2023.08.11. adding a option for not normalizing the bw file
+# 2024.01.23. add --fixedStep to bigwigCompare for not merging neighbouring bins with equal values.
+# 2025.05.22. refine the structure
+
+configfile: "config.yaml"
+
+# # fastq 信息
+_files_R1 = config['files_R1'] # dict型, key 为 sample
+_files_R2 = config['files_R2'] # dict型, key 为 sample
+# # 运行参数
+_output_dir = config["output_dir"]
+_thread = config['thread']
+_BinSize = str(config["binsize"])
+_normalize = config["normalize"]
+
+
+import os
+
+if _normalize == "True":
+    rule all:
+        input:
+            expand( os.path.join(_output_dir,"{sample}.fw.bed"), sample=_files_R1 ),
+            expand( os.path.join(_output_dir,"{sample}.rv.bed"), sample=_files_R1 ),
+            expand( os.path.join(_output_dir,"{sample}.fw.scaled.bw"), sample=_files_R1 ),
+            expand( os.path.join(_output_dir,"{sample}.rv.scaled.bw"), sample=_files_R1 ),
+            expand( os.path.join(_output_dir,"{sample}." + _BinSize + ".add.bdg"),sample=_files_R1 ),
+elif _normalize == "False":
+    rule all:
+        input:
+            expand( os.path.join(_output_dir,"{sample}.fw.bed"), sample=_files_R1 ),
+            expand( os.path.join(_output_dir,"{sample}.rv.bed"), sample=_files_R1 ),
+            expand( os.path.join(_output_dir,"{sample}.fw.raw.bw"), sample=_files_R1 ),
+            expand( os.path.join(_output_dir,"{sample}.rv.raw.bw"), sample=_files_R1 ),
+else:
+    raise ValueError('Please provide "True" or "False" for "--normalize" when running offtracker_config.py')
+
+
+rule chromap:
+    input:
+        R1=lambda w: _files_R1[w.sample],
+        R2=lambda w: _files_R2[w.sample]
+    threads:
+        _threads
+    params:
+        index=config["index"],
+        fasta=config["fasta"]
+    output:
+        temp(os.path.join(_output_dir,"{sample}.chromapx.bed"))
+    shell:
+        """
+        chromap -l 3000 --low-mem --BED --remove-pcr-duplicates \
+        --min-read-length 10 --allocate-multi-mappings \
+        -x {params.index} -r {params.fasta} -t {threads} -1 {input.R1} -2 {input.R2} -o {output}
+        """
+
+if config["blacklist"] != 'none':
+    rule remove_blacklist:
+        input:
+            os.path.join(_output_dir,"{sample}.chromapx.bed")
+        threads:
+            _threads
+        params:
+            blacklist=config["blacklist"]
+        output:
+            temp(os.path.join(_output_dir,"{sample}.filtered.bed"))
+        shell:
+            "bedtools intersect -a {input} -b {params.blacklist} -v > {output}"
+
+    rule bed2fr:
+        input:
+            os.path.join(_output_dir,"{sample}.filtered.bed")
+        threads:
+            _threads
+        params:
+            dir_script=config["utility_dir"],
+            ignore_chr=config["ignore_chr"],
+        output:
+            fw=os.path.join(_output_dir,"{sample}.fw.bed"),
+            rv=os.path.join(_output_dir,"{sample}.rv.bed")
+        shell:
+            "python {params.dir_script}/1.1_bed2fr.py -b {input} {params.ignore_chr}"
+else:
+    rule bed2fr:
+        input:
+            os.path.join(_output_dir,"{sample}.chromapx.bed")
+        threads:
+            _threads
+        params:
+            dir_script=config["utility_dir"],
+            ignore_chr=config["ignore_chr"],
+        output:
+            fw=os.path.join(_output_dir,"{sample}.fw.bed"),
+            rv=os.path.join(_output_dir,"{sample}.rv.bed")
+        shell:
+            "python {params.dir_script}/1.1_bed2fr.py -b {input} {params.ignore_chr}"    
+
+rule bed2bdg_fw:
+    input:
+        os.path.join(_output_dir,"{sample}.fw.bed")
+    threads:
+        _threads
+    params:
+        gl=config["genomelen"]
+    output:
+        temp(os.path.join(_output_dir,"{sample}.fw.bdg"))
+    shell:
+        "bedtools genomecov -bg -i {input} -g {params.gl} > {output}"
+
+rule bed2bdg_rv:
+    input:
+        os.path.join(_output_dir,"{sample}.rv.bed")
+    threads:
+        _threads
+    params:
+        gl=config["genomelen"]
+    output:
+        temp(os.path.join(_output_dir,"{sample}.rv.bdg"))
+    shell:
+        "bedtools genomecov -bg -i {input} -g {params.gl} > {output}"
+
+rule bdg_sort_fw:
+    input:
+        fw=os.path.join(_output_dir,"{sample}.fw.bdg")
+    threads:
+        _threads
+    output:
+        temp(os.path.join(_output_dir,"{sample}.fw.sorted.bdg"))
+    shell:
+        "bedtools sort -i {input.fw} > {output}"
+
+rule bdg_sort_rv:
+    input:
+        rv=os.path.join(_output_dir,"{sample}.rv.bdg")
+    threads:
+        _threads
+    output:
+        temp(os.path.join(_output_dir,"{sample}.rv.sorted.bdg"))
+    shell:
+        "bedtools sort -i {input.rv} > {output}"
+
+if _normalize == "True":
+    rule bdg_normalize_fw:
+        input:
+            bdg=os.path.join(_output_dir,"{sample}.fw.sorted.bdg"),
+            bed=os.path.join(_output_dir,"{sample}.fw.bed")
+        threads:
+            _threads
+        params:
+            dir_script=config["utility_dir"]
+        output:
+            temp(os.path.join(_output_dir,"{sample}.fw.scaled.bdg"))
+        shell:
+            "python {params.dir_script}/1.3_bdg_normalize_v4.0.py --bdg {input.bdg} --bed {input.bed}"
+    
+    rule bdg_normalize_rv:
+        input:
+            bdg=os.path.join(_output_dir,"{sample}.rv.sorted.bdg"),
+            bed=os.path.join(_output_dir,"{sample}.rv.bed")
+        threads:
+            _threads
+        params:
+            dir_script=config["utility_dir"]
+        output:
+            temp(os.path.join(_output_dir,"{sample}.rv.scaled.bdg"))
+        shell:
+            "python {params.dir_script}/1.3_bdg_normalize_v4.0.py --bdg {input.bdg} --bed {input.bed}"
+    
+    rule bdg2bw_fw:
+        input:
+            os.path.join(_output_dir,"{sample}.fw.scaled.bdg")
+        threads:
+            _threads
+        params:
+            gl=config["genomelen"],
+            dir_script=config["utility_dir"]
+        output:
+            os.path.join(_output_dir,"{sample}.fw.scaled.bw")
+        shell:
+            "{params.dir_script}/bedGraphToBigWig {input} {params.gl} {output}"
+    
+    rule bdg2bw_rv:
+        input:
+            os.path.join(_output_dir,"{sample}.rv.scaled.bdg")
+        threads:
+            _threads
+        params:
+            gl=config["genomelen"],
+            dir_script=config["utility_dir"]
+        output:
+            os.path.join(_output_dir,"{sample}.rv.scaled.bw")
+        shell:
+            "{params.dir_script}/bedGraphToBigWig {input} {params.gl} {output}"
+    
+    rule bwAdd:
+        input:
+            fw=os.path.join(_output_dir,"{sample}.fw.scaled.bw"),
+            rv=os.path.join(_output_dir,"{sample}.rv.scaled.bw")
+        threads:
+            _threads
+        output:
+            os.path.join(_output_dir,"{sample}." + _BinSize + ".add.bdg")
+        shell:
+            """
+            bigwigCompare --binSize {_BinSize} -p {threads} --verbose -o {output} \
+            --outFileFormat bedgraph --fixedStep \
+            --bigwig1 {input.fw} \
+            --bigwig2 {input.rv} \
+            --operation add 
+            """
+else:
+    rule bdg_reverse_rv:
+        input:
+            os.path.join(_output_dir,"{sample}.rv.sorted.bdg")
+        threads:
+            _threads
+        output:
+            temp(os.path.join(_output_dir,"{sample}.rv.sorted_r.bdg"))
+        shell:
+            "awk -F '\t' -v OFS='\t' '{{$4=-$4; print}}' {input} > {output}"
+    
+    rule bdg2bw_fw:
+        input:
+            os.path.join(_output_dir,"{sample}.fw.sorted.bdg")
+        threads:
+            _threads
+        params:
+            gl=config["genomelen"],
+            dir_script=config["utility_dir"]
+        output:
+            os.path.join(_output_dir,"{sample}.fw.raw.bw")
+        shell:
+            "{params.dir_script}/bedGraphToBigWig {input} {params.gl} {output}"
+    
+    rule bdg2bw_rv:
+        input:
+            os.path.join(_output_dir,"{sample}.rv.sorted_r.bdg")
+        threads:
+            _threads
+        params:
+            gl=config["genomelen"],
+            dir_script=config["utility_dir"]
+        output:
+            os.path.join(_output_dir,"{sample}.rv.raw.bw")
+        shell:
+            "{params.dir_script}/bedGraphToBigWig {input} {params.gl} {output}"
+
+
+
+

offtracker-2.7.10/offtracker/mapping/1.1_bed2fr_v4.5.py → offtracker-2.10.0/offtracker/utility/1.1_bed2fr.py

@@ -8,19 +8,21 @@ parser.description='这算一个小彩蛋'
 # 2022.10.21. v3.0: 文件名长度 chromap -> filtered
 # 2022.10.26. v4.0: f,r 改成 fw,rv
 # 2022.01.11. v4.5: 只取 common chromosomes (chr1-chr22, chrX, chrY, chrM)
+# 2025.06.05. v5.0: 增加 ignore_chr 选项，默认只取 common chromosomes
 
 # 单文件处理脚本，配合snakemake使用
 
 parser.add_argument("-b", "--bed", type=str, metavar="dir_bed" , required=True, help="dir of bed file")
+parser.add_argument('--ignore_chr', action='store_true', help='If not set, only chr1-chr22, chrX, chrY, chrM will be analyzed.')
 
 args = parser.parse_args()
 
 bed_file = pd.read_csv( args.bed, sep='\t', header=None)
 
-common_chr = pd.Series(['chr']*22).str[:] + pd.Series(range(1,23)).astype(str).str[:]
-common_chr = pd.concat([common_chr, pd.Series(['chrX','chrY','chrM'])]).to_numpy()
-
-bed_file = bed_file[bed_file[0].isin(common_chr)]
+if not args.ignore_chr:
+    common_chr = pd.Series(['chr']*22).str[:] + pd.Series(range(1,23)).astype(str).str[:]
+    common_chr = pd.concat([common_chr, pd.Series(['chrX','chrY','chrM'])]).to_numpy()
+    bed_file = bed_file[bed_file[0].isin(common_chr)]
 
 bed_f = bed_file[bed_file[5]=='+']
 bed_r = bed_file[bed_file[5]=='-']

{offtracker-2.7.10 → offtracker-2.10.0/offtracker.egg-info}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: offtracker
-Version: 2.7.10
+Version: 2.10.0
 Summary: Tracking-seq data analysis
 Home-page: https://github.com/Lan-lab/offtracker
 Author: Runda Xu
-Author-email: runda.xu@foxmail.com
+Author-email: xrd18@tsinghua.org.cn
 Requires-Python: >=3.6.0
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
@@ -22,9 +22,10 @@ OFF-TRACKER is an end to end pipeline of Tracking-seq data analysis for detectin
 ## Dependency
 
 ```bash
-# We recommend creating a new enviroment using mamba/conda to avoid compatibility problems
+# We recommend creating a new environment using mamba/conda to avoid compatibility problems
 # If you don't use mamba, just replace the code with conda 
-mamba create -n offtracker -c bioconda blast snakemake pybedtools
+# Windows systems may not be compatible with pybedtools.
+mamba create -n offtracker -c bioconda blast snakemake pybedtools chromap
 ```
 
 
@@ -58,32 +59,69 @@ chromap -i -r /Your_Path_To_Reference/hg38_genome.fa \
 -o /Your_Path_To_Reference/hg38_genome.chromap.index
 
 # Generate candidate regions by sgRNA sequence (need once for each genome and sgRNA)
-# --name: the name of the sgRNA, which will be used in the following analysis
+# --name: a user-defined name of the sgRNA, which will be used in the following analysis.
 offtracker_candidates.py -t 8 -g hg38 \
 -r /Your_Path_To_Reference/hg38_genome.fa \
 -b /Your_Path_To_Reference/hg38_genome.blastdb \
 --name 'VEGFA2' --sgrna 'GACCCCCTCCACCCCGCCTC' --pam 'NGG' \
--o /Your_Path_To_Candidates
+-o /Your_Path_To_Candidates_Folder
 
 ```
 
+
+## Quality control and adapter trimming
+
+```bash
+# Generate snakemake config file for quality control and adapter trimming.
+offtracker_qc.py -t 4 \
+-f /Your_Path_To_Input_Folder \
+--subfolder 0
+
+cd /Your_Path_To_Input_Folder/Trimmed_data
+snakemake -np # dry run to check whether everything is alright
+nohup snakemake --cores 16 1>${outdir}/sm_qc.log 2>&1 &
+
+"""
+Set “--subfolder 0” if the file structure is like: 
+| - Input_Folder
+  | - sample1_R1.fastq.gz
+  | - sample1_R2.fastq.gz
+  | - sample2_R1.fastq.gz
+  | - sample2_R2.fastq.gz
+Set “--subfolder 1” if the file structure is like:
+| - Input_Folder
+  | - Sample1_Folder
+    | - sample1_R1.fastq.gz
+    | - sample1_R2.fastq.gz
+  | - Sample2_Folder
+    | - sample2_R1.fastq.gz
+    | - sample2_R2.fastq.gz
+
+The script “offtracker_qc.py” will create a “Trimmed_data” folder under /Your_Path_To_Input_Folder. 
+If “-o /Your_Path_To_Output” is set, the output will be redirected to /Your_Path_To_Output.
+"""
+```
+
 ## Strand-specific mapping of Tracking-seq data 
 
 ```bash
-# Generate snakemake config file 
-# --subfolder: If different samples are in seperate folders, set this to 1
-# if -o is not set, the output will be in the same folder as the fastq files
+
+# Generate snakemake config file for mapping
+# Results will be generated in /Your_Path_To_Output, if -o is not set, the output will be in the same folder as the fastq files
 offtracker_config.py -t 8 -g hg38 --blacklist hg38 \
 -r /Your_Path_To_Reference/hg38_genome.fa \
 -i /Your_Path_To_Reference/hg38_genome.chromap.index \
--f /Your_Path_To_Fastq \
+-f /Your_Path_To_Trimmed_Data \
 -o /Your_Path_To_Output \ 
 --subfolder 0 
 
+# Warning: Do not contain "fastq" or "fq" in the folder name, otherwise the program may treat the folder as a fastq file
+# This problem may be fixed in the future
+
 # Run the snakemake program
 cd /Your_Path_To_Fastq
 snakemake -np # dry run
-nohup snakemake --cores 16 1>snakemake.log 2>snakemake.err &
+nohup snakemake --cores 16 1>sm_mapping.log 2>sm_mapping.err &
 
 ## about cores
 # --cores of snakemake must be larger than -t of offtracker_config.py
@@ -98,7 +136,7 @@ nohup snakemake --cores 16 1>snakemake.log 2>snakemake.err &
 ## Analyzing the genome-wide off-target sites
 
 ```bash
-# In this part, multiple samples in the same condition can be analyzed in a single run by pattern recogonization of sample names
+# In this part, multiple samples in the same condition can be analyzed in a single run by pattern recognition of sample names
 
 offtracker_analysis.py -g hg38 --name "VEGFA2" \
 --exp 'Cas9_VEGFA2' \
@@ -127,19 +165,18 @@ offtracker_plot.py --result Your_Offtracker_Result_CSV \
 --sgrna 'GACCCCCTCCACCCCGCCTC' --pam 'NGG'
 
 # The default output is a pdf file with Offtracker_result_{outname}.pdf
-# Change the suffix of the output file to change the format (e.g.: .png)
+# Assigning a specific output file with another suffix can change the format. e.g., "--output Offtracker_plot.png" will generate a png file.
 # The orange dash line indicates the empirical threshold of Track score = 2
 # Empirically, the off-target sites with Track score < 2 are less likely to be real off-target sites.
 ```
 
 
-## Note1
+## Note1, when not using hg38 or mm10
 
-The default setting only includes chr1-chr22, chrX, chrY, and chrM. Please make sure the reference genome contains "chr" at the beginning. 
+The default setting only includes chr1-chr22, chrX, chrY, and chrM. (only suitable for human and mouse) \
+If you are using reference genomes without "chr" at the beginning, or want to analyze all chromosomes or other species, you can set "--ignore_chr" when running offtracker_config.py to skip chromosome filter.
 
-Currently, this software is only ready-to-use for mm10 and hg38. For any other genome, e.g., hg19, please add genome size file named "hg19.chrom.sizes" to .\offtracker\mapping and instal manually. Besides, add "--blacklist none" or "--blacklist Your_Blacklist" (e.g., ENCODE blacklist) when running offtracker_config.py, because we only provide blacklists for mm10 and hg38.
-
-If you have a requirement for species other than human/mouse, please post an issue.
+Currently, this software is only ready-to-use for mm10 and hg38. For any other genome, e.g., hg19, please add a genome size file named "hg19.chrom.sizes" to .\offtracker\utility. Besides, add "--blacklist none" or "--blacklist Your_Blacklist" (e.g., ENCODE blacklist) when running offtracker_config.py, because we only include blacklists for mm10 and hg38.
 
 ## Note2
 
@@ -172,6 +209,7 @@ These files can be visualized in genome browser like IGV:
 
 ![signal](https://github.com/Lan-lab/offtracker/blob/main/example_output/signals_example.png?raw=true)
 
+The signal (coverage) for each sample is normalized to 1e7/total_reads. As only reads mapping to chr6 were extracted in the example data, the signal range is much higher than that of the whole genome samples. 
 
 ## Whole genome off-target analysis
 
@@ -183,7 +221,13 @@ After that, you can visualize the off-target sites with their genomic sequence (
 
 # Citation
 
+If you use Tracking-seq or OFF-TRACKER in your research, please cite the following paper:
+
+Zhu, M., Xu, R., Yuan, J., Wang, J. et al. Tracking-seq reveals the heterogeneity of off-target effects in CRISPR–Cas9-mediated genome editing. Nat Biotechnol (2024). https://doi.org/10.1038/s41587-024-02307-y
 
+The signal visualization of .bw file here was generated by the Integrative Genomics Viewer (IGV) software. The signal visualization in the Tracking-seq article above was generated by either IGV or pyGenomeTracks:
 
+Robinson, J., Thorvaldsdóttir, H., Winckler, W. et al. Integrative genomics viewer. Nat Biotechnol 29, 24–26 (2011). https://doi.org/10.1038/nbt.1754
 
+Lopez-Delisle L, Rabbani L, Wolff J, Bhardwaj V, Backofen R, Grüning B, Ramírez F, Manke T. pyGenomeTracks: reproducible plots for multivariate genomic data sets. Bioinformatics. 2020 Aug 3:btaa692. doi: 10.1093/bioinformatics/btaa692.
 

offtracker-2.10.0/offtracker.egg-info/SOURCES.txt

@@ -0,0 +1,28 @@
+LICENSE.txt
+MANIFEST.in
+README.md
+setup.py
+offtracker/X_offplot.py
+offtracker/X_offtracker.py
+offtracker/X_sequence.py
+offtracker/__init__.py
+offtracker/_version.py
+offtracker.egg-info/PKG-INFO
+offtracker.egg-info/SOURCES.txt
+offtracker.egg-info/dependency_links.txt
+offtracker.egg-info/requires.txt
+offtracker.egg-info/top_level.txt
+offtracker/snakefile/Snakefile_QC.smk
+offtracker/snakefile/Snakefile_offtracker.smk
+offtracker/utility/1.1_bed2fr.py
+offtracker/utility/1.3_bdg_normalize_v4.0.py
+offtracker/utility/bedGraphToBigWig
+offtracker/utility/hg38.chrom.sizes
+offtracker/utility/mm10.chrom.sizes
+offtracker/utility/offtracker_blacklist_hg38.merged.bed
+offtracker/utility/offtracker_blacklist_mm10.merged.bed
+scripts/offtracker_analysis.py
+scripts/offtracker_candidates.py
+scripts/offtracker_config.py
+scripts/offtracker_plot.py
+scripts/offtracker_qc.py

{offtracker-2.7.10 → offtracker-2.10.0}/scripts/offtracker_analysis.py

@@ -27,6 +27,7 @@ def main():
     parser.add_argument('--exp'          , type=str, default='all',    nargs='+', help='A substring mark in the name of experimental samples. The default is to use all samples other than control' )
     parser.add_argument('--control'      , type=str, default='none',   nargs='+', help='A substring mark in the name of control samples. The default is no control. "others" for all samples other than --exp.' )
     parser.add_argument('--fdr'          , type=int, default=0.05,     help='FDR threshold for the final result. Default is 0.05.')
+    parser.add_argument('--score'        , type=int, default=2,        help='Track score threshold for the final result. Default is 2.')
     parser.add_argument('--smooth'       , type=int, default=1,        help='Smooth strength for the signal.')
     parser.add_argument('--window'       , type=int, default=3,        help='Window size for smoothing the signal.')
     parser.add_argument('--binsize'      , type=int, default=100,      help='Window size for smoothing the signal.')
@@ -42,6 +43,7 @@ def main():
     parser.add_argument('--overwrite'    , action='store_true', help='Whether to overwrite existed dataframes.' )
     parser.add_argument('--clean'        , action='store_true', help='Whether to remove temp files')
 
+
     args = parser.parse_args()
 
     print(f'Runing offtracker verision: {offtracker.__version__}')
@@ -51,6 +53,7 @@ def main():
     pattern_exp = args.exp
     pattern_ctr = args.control
     fdr_thresh = args.fdr
+    score_thresh = args.score
     binsize = args.binsize
     flank_max = args.flank_max
     flank_regions = args.flank_regions
@@ -95,6 +98,8 @@ def main():
         all_sample_files.extend( bdg_files )
     all_sample_files = pd.Series(all_sample_files)
     all_sample_names = pd.Series(all_sample_names)
+    print('all sample names in the folders:')
+    print(all_sample_names)
     print('your string pattern for experimental groups: ', pattern_exp)
     ctr_samples = []
     if pattern_ctr == 'none':
@@ -341,14 +346,16 @@ def main():
         print('mean_score:{:.3f};std:{:.3f}'.format(mu,std))
         # pv and fdr
         df_result['pv'] = df_result[f'log2_track_score'].apply( lambda x: norm.sf(x,loc=mu,scale=std) )
-        df_result['pv'].clip(lower=1e-320,inplace=True)
+        df_result['pv'] = df_result['pv'].clip(lower=1e-320)
         df_result['fdr'] = offtracker.fdr(df_result['pv'])
         df_result['rank'] = range(1,len(df_result)+1)
         df_result.to_csv(output)        
         # 2024.06.03. 以防 fdr<=fdr_thresh 滤掉了 track_score>=2 的位点
         bool_fdr = df_result['fdr']<=fdr_thresh
-        bool_score = df_result['track_score']>=2
-        df_output = df_result[bool_fdr|bool_score].copy()
+        bool_score = df_result['track_score']>=score_thresh
+        # 2025.06.05. BE可能会形成单边信号，导致 track_score 为负数，也保留
+        bool_neg_score = df_result['track_score']<0
+        df_output = df_result[bool_fdr|bool_score|bool_neg_score].copy()
         if pattern_ctr != 'none':
             df_output = df_output[['target_location', 'best_strand','best_target','deletion','insertion','mismatch',
                                    'exp_L_length', 'exp_R_length','ctr_L_length','ctr_R_length','L_length','R_length','signal_length',