offtracker 2.7.8__zip → 2.10.0__zip

{offtracker-2.7.8 → offtracker-2.10.0}/scripts/offtracker_config.py

@@ -1,20 +1,22 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# 2023.08.11. v1.1	adding a option for not normalizing the bw file
+# 2023.08.11. adding a option for not normalizing the bw file
+# 2025.05.22. refine the structure
+# 2025.06.05. 增加 ignore_chr 选项，默认只取 common chromosomes，用于 1.1_bed2fr.py
 
 import argparse
 import os, glob, yaml
 import pandas as pd
 import shutil, re
 import offtracker
+import offtracker.X_sequence as xseq
 script_dir = os.path.abspath(os.path.dirname(offtracker.__file__))
-script_folder= os.path.join(script_dir, 'mapping')
-os.chmod( os.path.join(script_folder, 'bedGraphToBigWig'), 0o755)
+utility_dir = os.path.join(script_dir, 'utility')
 
 ###
 parser = argparse.ArgumentParser()
-parser.description='Mapping fastq files of Track-seq.'
+parser.description='Mapping fastq files of Tracking-seq.'
 parser.add_argument('-f','--folder', type=str, required=True,  help='Directory of the input folder' )
 parser.add_argument('-r','--ref'   , type=str, required=True,  help='The fasta file of reference genome')
 parser.add_argument('-i','--index' , type=str, required=True,  help='The index file of chromap')
@@ -25,12 +27,13 @@ parser.add_argument('-t','--thread', type=int, default=4,      help='Number of t
 parser.add_argument('--blacklist'  , type=str, default='same', help='Blacklist of genome regions in bed format. "none" for no filter')
 parser.add_argument('--binsize'    , type=str, default=100,    help='Bin size for calculating bw residue')
 parser.add_argument('--normalize'  , type=str, default='True', help='Whether to normalize the BigWig file. "True" or "False"')
+parser.add_argument('--ignore_chr' , action='store_true', help='If not set, only chr1-chr22, chrX, chrY, chrM will be analyzed.')
 
-args = parser.parse_args()
 
+args = parser.parse_args()
 
 if (args.genome == 'hg38') or (args.genome == 'mm10'):
-    dir_chrom_sizes = os.path.join(script_folder, f'{args.genome}.chrom.sizes')
+    dir_chrom_sizes = os.path.join(utility_dir, f'{args.genome}.chrom.sizes')
 else:
     dir_chrom_sizes = args.genome
 
@@ -42,7 +45,7 @@ if args.blacklist == 'same':
     args.blacklist = args.genome
     
 if (args.blacklist == 'hg38') or (args.blacklist == 'mm10'):
-    blacklist = os.path.join(script_folder, f'offtracker_blacklist_{args.blacklist}.merged.bed')
+    blacklist = os.path.join(utility_dir, f'offtracker_blacklist_{args.blacklist}.merged.bed')
 else:
     blacklist = args.blacklist
 
@@ -52,59 +55,40 @@ else:
     if not os.path.exists(args.outdir):
         os.makedirs(args.outdir)
 
-gz_R2 = []
-for fastq in ['*2.*fq','*2.*fastq','*2.*fq.gz','*2.*fastq.gz']:
-    fq_files = glob.glob( os.path.join(args.folder, args.subfolder*'*/', fastq ) )
-    print('{} {} samples detected'.format( len(fq_files), fastq[4:] ) )
-    gz_R2.extend( fq_files )
-
-gz_R2.sort()
-gz_R2 = pd.Series(gz_R2)
-suffix = gz_R2.str.extract('(fastq.*|fq.*)',expand=False)
-prefix = gz_R2.str.extract('(.*)(?:.fq|.fastq)',expand=False)
-
-nametype = None
-for a_type in ['_trimmed_2', '_2_val_2','_R2_val_2','_R2','_2']:
-    len_type = len(a_type)
-    if prefix[0][-len_type:] == a_type:
-        nametype = a_type
-        sample_dir = prefix.str[:-len_type]
-        break
-
-if nametype is None:
-    # pattern 搜索模式，可能会出 bug
-    # find "_R2." or "_2." in prefix[0]
-    pattern = re.compile(r'(_R2\.|_2\.)')
-    m = pattern.search(prefix[0])
-    if m:
-        nametype = prefix[0][m.span()[0]:]
-        len_type = len(nametype)
-        sample_dir = prefix.str[:-len_type]
-
-assert nametype is not None, 'No fastq detected or the file name is invaild!'
-
-sample_name = sample_dir.apply(os.path.basename)
+if args.ignore_chr:
+    args.ignore_chr = '--ignore_chr'
+else:
+    args.ignore_chr = ''
+
+# 搜索 folder 的 n级子目录下的所有 fastq/fastq.gz/fq/fq.gz 文件
+sample_names, files_R1, files_R2 = xseq.detect_fastq(args.folder, n_subfolder=args.subfolder, NGS_type=args.NGS_type)
+
+assert not isinstance(sample_names, str), 'No fastq file is detected!'
 
 dict_yaml = {
-    'suffix':suffix[0],
-    'sample':dict(zip(sample_name,sample_dir)),
+    # fastq 信息
+    'files_R1':dict(zip(sample_names,files_R1)),
+    'files_R2':dict(zip(sample_names,files_R2)), # 单端 files_R2=[] 结果会自动为 {}
+    'NGS_type':args.NGS_type,
+    # 输入输出文件夹
     'input_dir':args.folder,
     'output_dir':args.outdir,
+    # 运行参数
     'thread':args.thread,
     'index':args.index,
     'fasta':args.ref,
     'binsize':args.binsize,
     'blacklist':blacklist,
-    'nametype':nametype,
     'genomelen':dir_chrom_sizes,
     'normalize':args.normalize,
-    'script_folder':script_folder
+    'utility_dir':utility_dir,
+    'ignore_chr':args.ignore_chr,
     }
 
 with open( os.path.join(args.outdir,'config.yaml'), 'w') as outfile:
     yaml.dump(dict_yaml, outfile, default_flow_style=False)
 
-snakefile = os.path.join(script_dir, 'mapping/Snakefile_offtracker')
+snakefile = os.path.join(script_dir, 'snakefile/Snakefile_offtracker.smk')
 shutil.copy(snakefile, os.path.join(args.outdir,'Snakefile'))
 
 

offtracker-2.10.0/scripts/offtracker_plot.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import offtracker.X_offplot as xoffplot
+import pandas as pd
+import argparse
+import os
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.description='Draw the plot of the off-targets with genomic sequences.\nIf .pdf file is too large, try to use .png file instead.'
+    parser.add_argument('--result' , type=str, required=True,  help='The file of Offtracker_result_{outname}.csv' )
+    parser.add_argument('--sgrna'  , type=str, required=True,  help='Not including PAM' )
+    parser.add_argument('--pam'    , type=str, default='NGG',  help='PAM sequence. Default is "NGG".' )
+    parser.add_argument('--output' , type=str, default='same', help='The output file. Default is Offtracker_result_{outname}.pdf')
+
+    args = parser.parse_args()
+    if args.output == 'same':
+        dir_savefig = args.result.replace('.csv', '.pdf')
+    else:
+        dir_savefig = args.output
+
+    outname = os.path.basename(args.result).replace('Offtracker_result_', '').replace('.csv', '')
+    gRNA = args.sgrna
+    PAM = args.pam
+    full_seq = gRNA + PAM
+    
+    df_result = pd.read_csv(args.result)
+    n_pos = len(df_result)
+
+    xoffplot.offtable(df_result, full_seq, length_pam = len(PAM), col_seq='target', threshold=2,
+                      title=f'{outname} ({n_pos} sites)',
+                      savefig=dir_savefig)
+    
+    return f'The plot is saved as {dir_savefig}'
+
+if __name__ == '__main__' :
+    result = main()
+    print(result)

offtracker-2.10.0/scripts/offtracker_qc.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+THIS_VERSION = '0.4.1'
+
+import argparse
+import os, glob, yaml
+import pandas as pd
+import shutil, re
+import offtracker
+import offtracker.X_sequence as xseq
+
+script_dir = os.path.abspath(os.path.dirname(offtracker.__file__))
+utility_dir = os.path.join(script_dir, 'utility')
+os.chmod( os.path.join(utility_dir, 'bedGraphToBigWig'), 0o755)
+
+###
+parser = argparse.ArgumentParser()
+parser.description=f'xbulk_qc v{THIS_VERSION}. QC and trim fastq files.'
+parser.add_argument('-f','--folder', type=str, required=True,        help='Directory of the input folder' )
+parser.add_argument('-o','--outdir', type=str, default='same',       help='The output folder')
+parser.add_argument('--subfolder'  , type=int, default=0,            help='subfolder level')
+parser.add_argument('-t','--thread', type=int, default=8,            help='Number of threads to be used')
+parser.add_argument('--NGS_type'   , type=str, default='paired-end', help='paired-end or single-end')
+
+args = parser.parse_args()
+
+# 自动化的参数调整和报错
+if args.outdir == 'same':
+    args.outdir = os.path.join(args.folder,'Trimmed_data')
+    if not os.path.exists( args.outdir ):
+        os.makedirs( args.outdir )
+else:
+    if not os.path.exists(args.outdir):
+        os.makedirs(args.outdir)
+
+# 搜索 folder 的 n级子目录下的所有 fastq/fastq.gz/fq/fq.gz 文件
+sample_names, files_R1, files_R2 = xseq.detect_fastq(args.folder, n_subfolder=args.subfolder, NGS_type=args.NGS_type)
+
+assert not isinstance(sample_names, str), 'No fastq file is detected!'
+
+dict_yaml = {
+    # fastq 信息
+    'files_R1':dict(zip(sample_names,files_R1)),
+    'files_R2':dict(zip(sample_names,files_R2)), # 单端 files_R2=[] 结果会自动为 {}
+    'NGS_type':args.NGS_type,
+    # 输入输出文件夹
+    'input_dir':args.folder,
+    'output_dir':args.outdir,
+    # 运行参数
+    'thread':args.thread,
+    'utility_dir':utility_dir
+    }
+
+
+with open( os.path.join(args.outdir,'config.yaml'), 'w', encoding='utf-8') as outfile:
+    yaml.dump(dict_yaml, outfile, default_flow_style=False)
+
+snakefile = os.path.join(script_dir, 'snakefile/Snakefile_QC.smk')
+shutil.copy(snakefile, os.path.join(args.outdir,'Snakefile'))
+
+

{offtracker-2.7.8 → offtracker-2.10.0}/setup.py

@@ -11,7 +11,7 @@ from setuptools import find_packages, setup, Command
 NAME = 'offtracker'
 DESCRIPTION = 'Tracking-seq data analysis'
 AUTHOR = 'Runda Xu'
-EMAIL = 'runda.xu@foxmail.com'
+EMAIL = 'xrd18@tsinghua.org.cn'
 URL = 'https://github.com/Lan-lab/offtracker'
 REQUIRES_PYTHON = '>=3.6.0'
 
@@ -47,9 +47,13 @@ setup(
     author_email=EMAIL,
     url=URL,
     python_requires=REQUIRES_PYTHON,
-    packages=find_packages(),
-    package_data={'offtracker': ['mapping/*']},
-    scripts = ['scripts/offtracker_config.py','scripts/offtracker_candidates.py','scripts/offtracker_analysis.py'],
+    packages=['offtracker'],
+    package_data={'offtracker': ['snakefile/*','utility/*']},
+    scripts = ['scripts/offtracker_qc.py',
+               'scripts/offtracker_config.py',
+               'scripts/offtracker_candidates.py',
+               'scripts/offtracker_analysis.py',
+               'scripts/offtracker_plot.py'],
     install_requires=REQUIRED,
     include_package_data=True
 )

offtracker-2.7.8/PKG-INFO

@@ -1,146 +0,0 @@
-Metadata-Version: 2.1
-Name: offtracker
-Version: 2.7.8
-Summary: Tracking-seq data analysis
-Home-page: https://github.com/Lan-lab/offtracker
-Author: Runda Xu
-Author-email: runda.xu@foxmail.com
-Requires-Python: >=3.6.0
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
-
-
-OFF-TRACKER
-=======================
-
-OFF-TRACKER is an end to end pipeline of Tracking-seq data analysis for detecting off-target sites of any genome editing tools that generate double-strand breaks (DSBs) or single-strand breaks (SSBs).
-
-System requirements
------
-* Linux/Unix 
-* Python >= 3.6
-
-Dependency
------
-
-```bash
-# We recommend creating a new enviroment using mamba/conda to avoid compatibility problems
-# If you don't use mamba, just replace the code with conda 
-mamba create -n offtracker -c bioconda blast snakemake pybedtools
-```
-
-
-Installation 
------
-
-```bash
-# activate the environment
-conda activate offtracker
-
-# Direct installation with pip
-pip install offtracker
-
-# (Alternative) Download the offtracker from github
-git clone https://github.com/Lan-lab/offtracker.git 
-cd offtracker
-pip install .
-```
-
-
-Before analyzing samples
------
-
-```bash
-# Build blast index (only need once for each genome)
-makeblastdb -input_type fasta -title hg38 -dbtype nucl -parse_seqids \
--in /Your_Path_To_Reference/hg38_genome.fa \
--out /Your_Path_To_Reference/hg38_genome.blastdb \
--logfile /Your_Path_To_Reference/hg38_genome.blastdb.log
-
-# Build chromap index (only need once for each genome)
-chromap -i -r /Your_Path_To_Reference/hg38_genome.fa \
--o /Your_Path_To_Reference/hg38_genome.chromap.index
-
-# Generate candidate regions by sgRNA sequence (need once for each genome and sgRNA)
-offtracker_candidates.py -t 8 -g hg38 \
--r /Your_Path_To_Reference/hg38_genome.fa \
--b /Your_Path_To_Reference/hg38_genome.blastdb \
---name 'HEK4' --sgrna 'GGCACTGCGGCTGGAGGTGG' --pam 'NGG' \
--o /Your_Path_To_Candidates
-
-```
-
-Strand-specific mapping of Tracking-seq data 
------
-
-```bash
-# Generate snakemake config file 
-offtracker_config.py -t 8 -g hg38 --blacklist hg38 \
--r /Your_Path_To_Reference/hg38_genome.fa \
--i /Your_Path_To_Reference/hg38_genome.chromap.index \
--f /Your_Path_To_Fastq \
--o /Your_Path_To_Output \ 
---subfolder 0 
-
-# --subfolder: If different samples are in seperate folders, set this to 1
-# -o: Default is outputting to /Your_Path_To_Fastq
-
-# Run the snakemake program
-cd /Your_Path_To_Fastq
-snakemake -np # dry run
-nohup snakemake --cores 16 1>snakemake.log 2>snakemake.err &
-
-## about cores
-# --cores of snakemake must be larger than -t of offtracker_config.py
-# parallel number = cores/t
-
-## about output
-# This part will generate "*.fw.scaled.bw" and ".rv.scaled.bw" for IGV visualization
-# "*.fw.bed" and "*.rv.bed" are used in the next part.
-```
-
-
-Analyzing the off-target sites
------
-
-```bash
-# In this part, multiple samples in the same condition can be analyzed in a single run by pattern recogonization of sample names
-
-offtracker_analysis.py -g hg38 --name "HEK4" \
---exp 'Cas9_HEK4.*293' \
---control 'control' \
---outname 'Cas9_HEK4_293' \
--f /Your_Path_To_Output \
---seqfolder /Your_Path_To_Candidates
-
-# --name: the same as that in offtracker_candidates.py
-# --exp/--control: add one or multiple patterns of file name in regex
-
-
-# This step will generate Trackseq_result_{outname}.csv
-# Intermediate files are saved in ./temp folder, which can be deleted 
-# Keeping the intermediate files can make the analysis faster if involving previously analyzed samples (e.g. using the same control samples for different analyses)
-```
-
-
-Note1
---------------
-The default setting only includes chr1-chr22, chrX, chrY, and chrM.
-
-Please make sure the reference genome contains "chr" at the beginning. 
-
-If you have requirement for other chromosomes or species other than human/mouse, please post an issue.
-
-Note2
---------------
-Currently, this software is only ready-to-use for mm10 and hg38. 
-
-For any other genome, say hg19, please add genome size file named "hg19.chrom.sizes" to .\offtracker\mapping before install.
-
-Besides, add "--blacklist none" or "--blacklist Your_Blacklist" when running offtracker_config.py
-
-Note3
---------------
-The FDR in the Tracking-seq result is not rigorous to the real off-target probability.
-It is strongly recommended to observe the "fw.scaled.bw" and "rv.scaled.bw" using IGV to check each target location from the Tracking-seq result.
-

offtracker-2.7.8/README.md

@@ -1,134 +0,0 @@
-OFF-TRACKER
-=======================
-
-OFF-TRACKER is an end to end pipeline of Tracking-seq data analysis for detecting off-target sites of any genome editing tools that generate double-strand breaks (DSBs) or single-strand breaks (SSBs).
-
-System requirements
------
-* Linux/Unix 
-* Python >= 3.6
-
-Dependency
------
-
-```bash
-# We recommend creating a new enviroment using mamba/conda to avoid compatibility problems
-# If you don't use mamba, just replace the code with conda 
-mamba create -n offtracker -c bioconda blast snakemake pybedtools
-```
-
-
-Installation 
------
-
-```bash
-# activate the environment
-conda activate offtracker
-
-# Direct installation with pip
-pip install offtracker
-
-# (Alternative) Download the offtracker from github
-git clone https://github.com/Lan-lab/offtracker.git 
-cd offtracker
-pip install .
-```
-
-
-Before analyzing samples
------
-
-```bash
-# Build blast index (only need once for each genome)
-makeblastdb -input_type fasta -title hg38 -dbtype nucl -parse_seqids \
--in /Your_Path_To_Reference/hg38_genome.fa \
--out /Your_Path_To_Reference/hg38_genome.blastdb \
--logfile /Your_Path_To_Reference/hg38_genome.blastdb.log
-
-# Build chromap index (only need once for each genome)
-chromap -i -r /Your_Path_To_Reference/hg38_genome.fa \
--o /Your_Path_To_Reference/hg38_genome.chromap.index
-
-# Generate candidate regions by sgRNA sequence (need once for each genome and sgRNA)
-offtracker_candidates.py -t 8 -g hg38 \
--r /Your_Path_To_Reference/hg38_genome.fa \
--b /Your_Path_To_Reference/hg38_genome.blastdb \
---name 'HEK4' --sgrna 'GGCACTGCGGCTGGAGGTGG' --pam 'NGG' \
--o /Your_Path_To_Candidates
-
-```
-
-Strand-specific mapping of Tracking-seq data 
------
-
-```bash
-# Generate snakemake config file 
-offtracker_config.py -t 8 -g hg38 --blacklist hg38 \
--r /Your_Path_To_Reference/hg38_genome.fa \
--i /Your_Path_To_Reference/hg38_genome.chromap.index \
--f /Your_Path_To_Fastq \
--o /Your_Path_To_Output \ 
---subfolder 0 
-
-# --subfolder: If different samples are in seperate folders, set this to 1
-# -o: Default is outputting to /Your_Path_To_Fastq
-
-# Run the snakemake program
-cd /Your_Path_To_Fastq
-snakemake -np # dry run
-nohup snakemake --cores 16 1>snakemake.log 2>snakemake.err &
-
-## about cores
-# --cores of snakemake must be larger than -t of offtracker_config.py
-# parallel number = cores/t
-
-## about output
-# This part will generate "*.fw.scaled.bw" and ".rv.scaled.bw" for IGV visualization
-# "*.fw.bed" and "*.rv.bed" are used in the next part.
-```
-
-
-Analyzing the off-target sites
------
-
-```bash
-# In this part, multiple samples in the same condition can be analyzed in a single run by pattern recogonization of sample names
-
-offtracker_analysis.py -g hg38 --name "HEK4" \
---exp 'Cas9_HEK4.*293' \
---control 'control' \
---outname 'Cas9_HEK4_293' \
--f /Your_Path_To_Output \
---seqfolder /Your_Path_To_Candidates
-
-# --name: the same as that in offtracker_candidates.py
-# --exp/--control: add one or multiple patterns of file name in regex
-
-
-# This step will generate Trackseq_result_{outname}.csv
-# Intermediate files are saved in ./temp folder, which can be deleted 
-# Keeping the intermediate files can make the analysis faster if involving previously analyzed samples (e.g. using the same control samples for different analyses)
-```
-
-
-Note1
---------------
-The default setting only includes chr1-chr22, chrX, chrY, and chrM.
-
-Please make sure the reference genome contains "chr" at the beginning. 
-
-If you have requirement for other chromosomes or species other than human/mouse, please post an issue.
-
-Note2
---------------
-Currently, this software is only ready-to-use for mm10 and hg38. 
-
-For any other genome, say hg19, please add genome size file named "hg19.chrom.sizes" to .\offtracker\mapping before install.
-
-Besides, add "--blacklist none" or "--blacklist Your_Blacklist" when running offtracker_config.py
-
-Note3
---------------
-The FDR in the Tracking-seq result is not rigorous to the real off-target probability.
-It is strongly recommended to observe the "fw.scaled.bw" and "rv.scaled.bw" using IGV to check each target location from the Tracking-seq result.
-

offtracker-2.7.8/offtracker/_version.py

@@ -1,28 +0,0 @@
-__version__ = "2.7.8"
-# 2023.08.11. v1.1.0	adding a option for not normalizing the bw file
-# 2023.10.26. v1.9.0	prerelease for v2.0
-# 2023.10.27. v2.0.0	大更新，还没微调
-# 2023.10.28. v2.1.0 修复bug，增加计算信号长度的功能
-# 2023.10.28. v2.2.0 修复bug，改变计算信号长度的算法
-# 2023.10.29. v2.3.0 增加 overall signal 计算
-# 2023.11.01. v2.3.1 增加 signal_only 选项
-# 2023.11.02. v2.3.2 修改 sample signal 和 group mean 的计算顺序
-# 2023.11.04. v2.3.3 修复 overall score 标准化时排序错误的问题
-# 2023.11.05. v2.3.4 修复判断单边溢出信号时的列名选取错误
-# 2023.11.13. v2.3.5 微调 track score
-# 2023.12.05. v2.3.6 candidates 增加 cleavage site，修正 alignment 有 deletion 会错位的 bug
-# 2023.12.05. v2.3.7 用 cleavage site 代替 midpoint # 还没改完
-# 2023.12.07. v2.3.8 df_score 增加 df_exp, df_ctr 各自列。修复没 df_ctr 时的 bug。track score 用 proximal
-# 2023.12.09. v2.4.0 为了兼顾 proximal 和 overall，当 normalized overall signal 高于 2 时，增加 overall signal 的加分
-# 2023.12.09. v2.5.0 尝试新的加权位置
-# 2023.12.10. v2.6.0 加入 trackseq v4 的计算分支，即考虑 Region 内的 positive_pct，避免短而尖锐的信号
-# 2023.12.10. v2.6.1 有些非特异信号数值很大，如果在 control 组是大负数，可能导致减 control 后假高信号，因此给负数一个 clip
-# 2023.12.30. v2.7.0 增加 X_offplot 模块，用于绘图
-# 2023.12.31. v2.7.1 control 的负数值 clip 由 -5 改为 -1，进一步减少假阳性。另外不加 overall 了
-# 2024.01.01. v2.7.2 权重改为 proximal + pct = 1 + 1. 防信号外溢假阳性标准由<0改为<=0
-# 2024.01.02. v2.7.3 flank regions 默认值改为 1000 2000 3000 5000。之前 control 的负数值 clip 相当于直接在 final score，现在改为每个单独 clip 后重新算 score，默认值为 CtrClip=-0.5
-# 2024.01.03. v2.7.4 更新了 blacklist.bed
-# 2024.01.04. v2.7.5 更新了 hg38 blacklist.bed
-# 2024.01.12. v2.7.6 修复小bug，输出 fdr 改为 <0.05。
-# 2024.01.23. v2.7.7 Snakefile_offtracker: add --fixedStep to bigwigCompare for not merging neighbouring bins with equal values.
-# 2024.02.01. v2.7.8 逐步添加 X_offplot.py 功能